annotate python/yacc.py @ 323:e9fe6988497c

Used burg for generating expressions
author Windel Bouwman
date Thu, 30 Jan 2014 19:03:24 +0100
parents 44f336460c2a
children 6f4753202b9a
rev   line source
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
1 #!/usr/bin/python
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
2
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
3 """
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
4 Parser generator utility. This script can generate a python script from a
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
5 grammar description.
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
6
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
7 Invoke the script on a grammar specification file:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
8
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
9 .. code::
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
10
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
11 $ ./yacc.py test.x -o test_parser.py
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
12
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
13 And use the generated parser by deriving a user class:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
14
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
15
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
16 .. code::
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
17
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
18 import test_parser
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
19 class MyParser(test_parser.Parser):
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
20 pass
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
21 p = MyParser()
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
22 p.parse()
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
23
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
24
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
25 Alternatively you can load the parser on the fly:
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
26
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
27 .. code::
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
28
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
29 import yacc
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
30 parser_mod = yacc.load_as_module('mygrammar.x')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
31 class MyParser(parser_mod.Parser):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
32 pass
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
33 p = MyParser()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
34 p.parse()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
35
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
36 """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
37
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
38 import argparse
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
39 import re
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
40 import sys
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
41 import datetime
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
42 import types
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
43 import io
323
e9fe6988497c Used burg for generating expressions
Windel Bouwman
parents: 322
diff changeset
44 import logging
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
45 from pyyacc import Grammar, print_grammar
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
46
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
47
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
48 class XaccLexer:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
49 def __init__(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
50 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
51
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
52 def feed(self, txt):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
53 # Create a regular expression for the lexing part:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
54 tok_spec = [
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
55 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
56 ('STRING', r"'[^']*'"),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
57 ('BRACEDCODE', r"\{[^\}]*\}"),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
58 ('OTHER', r'[:;\|]'),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
59 ('SKIP', r'[ ]')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
60 ]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
61 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
62 gettok = re.compile(tok_re).match
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
63
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
64 lines = txt.split('\n')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
65
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
66 def tokenize_line(line):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
67 """ Generator that splits up a line into tokens """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
68 mo = gettok(line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
69 pos = 0
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
70 while mo:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
71 typ = mo.lastgroup
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
72 val = mo.group(typ)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
73 if typ == 'ID':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
74 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
75 elif typ == 'STRING':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
76 typ = 'ID'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
77 yield (typ, val[1:-1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
78 elif typ == 'OTHER':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
79 typ = val
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
80 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
81 elif typ == 'BRACEDCODE':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
82 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
83 elif typ == 'SKIP':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
84 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
85 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
86 raise NotImplementedError(str(typ))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
87 pos = mo.end()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
88 mo = gettok(line, pos)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
89 if len(line) != pos:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
90 raise ParseError('Lex fault at {}'.format(line))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
91
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
92 def tokenize():
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
93 section = 0
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
94 for line in lines:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
95 line = line.strip()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
96 if not line:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
97 continue # Skip empty lines
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
98 if line == '%%':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
99 section += 1
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
100 yield('%%', '%%')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
101 continue
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
102 if section == 0:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
103 if line.startswith('%tokens'):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
104 yield('%tokens', '%tokens')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
105 yield from tokenize_line(line[7:])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
106 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
107 yield ('HEADER', line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
108 elif section == 1:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
109 yield from tokenize_line(line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
110 yield ('eof', 'eof')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
111 self.tokens = tokenize()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
112 self.token = self.tokens.__next__()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
113
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
114 def next_token(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
115 t = self.token
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
116 if t[0] != 'eof':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
117 self.token = self.tokens.__next__()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
118 return t
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
119
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
120
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
121 class ParseError(Exception):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
122 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
123
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
124
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
125 class XaccParser:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
126 """ Implements a recursive descent parser to parse grammar rules.
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
127 We could have made an generated parser, but that would yield a chicken
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
128 egg issue.
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
129 """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
130 def __init__(self, lexer):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
131 self.lexer = lexer
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
132
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
133 @property
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
134 def Peak(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
135 """ Sneak peak to the next token in line """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
136 return self.lexer.token[0]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
137
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
138 def next_token(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
139 """ Take the next token """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
140 return self.lexer.next_token()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
141
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
142 def consume(self, typ):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
143 """ Eat next token of type typ or raise an exception """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
144 if self.Peak == typ:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
145 return self.next_token()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
146 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
147 raise ParseError('Expected {}, but got {}'.format(typ, self.Peak))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
148
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
149 def has_consumed(self, typ):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
150 """ Consume typ if possible and return true if so """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
151 if self.Peak == typ:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
152 self.consume(typ)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
153 return True
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
154 return False
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
155
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
156 def parse_grammar(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
157 """ Entry parse function into recursive descent parser """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
158 # parse header
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
159 headers = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
160 terminals = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
161 while self.Peak in ['HEADER', '%tokens']:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
162 if self.Peak == '%tokens':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
163 self.consume('%tokens')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
164 while self.Peak == 'ID':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
165 terminals.append(self.consume('ID')[1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
166 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
167 headers.append(self.consume('HEADER')[1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
168 self.consume('%%')
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
169 self.headers = headers
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
170 self.grammar = Grammar(terminals)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
171 while self.Peak != 'eof':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
172 self.parse_rule()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
173 return self.grammar
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
174
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
175 def parse_symbol(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
176 return self.consume('ID')[1]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
177
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
178 def parse_rhs(self):
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
179 """ Parse the right hand side of a rule definition """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
180 symbols = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
181 while self.Peak not in [';', 'BRACEDCODE', '|']:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
182 symbols.append(self.parse_symbol())
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
183 if self.Peak == 'BRACEDCODE':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
184 action = self.consume('BRACEDCODE')[1]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
185 action = action[1:-1].strip()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
186 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
187 action = None
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
188 return symbols, action
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
189
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
190 def parse_rule(self):
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
191 """ Parse a rule definition """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
192 p = self.parse_symbol()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
193 self.consume(':')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
194 symbols, action = self.parse_rhs()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
195 self.grammar.add_production(p, symbols, action)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
196 while self.has_consumed('|'):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
197 symbols, action = self.parse_rhs()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
198 self.grammar.add_production(p, symbols, action)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
199 self.consume(';')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
200
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
201
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
202 class XaccGenerator:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
203 """ Generator that writes generated parser to file """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
204 def __init__(self):
323
e9fe6988497c Used burg for generating expressions
Windel Bouwman
parents: 322
diff changeset
205 self.logger = logging.getLogger('yacc')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
206
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
207 def generate(self, grammar, headers, output_file):
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
208 self.output_file = output_file
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
209 self.grammar = grammar
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
210 self.headers = headers
323
e9fe6988497c Used burg for generating expressions
Windel Bouwman
parents: 322
diff changeset
211 self.logger.info('Generating parser for grammar {}'.format(grammar))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
212 self.action_table, self.goto_table = grammar.doGenerate()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
213 self.generate_python_script()
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
214
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
215 def print(self, *args):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
216 """ Print helper function that prints to output file """
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
217 print(*args, file=self.output_file)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
218
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
219 def generate_python_script(self):
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
220 """ Generate python script with the parser table """
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
221 self.print('#!/usr/bin/python')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
222 stamp = datetime.datetime.now().ctime()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
223 self.print('""" Automatically generated by xacc on {} """'.format(stamp))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
224 self.print('from pyyacc import LRParser, Reduce, Shift, Accept, Production, Grammar')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
225 self.print('from ppci import Token')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
226 self.print('')
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
227 for h in self.headers:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
228 print(h, file=output_file)
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
229 self.print('')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
230 self.print('class Parser(LRParser):')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
231 self.print(' def __init__(self):')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
232 # Generate rules:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
233 self.print(' self.start_symbol = "{}"'.format(self.grammar.start_symbol))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
234 self.print(' self.grammar = Grammar({})'.format(self.grammar.terminals))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
235 for rule_number, rule in enumerate(self.grammar.productions):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
236 rule.f_name = 'action_{}_{}'.format(rule.name, rule_number)
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
237 self.print(' self.grammar.add_production("{}", {}, self.{})'.format(rule.name, rule.symbols, rule.f_name))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
238 # Fill action table:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
239 self.print(' self.action_table = {}')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
240 for state in self.action_table:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
241 action = self.action_table[state]
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
242 self.print(' self.action_table[{}] = {}'.format(state, action))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
243 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
244
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
245 # Fill goto table:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
246 self.print(' self.goto_table = {}')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
247 for gt in self.goto_table:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
248 to = self.goto_table[gt]
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
249 self.print(' self.goto_table[{}] = {}'.format(gt, to))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
250 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
251
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
252 # Generate a function for each action:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
253 for rule in self.grammar.productions:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
254 M = len(rule.symbols)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
255 args = ', '.join('arg{}'.format(n + 1) for n in range(M))
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
256 self.print(' def {}(self, {}):'.format(rule.f_name, args))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
257 if rule.f == None:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
258 semantics = 'pass'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
259 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
260 semantics = str(rule.f)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
261 if semantics.strip() == '':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
262 semantics = 'pass'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
263 for n in range(M):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
264 semantics = semantics.replace('${}'.format(n + 1), 'arg{}'.format(n + 1))
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
265 self.print(' {}'.format(semantics))
322
44f336460c2a Half of use of burg spec for arm
Windel Bouwman
parents: 321
diff changeset
266 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
267
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
268
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
269 def make_argument_parser():
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
270 # Parse arguments:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
271 parser = argparse.ArgumentParser(description='xacc compiler compiler')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
272 parser.add_argument('source', type=argparse.FileType('r'), \
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
273 help='the parser specification')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
274 parser.add_argument('-o', '--output', type=argparse.FileType('w'), \
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
275 default=sys.stdout)
322
44f336460c2a Half of use of burg spec for arm
Windel Bouwman
parents: 321
diff changeset
276 return parser
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
277
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
278
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
279 def load_as_module(filename):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
280 """ Load a parser spec file, generate LR tables and create module """
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
281 ob = io.StringIO()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
282 args = argparse.Namespace(source=open(filename), output=ob)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
283 main(args)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
284
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
285 parser_mod = types.ModuleType('generated_parser')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
286 exec(ob.getvalue(), parser_mod.__dict__)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
287 return parser_mod
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
288
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
289 def main(args):
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
290 src = args.source.read()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
291 args.source.close()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
292
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
293 # Construction of generator parts:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
294 lexer = XaccLexer()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
295 parser = XaccParser(lexer)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
296 generator = XaccGenerator()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
297
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
298 # Sequence source through the generator parts:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
299 lexer.feed(src)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
300 grammar = parser.parse_grammar()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
301 # TODO: optionize this: print_grammar(grammar)
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
302 generator.generate(grammar, parser.headers, args.output)
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
303
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
304
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
305 if __name__ == '__main__':
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
306 args = make_argument_parser().parse_args()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
307 main(args)