annotate python/yacc.py @ 321:8c569fbe60e4

Load yacc and burg dynamic
author Windel Bouwman
date Sun, 19 Jan 2014 18:48:45 +0100
parents 8d07a4254f04
children 44f336460c2a
rev   line source
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
1 #!/usr/bin/python
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
2
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
3 """
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
4 Parser generator utility. This script can generate a python script from a
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
5 grammar description.
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
6
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
7 Invoke the script on a grammar specification file:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
8
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
9 .. code::
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
10
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
11 $ ./yacc.py test.x -o test_parser.py
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
12
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
13 And use the generated parser by deriving a user class:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
14
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
15
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
16 .. code::
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
17
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
18 import test_parser
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
19 class MyParser(test_parser.Parser):
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
20 pass
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
21 p = MyParser()
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
22 p.parse()
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
23
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
24
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
25 Alternatively you can load the parser on the fly:
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
26
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
27 .. code::
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
28
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
29 import yacc
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
30 parser_mod = yacc.load_as_module('mygrammar.x')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
31 class MyParser(parser_mod.Parser):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
32 pass
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
33 p = MyParser()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
34 p.parse()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
35
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
36 """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
37
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
38 import argparse
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
39 import re
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
40 import sys
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
41 import datetime
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
42 import types
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
43 import io
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
44 from pyyacc import Grammar, print_grammar
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
45
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
46
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
47 class XaccLexer:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
48 def __init__(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
49 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
50
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
51 def feed(self, txt):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
52 # Create a regular expression for the lexing part:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
53 tok_spec = [
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
54 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
55 ('STRING', r"'[^']*'"),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
56 ('BRACEDCODE', r"\{[^\}]*\}"),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
57 ('OTHER', r'[:;\|]'),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
58 ('SKIP', r'[ ]')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
59 ]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
60 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
61 gettok = re.compile(tok_re).match
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
62
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
63 lines = txt.split('\n')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
64
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
65 def tokenize_line(line):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
66 """ Generator that splits up a line into tokens """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
67 mo = gettok(line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
68 pos = 0
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
69 while mo:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
70 typ = mo.lastgroup
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
71 val = mo.group(typ)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
72 if typ == 'ID':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
73 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
74 elif typ == 'STRING':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
75 typ = 'ID'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
76 yield (typ, val[1:-1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
77 elif typ == 'OTHER':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
78 typ = val
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
79 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
80 elif typ == 'BRACEDCODE':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
81 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
82 elif typ == 'SKIP':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
83 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
84 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
85 raise NotImplementedError(str(typ))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
86 pos = mo.end()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
87 mo = gettok(line, pos)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
88 if len(line) != pos:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
89 raise ParseError('Lex fault at {}'.format(line))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
90
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
91 def tokenize():
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
92 section = 0
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
93 for line in lines:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
94 line = line.strip()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
95 if not line:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
96 continue # Skip empty lines
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
97 if line == '%%':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
98 section += 1
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
99 yield('%%', '%%')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
100 continue
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
101 if section == 0:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
102 if line.startswith('%tokens'):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
103 yield('%tokens', '%tokens')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
104 yield from tokenize_line(line[7:])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
105 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
106 yield ('HEADER', line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
107 elif section == 1:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
108 yield from tokenize_line(line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
109 yield ('eof', 'eof')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
110 self.tokens = tokenize()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
111 self.token = self.tokens.__next__()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
112
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
113 def next_token(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
114 t = self.token
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
115 if t[0] != 'eof':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
116 self.token = self.tokens.__next__()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
117 return t
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
118
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
119
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
120 class ParseError(Exception):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
121 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
122
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
123
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
124 class XaccParser:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
125 """ Implements a recursive descent parser to parse grammar rules.
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
126 We could have made an generated parser, but that would yield a chicken
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
127 egg issue.
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
128 """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
129 def __init__(self, lexer):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
130 self.lexer = lexer
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
131
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
132 @property
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
133 def Peak(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
134 """ Sneak peak to the next token in line """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
135 return self.lexer.token[0]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
136
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
137 def next_token(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
138 """ Take the next token """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
139 return self.lexer.next_token()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
140
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
141 def consume(self, typ):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
142 """ Eat next token of type typ or raise an exception """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
143 if self.Peak == typ:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
144 return self.next_token()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
145 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
146 raise ParseError('Expected {}, but got {}'.format(typ, self.Peak))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
147
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
148 def has_consumed(self, typ):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
149 """ Consume typ if possible and return true if so """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
150 if self.Peak == typ:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
151 self.consume(typ)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
152 return True
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
153 return False
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
154
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
155 def parse_grammar(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
156 """ Entry parse function into recursive descent parser """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
157 # parse header
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
158 headers = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
159 terminals = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
160 while self.Peak in ['HEADER', '%tokens']:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
161 if self.Peak == '%tokens':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
162 self.consume('%tokens')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
163 while self.Peak == 'ID':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
164 terminals.append(self.consume('ID')[1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
165 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
166 headers.append(self.consume('HEADER')[1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
167 self.consume('%%')
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
168 self.headers = headers
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
169 self.grammar = Grammar(terminals)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
170 while self.Peak != 'eof':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
171 self.parse_rule()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
172 return self.grammar
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
173
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
174 def parse_symbol(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
175 return self.consume('ID')[1]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
176
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
177 def parse_rhs(self):
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
178 """ Parse the right hand side of a rule definition """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
179 symbols = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
180 while self.Peak not in [';', 'BRACEDCODE', '|']:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
181 symbols.append(self.parse_symbol())
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
182 if self.Peak == 'BRACEDCODE':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
183 action = self.consume('BRACEDCODE')[1]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
184 action = action[1:-1].strip()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
185 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
186 action = None
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
187 return symbols, action
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
188
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
189 def parse_rule(self):
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
190 """ Parse a rule definition """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
191 p = self.parse_symbol()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
192 self.consume(':')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
193 symbols, action = self.parse_rhs()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
194 self.grammar.add_production(p, symbols, action)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
195 while self.has_consumed('|'):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
196 symbols, action = self.parse_rhs()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
197 self.grammar.add_production(p, symbols, action)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
198 self.consume(';')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
199
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
200
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
201 class XaccGenerator:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
202 """ Generator that writes generated parser to file """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
203 def __init__(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
204 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
205
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
206 def generate(self, grammar, headers, output_file):
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
207 self.output_file = output_file
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
208 self.grammar = grammar
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
209 self.headers = headers
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
210 self.action_table, self.goto_table = grammar.doGenerate()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
211 self.generate_python_script()
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
212
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
213 def print(self, *args):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
214 """ Print helper function that prints to output file """
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
215 print(*args, file=self.output_file)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
216
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
217 def generate_python_script(self):
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
218 """ Generate python script with the parser table """
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
219 self.print('#!/usr/bin/python')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
220 stamp = datetime.datetime.now().ctime()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
221 self.print('""" Automatically generated by xacc on {} """'.format(stamp))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
222 self.print('from pyyacc import LRParser, Reduce, Shift, Accept, Production, Grammar')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
223 self.print('from ppci import Token')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
224 self.print('')
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
225 for h in self.headers:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
226 print(h, file=output_file)
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
227 self.print('')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
228 self.print('class Parser(LRParser):')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
229 self.print(' def __init__(self):')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
230 # Generate rules:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
231 self.print(' self.start_symbol = "{}"'.format(self.grammar.start_symbol))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
232 self.print(' self.grammar = Grammar({})'.format(self.grammar.terminals))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
233 for rule_number, rule in enumerate(self.grammar.productions):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
234 rule.f_name = 'action_{}_{}'.format(rule.name, rule_number)
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
235 self.print(' self.grammar.add_production("{}", {}, self.{})'.format(rule.name, rule.symbols, rule.f_name))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
236 # Fill action table:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
237 self.print(' self.action_table = {}')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
238 for state in self.action_table:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
239 action = self.action_table[state]
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
240 self.print(' self.action_table[{}] = {}'.format(state, action))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
241 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
242
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
243 # Fill goto table:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
244 self.print(' self.goto_table = {}')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
245 for gt in self.goto_table:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
246 to = self.goto_table[gt]
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
247 self.print(' self.goto_table[{}] = {}'.format(gt, to))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
248 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
249
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
250 # Generate a function for each action:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
251 for rule in self.grammar.productions:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
252 M = len(rule.symbols)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
253 args = ', '.join('arg{}'.format(n + 1) for n in range(M))
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
254 self.print(' def {}(self, {}):'.format(rule.f_name, args))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
255 if rule.f == None:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
256 semantics = 'pass'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
257 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
258 semantics = str(rule.f)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
259 if semantics.strip() == '':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
260 semantics = 'pass'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
261 for n in range(M):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
262 semantics = semantics.replace('${}'.format(n + 1), 'arg{}'.format(n + 1))
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
263 self.print(' {}'.format(semantics))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
264
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
265
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
266 def make_argument_parser():
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
267 # Parse arguments:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
268 parser = argparse.ArgumentParser(description='xacc compiler compiler')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
269 parser.add_argument('source', type=argparse.FileType('r'), \
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
270 help='the parser specification')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
271 parser.add_argument('-o', '--output', type=argparse.FileType('w'), \
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
272 default=sys.stdout)
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
273
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
274
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
275 def load_as_module(filename):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
276 """ Load a parser spec file, generate LR tables and create module """
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
277 ob = io.StringIO()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
278 args = argparse.Namespace(source=open(filename), output=ob)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
279 main(args)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
280
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
281 parser_mod = types.ModuleType('generated_parser')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
282 exec(ob.getvalue(), parser_mod.__dict__)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
283 return parser_mod
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
284
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
285 def main(args):
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
286 src = args.source.read()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
287 args.source.close()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
288
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
289 # Construction of generator parts:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
290 lexer = XaccLexer()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
291 parser = XaccParser(lexer)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
292 generator = XaccGenerator()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
293
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
294 # Sequence source through the generator parts:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
295 lexer.feed(src)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
296 grammar = parser.parse_grammar()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
297 # TODO: optionize this: print_grammar(grammar)
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
298 generator.generate(grammar, parser.headers, args.output)
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
299
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
300
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
301 if __name__ == '__main__':
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
302 args = make_argument_parser().parse_args()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
303 main(args)