Mercurial > lcfOS
view python/ppci/assembler.py @ 354:5477e499b039
Added some sort of string functionality
author | Windel Bouwman |
---|---|
date | Thu, 13 Mar 2014 18:59:06 +0100 |
parents | 3bb7dcfe5529 |
children | 19eacf4f7270 |
line wrap: on
line source
import re import pyyacc from . import Token, CompilerError, SourceLocation from .target import Target, Label def bit_type(value): assert value < (2**32) assert value >= 0 t = 'val32' for n in [16, 12, 8, 5, 3]: if value < (2**n): t = 'val{}'.format(n) return t def tokenize(s, kws): """ Tokenizer, generates an iterator that returns tokens! This GREAT example was taken from python re doc page! """ tok_spec = [ ('REAL', r'\d+\.\d+'), ('HEXNUMBER', r'0x[\da-fA-F]+'), ('NUMBER', r'\d+'), ('ID', r'[A-Za-z][A-Za-z\d_]*'), ('SKIP', r'[ \t]'), ('LEESTEKEN', r':=|[\.,=:\-+*\[\]/\(\)]|>=|<=|<>|>|<|}|{'), ('STRING', r"'.*?'"), ('COMMENT', r";.*") ] tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec) gettok = re.compile(tok_re).match line = 1 pos = line_start = 0 mo = gettok(s) while mo is not None: typ = mo.lastgroup val = mo.group(typ) if typ == 'NEWLINE': line_start = pos line += 1 elif typ != 'SKIP': if typ == 'LEESTEKEN': typ = val elif typ == 'NUMBER': val = int(val) elif typ == 'HEXNUMBER': val = int(val[2:], 16) typ = 'NUMBER' elif typ == 'REAL': val = float(val) elif typ == 'STRING': val = val[1:-1] elif typ == 'ID': if val.lower() in kws: # ['r3', 'sp', 'add', 'yield', 'r4', 'r0', 'r1', 'sub', 'r5', 'r6', 'r2']: typ = val.lower() col = mo.start() - line_start loc = SourceLocation('', line, col, 0) # TODO retrieve length? if typ == 'NUMBER': typ = bit_type(val) yield Token(typ, val, loc) pos = mo.end() mo = gettok(s, pos) if pos != len(s): col = pos - line_start loc = SourceLocation('', line, col, 0) raise CompilerError('Unexpected character {0}'.format(s[pos]), loc) yield Token('EOF', pyyacc.EOF) class Lexer: def __init__(self, src, kws): self.tokens = tokenize(src, kws) self.curTok = self.tokens.__next__() def next_token(self): t = self.curTok if t.typ != 'EOF': self.curTok = self.tokens.__next__() return t class Parser: def add_rule(self, prod, rhs, f): """ Helper function to add a rule, why this is required? """ if prod == 'instruction': def f_wrap(*args): i = f(args) self.emit(i) else: def f_wrap(*rhs): return f(rhs) self.g.add_production(prod, rhs, f_wrap) def __init__(self, kws, instruction_rules, emit): # Construct a parser given a grammar: tokens2 = ['ID', 'NUMBER', ',', '[', ']', ':', '+', '-', '*', pyyacc.EPS, 'COMMENT', '{', '}', pyyacc.EOF, 'val32', 'val16', 'val12', 'val8', 'val5', 'val3'] tokens2.extend(kws) self.kws = kws g = pyyacc.Grammar(tokens2) self.g = g # Global structure of assembly line: g.add_production('asmline', ['asmline2']) g.add_production('asmline', ['asmline2', 'COMMENT']) g.add_production('asmline2', ['label', 'instruction']) g.add_production('asmline2', ['instruction']) g.add_production('asmline2', ['label']) g.add_production('asmline2', []) g.add_production('label', ['ID', ':'], self.p_label) #g.add_production('label', []) # Add instruction rules for the target in question: for prod, rhs, f in instruction_rules: self.add_rule(prod, rhs, f) #g.add_production('instruction', []) g.add_production('expression', ['term'], lambda x: x) g.add_production('expression', ['expression', 'addop', 'term'], self.p_binop) g.add_production('addop', ['-'], lambda x: x.val) g.add_production('addop', ['+'], lambda x: x.val) g.add_production('mulop', ['*'], lambda x: x.val) g.add_production('term', ['factor'], lambda x: x) g.add_production('term', ['term', 'mulop', 'factor'], self.p_binop) g.add_production('factor', ['ID'], lambda name: ASymbol(name.val)) g.add_production('factor', ['NUMBER'], lambda num: ANumber(int(num.val))) g.start_symbol = 'asmline' self.emit = emit self.p = g.generate_parser() # print('length of table:', len(self.p.action_table)) # Parser handlers: def p_ins_1(self, opc, ops): ins = AInstruction(opc, ops) self.emit(ins) def p_ins_2(self, opc): self.p_ins_1(opc, []) def p_operands_1(self, op1): return [op1] def p_operands_2(self, ops, comma, op2): assert type(ops) is list ops.append(op2) return ops def p_listitems_1(self, li1): return [li1] def p_listitems_2(self, lis, comma, li2): assert type(lis) is list lis.append(li2) return lis def p_list_op(self, brace_open, lst, brace_close): return AUnop('{}', lst) def p_mem_op(self, brace_open, exp, brace_close): return AUnop('[]', exp) def p_label(self, lname, cn): lab = Label(lname.val) self.emit(lab) def p_binop(self, exp1, op, exp2): return ABinop(op, exp1, exp2) def parse(self, lexer): self.p.parse(lexer) class Assembler: def __init__(self, target): self.target = target assert isinstance(target, Target) self.parser = Parser(target.asm_keywords, target.assembler_rules, self.emit) def emit(self, *args): self.stream.emit(*args) # Top level interface: def parse_line(self, line): """ Parse line into assembly instructions """ tokens = Lexer(line, self.target.asm_keywords) self.parser.parse(tokens) def assemble(self, asmsrc, stream): """ Assemble this source snippet """ if hasattr(asmsrc, 'read'): asmsrc2 = asmsrc.read() asmsrc.close() asmsrc = asmsrc2 # TODO: use generic newline?? # TODO: the bothersome newline ... self.stream = stream for line in asmsrc.split('\n'): self.parse_line(line) self.stream = None