Mercurial > lcfOS
changeset 366:39bf68bf1891
Fix sample tests and deterministic build
| author | Windel Bouwman |
|---|---|
| date | Fri, 21 Mar 2014 09:43:01 +0100 |
| parents | 98ff43cfdd36 |
| children | 577ed7fb3fe4 |
| files | doc/compiler.rst python/ppci/buildtasks.py python/ppci/c3/builder.py python/ppci/codegen/codegen.py python/ppci/codegen/interferencegraph.py python/ppci/codegen/registerallocator.py python/ppci/irmach.py python/ppci/outstream.py python/ppci/recipe.py python/ppci/target/basetarget.py python/ppci/target/target_list.py python/zcc.py test/grind.py test/testemulation.py test/testsamples.py test/testzcc.py |
| diffstat | 16 files changed, 141 insertions(+), 93 deletions(-) [+] |
line wrap: on
line diff
--- a/doc/compiler.rst Wed Mar 19 22:32:04 2014 +0100 +++ b/doc/compiler.rst Fri Mar 21 09:43:01 2014 +0100 @@ -109,6 +109,9 @@ // .. autoclass:: ppci.irmach.AbstractInstruction +To select instruction, a tree rewrite system is used. This is also called +bottom up rewrite generator (BURG). See pyburg. + Register allocation ~~~~~~~~~~~~~~~~~~~ @@ -116,3 +119,12 @@ The selected instructions are used to select correct registers. +code emission +~~~~~~~~~~~~~ + +Code is emitted using the outputstream class. The assembler and compiler use +this class to emit instructions to. The stream can output to object file +or to a logger. + +.. autoclass:: ppci.outstream.OutputStream +
--- a/python/ppci/buildtasks.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/buildtasks.py Fri Mar 21 09:43:01 2014 +0100 @@ -16,8 +16,38 @@ from .objectfile import ObjectFile from .linker import Linker from .outstream import BinaryOutputStream, MasterOutputStream, LoggerOutputStream +from .target.target_list import targets +from .target import Target +def fix_target(tg): + """ Try to return an instance of the Target class """ + if isinstance(tg, Target): + return tg + elif isinstance(tg, str): + if tg in targets: + return targets[tg] + else: + raise TaskError('Target {} not found'.format(tg)) + raise TaskError('Invalid target {}'.format(tg)) + + +def fix_file(f): + """ Determine if argument is a file like object or make it so! """ + if hasattr(f, 'read'): + # Assume this is a file like object + return f + elif isinstance(f, str): + return open(f, 'r') + else: + raise TaskError('cannot use {} as input'.format(f)) + +def fix_object(o): + if isinstance(o, ObjectFile): + return o + else: + raise TaskError('Cannot use {} as objectfile'.format(o)) + class BuildTask(Task): """ Base task for all kind of weird build tasks """ def __init__(self, name): @@ -30,23 +60,27 @@ output into an object file """ def __init__(self, source, target, output_object): super().__init__('Assemble') - self.source = source + self.source = fix_file(source) self.output = output_object - self.ostream = BinaryOutputStream(self.output) - self.assembler = Assembler(target) + o2 = BinaryOutputStream(self.output) + o1 = LoggerOutputStream() + self.ostream = MasterOutputStream([o1, o2]) + self.assembler = Assembler(fix_target(target)) def run(self): + self.logger.debug('Assembling into code section') self.ostream.select_section('code') self.assembler.assemble(self.source, self.ostream) + self.logger.debug('Assembling finished') class Compile(BuildTask): """ Task that compiles C3 source for some target into an object file """ def __init__(self, sources, includes, target, output_object): super().__init__('Compile') - self.sources = sources - self.includes = includes - self.target = target + self.sources = list(map(fix_file, sources)) + self.includes = list(map(fix_file, includes)) + self.target = fix_target(target) self.output = output_object def run(self): @@ -78,9 +112,7 @@ o2 = BinaryOutputStream(self.output) o1 = LoggerOutputStream() - o = MasterOutputStream() - o.add_substream(o1) - o.add_substream(o2) + o = MasterOutputStream([o1, o2]) cg.generate(ircode, o) if not c3b.ok: @@ -92,7 +124,7 @@ """ Link together a collection of object files """ def __init__(self, objects, layout, output_file): super().__init__('Link') - self.objects = objects + self.objects = list(map(fix_object, objects)) self.linker = Linker() self.duration = 0.1337 self.layout = layout
--- a/python/ppci/c3/builder.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/c3/builder.py Fri Mar 21 09:43:01 2014 +0100 @@ -105,9 +105,9 @@ def doParse(src): tokens = self.lexer.lex(src) return self.parser.parseSource(tokens) - s_pkgs = set(map(doParse, srcs)) - i_pkgs = set(map(doParse, imps)) - all_pkgs = s_pkgs | i_pkgs + s_pkgs = list(map(doParse, srcs)) + i_pkgs = list(map(doParse, imps)) + all_pkgs = s_pkgs + i_pkgs if not all(all_pkgs): self.ok = False return @@ -126,7 +126,7 @@ # Generate intermediate code (phase 2) # Only return ircode when everything is OK - for pkg in all_pkgs & s_pkgs: + for pkg in s_pkgs: yield self.cg.gencode(pkg) if not all(pkg.ok for pkg in all_pkgs): self.ok = False
--- a/python/ppci/codegen/codegen.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/codegen/codegen.py Fri Mar 21 09:43:01 2014 +0100 @@ -53,7 +53,6 @@ assert isinstance(ircode, ir.Module) outs.select_section('data') for global_variable in ircode.Variables: - print(global_variable) self.target.emit_global(outs, ir.label_name(global_variable)) outs.select_section('code')
--- a/python/ppci/codegen/interferencegraph.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/codegen/interferencegraph.py Fri Mar 21 09:43:01 2014 +0100 @@ -12,6 +12,8 @@ def __repr__(self): return '{}({})'.format(self.temps, self.color) + def __gt__(self, other): + return str(self.temps) > str(other.temps) class InterferenceGraph(Graph): """
--- a/python/ppci/codegen/registerallocator.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/codegen/registerallocator.py Fri Mar 21 09:43:01 2014 +0100 @@ -3,7 +3,11 @@ from .interferencegraph import InterferenceGraph # Nifty first function: -first = lambda x: next(iter(x)) +def first(x): + """ Take the first element of a collection after sorting the things """ + x = list(x) + x.sort() + return next(iter(x)) class RegisterAllocator:
--- a/python/ppci/irmach.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/irmach.py Fri Mar 21 09:43:01 2014 +0100 @@ -40,6 +40,10 @@ self.others = tuple(others) self.ismove = ismove + def __gt__(self, other): + """ To make the class fit for sorting """ + return str(self) > str(other) + def __repr__(self): """ Substitutes source, dst and labels in the string """ if isinstance(self.assem, Instruction):
--- a/python/ppci/outstream.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/outstream.py Fri Mar 21 09:43:01 2014 +0100 @@ -1,7 +1,7 @@ import logging import binascii -from ppci.target import Instruction, Alignment -from ppci.objectfile import ObjectFile +from .target import Instruction, Alignment +from .objectfile import ObjectFile """ The output stream is a stream of instructions that can be output @@ -10,6 +10,7 @@ class OutputStream: + """ Interface to generator code with. """ def emit(self, item): raise NotImplementedError('Abstract base class') @@ -17,28 +18,6 @@ raise NotImplementedError('Abstract base class') -class OutputStreamWriter: - def __init__(self, extra_indent=''): - self.extra_indent = extra_indent - - def dump(self, stream, f): - for s in sorted(stream.sections.keys()): - # print('.section '+ s) - self.dumpSection(stream.sections[s], f) - - def dumpSection(self, s, f): - for i in s.instructions: - addr = i.address - insword = i.encode() - assert type(insword) is bytes - insword = binascii.hexlify(bytes(reversed(insword))).decode('ascii') - asm = str(i) - if len(insword) == 0: - print(' {}'.format(asm), file=f) - else: - print(' 0x{0:08x} 0x{1} {2}'.format(addr, insword, asm), file=f) - - class BinaryOutputStream(OutputStream): """ Output stream that writes to object file """ def __init__(self, obj_file): @@ -91,8 +70,8 @@ class MasterOutputStream(OutputStream): """ Stream that emits to multiple sub streams """ - def __init__(self): - self.substreams = [] + def __init__(self, substreams=[]): + self.substreams = list(substreams) # Use copy constructor!!! def add_substream(self, output_stream): self.substreams.append(output_stream)
--- a/python/ppci/recipe.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/recipe.py Fri Mar 21 09:43:01 2014 +0100 @@ -3,11 +3,7 @@ from .buildtasks import Compile, Assemble, Link from .objectfile import ObjectFile -from .target.target_list import target_list - - -targets = {t.name: t for t in target_list} -targetnames = list(targets.keys()) +from .target.target_list import targets class RecipeLoader:
--- a/python/ppci/target/basetarget.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/target/basetarget.py Fri Mar 21 09:43:01 2014 +0100 @@ -73,6 +73,9 @@ def __init__(self, name): self.name = name + def __gt__(self, other): + return self.num > other.num + class LabelAddress: def __init__(self, name):
--- a/python/ppci/target/target_list.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/ppci/target/target_list.py Fri Mar 21 09:43:01 2014 +0100 @@ -8,3 +8,5 @@ thumb_target = ThumbTarget() target_list = [arm_target, thumb_target] +targets = {t.name: t for t in target_list} +targetnames = list(targets.keys())
--- a/python/zcc.py Wed Mar 19 22:32:04 2014 +0100 +++ b/python/zcc.py Fri Mar 21 09:43:01 2014 +0100 @@ -9,7 +9,7 @@ from ppci.tasks import TaskRunner from ppci.report import RstFormatter from ppci.objectfile import ObjectFile -from ppci.target.target_list import target_list +from ppci.target.target_list import targets, targetnames from ppci.recipe import RecipeLoader import ppci @@ -22,9 +22,6 @@ return numeric_level -targets = {t.name: t for t in target_list} -targetnames = list(targets.keys()) - def make_parser(): parser = argparse.ArgumentParser(description='lcfos Compiler')
--- a/test/grind.py Wed Mar 19 22:32:04 2014 +0100 +++ b/test/grind.py Fri Mar 21 09:43:01 2014 +0100 @@ -17,5 +17,6 @@ p = cProfile.Profile() s = p.run('runtests()') stats = pstats.Stats(p) - stats.sort_stats('tottime') + #stats.sort_stats('tottime') + stats.sort_stats('cumtime') stats.print_stats(.1)
--- a/test/testemulation.py Wed Mar 19 22:32:04 2014 +0100 +++ b/test/testemulation.py Fri Mar 21 09:43:01 2014 +0100 @@ -23,26 +23,28 @@ pass # Listen to the control socket: - qemu_control = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - qemu_control.bind('qemucontrol.sock') - qemu_control.listen(0) + qemu_control_serve = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + qemu_control_serve.bind('qemucontrol.sock') + qemu_control_serve.listen(0) # Listen to the serial output: - qemu_serial = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - qemu_serial.bind('qemuserial.sock') - qemu_serial.listen(0) + qemu_serial_serve = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + qemu_serial_serve.bind('qemuserial.sock') + qemu_serial_serve.listen(0) args = ['qemu-system-arm', '-M', machine, '-m', '16M', '-nographic', '-kernel', kernel, '-monitor', 'unix:qemucontrol.sock', '-serial', 'unix:qemuserial.sock', '-S'] - p = subprocess.Popen(args) - #stdout=subprocess.DEVNULL, - #stderr=subprocess.DEVNULL) + p = subprocess.Popen(args, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) # Give process some time to boot: - qemu_serial, address_peer = qemu_serial.accept() - qemu_control, address_peer = qemu_control.accept() + qemu_serial, address_peer = qemu_serial_serve.accept() + qemu_control, address_peer = qemu_control_serve.accept() + + # Give the go command: qemu_control.send('cont\n'.encode('ascii')) qemu_serial.settimeout(0.2) @@ -63,9 +65,10 @@ p.wait(timeout=3) except subprocess.TimeoutExpired: p.kill() - print(p.communicate()) qemu_control.close() qemu_serial.close() + qemu_control_serve.close() + qemu_serial_serve.close() try: os.remove('qemucontrol.sock')
--- a/test/testsamples.py Wed Mar 19 22:32:04 2014 +0100 +++ b/test/testsamples.py Fri Mar 21 09:43:01 2014 +0100 @@ -1,13 +1,23 @@ import unittest +import os +import io from testemulation import runQemu -from ppci.recipe import RecipeLoader +from testzcc import relpath from ppci.tasks import TaskRunner +from ppci.buildtasks import Assemble, Compile, Link +from ppci.objectfile import ObjectFile +startercode = """ +mov sp, 0x30000 ; setup stack pointer +BL sample_start ; Branch to sample start +local_loop: +B local_loop +""" class Samples: def testPrint(self): snippet = """ - module testsample; + module sample; import io; function void start() { @@ -18,12 +28,12 @@ def testForLoopPrint(self): snippet = """ - module testsample; + module sample; import io; function void start() { var int i; - for (i=0; i<10; i++) + for (i=0; i<10; i = i + 1) { io.print2("A = ", i); } @@ -32,9 +42,9 @@ res = "".join("A = 0x{0:08X}\n".format(a) for a in range(10)) self.do(snippet, res) - def testForLoopPrint(self): + def testGlobalVariable(self): snippet = """ - module testglobal; + module sample; import io; var int G; function void do1() @@ -52,9 +62,9 @@ G = 0; do1(); do1(); - do2(); + do5(); do1(); - do2(); + do5(); } """ res = "".join("G=0x{0:08X}\n".format(a) for a in [1,2,7,8,13]) @@ -63,25 +73,30 @@ class TestSamplesOnVexpress(unittest.TestCase, Samples): def do(self, src, expected_output): + # Construct binary file from snippet: + o1 = ObjectFile() + o2 = ObjectFile() + asmb = Assemble(io.StringIO(startercode), 'arm', o1) + comp = Compile([ + relpath('..', 'kernel', 'src', 'io.c3'), + relpath('..', 'kernel', 'arch', 'vexpressA9.c3'), + io.StringIO(src)], [], 'arm', o2) + sample_filename = 'testsample.bin' + layout = {'code': 0x10000, 'data':0x20000} + link = Link([o1, o2], layout, sample_filename) + + # Create task executor: runner = TaskRunner() - recipe_loader = RecipeLoader(runner) - # print(expected_output) - return - # TODO: improve recipe loading?? - recipe_loader.load_dict({ - 'link': { - 'inputs': [ - ], - 'layout': { - 'code': 0x60010000, - 'data': 0x60020000 - }, - 'output': 'tst.bin' - } - }) - runner.add_task(Compile()) + runner.add_task(asmb) + runner.add_task(comp) + runner.add_task(link) runner.run_tasks() - res = runQemu('tst.bin', machine='vexpress-a9') + # Check bin file exists: + self.assertTrue(os.path.isfile(sample_filename)) + + # Run bin file in emulator: + res = runQemu(sample_filename, machine='vexpress-a9') + os.remove(sample_filename) self.assertEqual(expected_output, res)
--- a/test/testzcc.py Wed Mar 19 22:32:04 2014 +0100 +++ b/test/testzcc.py Fri Mar 21 09:43:01 2014 +0100 @@ -35,7 +35,7 @@ os.chdir(testdir) def do(self, filenames, imps=[], extra_args=[]): - basedir = os.path.join('..', 'examples', 'c3') + basedir = relpath('..', 'examples', 'c3') arg_list = ['compile'] arg_list += [os.path.join(basedir, fn) for fn in filenames] for fn in imps: @@ -65,11 +65,10 @@ recipe = relpath('..', 'kernel', 'arm.yaml') self.buildRecipe(recipe) - @unittest.skip('Too difficult to fix') def testKernelBuildsEqualTwice(self): """ Build kernel two times and check the output is equal """ - recipe = relpath('..', 'kernel', 'recipe.yaml') - bin_filename = relpath('..', 'kernel', 'kernel.bin') + recipe = relpath('..', 'kernel', 'arm.yaml') + bin_filename = relpath('..', 'kernel', 'kernel_arm.bin') self.buildRecipe(recipe) with open(bin_filename, 'rb') as f: a = f.read()