# HG changeset patch # User Olivier Breuleux # Date 1231804030 18000 # Node ID 605ab704abc3f1ad1d1612d69efced6f3e08dfe2 # Parent 7ddb286162f66656aa6ca983ece7c38e5d264697# Parent 8d0b73c7d76803d85c876eb2167bac798bed5767 merge diff -r 8d0b73c7d768 -r 605ab704abc3 pylearn/dbdict/newstuff.py --- a/pylearn/dbdict/newstuff.py Thu Jan 08 15:11:54 2009 -0500 +++ b/pylearn/dbdict/newstuff.py Mon Jan 12 18:47:10 2009 -0500 @@ -47,7 +47,7 @@ """nested dictionary -> flat dictionary with '.' notation """ d = {} def helper(d, prefix, obj): - if isinstance(obj, (str, int, float)): + if isinstance(obj, (str, int, float, list, tuple)): d[prefix] = obj #convert(obj) else: if isinstance(obj, dict): @@ -502,6 +502,8 @@ help = 'redirect stdout and stderr to the workdir/stdout and workdir/stderr files') parser_cmdline.add_option('-w', '--workdir', action = 'store', dest = 'workdir', default = None, help = 'the working directory in which to run the experiment') +parser_cmdline.add_option('-n', '--dry-run', action = 'store_true', dest = 'dry_run', default = False, + help = 'use this option to run the whole experiment in a temporary working directory (cleaned after use)') def runner_cmdline(options, experiment, *strings): """ @@ -522,14 +524,98 @@ state = expand(parse(*strings)) state.setdefault('dbdict', DD()).experiment = experiment experiment = resolve(experiment) - workdir = options.workdir or format_d(state, sep=',', space = False) + if options.workdir and options.dry_run: + raise UsageError('Please use only one of: --workdir, --dry-run.') + if options.workdir: + workdir = options.workdir + elif options.dry_run: + workdir = tempfile.mkdtemp() + else: + workdir = format_d(state, sep=',', space = False) channel = StandardChannel(workdir, experiment, state, redirect_stdout = options.redirect or options.redirect_stdout, redirect_stderr = options.redirect or options.redirect_stderr) channel.run(force = options.force) + if options.dry_run: + shutil.rmtree(workdir, ignore_errors=True) + +runner_registry['cmdline'] = (parser_cmdline, runner_cmdline) -runner_registry['cmdline'] = (parser_cmdline, runner_cmdline) + + + +parser_filemerge = OptionParser(usage = '%prog filemerge [options] ...') +parser_filemerge.add_option('-f', '--force', action = 'store_true', dest = 'force', default = False, + help = 'force running the experiment even if it is already running or completed') +parser_filemerge.add_option('--redirect-stdout', action = 'store_true', dest = 'redirect_stdout', default = False, + help = 'redirect stdout to the workdir/stdout file') +parser_filemerge.add_option('--redirect-stderr', action = 'store_true', dest = 'redirect_stderr', default = False, + help = 'redirect stderr to the workdir/stdout file') +parser_filemerge.add_option('-r', '--redirect', action = 'store_true', dest = 'redirect', default = False, + help = 'redirect stdout and stderr to the workdir/stdout and workdir/stderr files') +parser_filemerge.add_option('-w', '--workdir', action = 'store', dest = 'workdir', default = None, + help = 'the working directory in which to run the experiment') +parser_filemerge.add_option('-n', '--dry-run', action = 'store_true', dest = 'dry_run', default = False, + help = 'use this option to run the whole experiment in a temporary working directory (cleaned after use)') + +def runner_filemerge(options, experiment, mainfile, *other_files): + """ + Start an experiment with parameters given in files. + + Usage: filemerge [options] ... + + Run an experiment with parameters provided in plain text files. + A single experiment will be run with the union of all the + parameters listed in the files. + + Example: + + text.first = "hello" + text.second = "world" + + + number = 12 + numbers.a = 55 + numbers.b = 56 + + Given these files, the following command using filemerge: + $ dbdict-run filemerge mymodule.my_experiment blah1.txt blah2.txt + + is equivalent to this one using cmdline: + $ dbdict-run cmdline mymodule.my_experiment \\ + text.first=hello text.second=world \\ + number=12 numbers.a=55 numbers.b=56 + """ + with open(mainfile) as f: + _state = parse(*map(str.strip, f.readlines())) + for file in other_files: + if '=' in file: + _state.update(parse(file)) + else: + with open(file) as f: + _state.update(parse(*map(str.strip, f.readlines()))) + state = expand(_state) + state.setdefault('dbdict', DD()).experiment = experiment + experiment = resolve(experiment) + if options.workdir and options.dry_run: + raise UsageError('Please use only one of: --workdir, --dry-run.') + if options.workdir: + workdir = options.workdir + elif options.dry_run: + workdir = tempfile.mkdtemp() + else: + workdir = format_d(state, sep=',', space = False) + channel = StandardChannel(workdir, + experiment, state, + redirect_stdout = options.redirect or options.redirect_stdout, + redirect_stderr = options.redirect or options.redirect_stderr) + channel.run(force = options.force) + if options.dry_run: + shutil.rmtree(workdir, ignore_errors=True) + +runner_registry['filemerge'] = (parser_filemerge, runner_filemerge) + @@ -675,7 +761,7 @@ print 'parameters How to list the parameters for an experiment.' print print bold('Available commands: (use help for more info)') - for name, command in sorted(runner_registry.iteritems()): + for name, (parser, command) in sorted(runner_registry.iteritems()): print name.ljust(20), format_help(command).split('\n')[0] return elif topic == 'experiment':