# HG changeset patch # User James Bergstra # Date 1226545220 18000 # Node ID 798607a058bd3409c2109fb79a980a14056f9f6b # Parent b054271b25040ced38912a9a08f30bae15a7f8fc added missing files diff -r b054271b2504 -r 798607a058bd LICENSE --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/LICENSE Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,24 @@ +Copyright (c) 2008, Theano Development Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Theano nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff -r b054271b2504 -r 798607a058bd README.txt diff -r b054271b2504 -r 798607a058bd bin/dbdict-query --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/dbdict-query Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from dbdict.tools import standalone_query +standalone_query() diff -r b054271b2504 -r 798607a058bd bin/dbdict-run --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/dbdict-run Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from pylearn.dbdict.dbdict_run import RunExperiment +RunExperiment() diff -r b054271b2504 -r 798607a058bd bin/dbdict-run-job --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/dbdict-run-job Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from dbdict.tools import standalone_run_job +standalone_run_job() diff -r b054271b2504 -r 798607a058bd pylearn/algorithms/minimizer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/algorithms/minimizer.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,43 @@ +"""Define the interface and factory for gradient-based minimizers. +""" +from theano.compile import module + +_minimizers = {} + +class DummyMinimizer(module.FancyModule): + """ The idea of a minimizer is that it provides an `step` function that will + eventually converge toward (maybe realize?) the minimum of a cost function. + + The step_cost function takes a step and returns the cost associated with either + the current or previous parameter values (return whichever is easiest to compute, it's + meant for user feedback.) + + """ + def __init__(self, args, cost, parameters, gradients=None): + super(DummyMinimizer, self).__init__() + #gradients = T.grad(cost, parameters) if gradients is None else gradients + #self.step = module.Method(args, None) + #self.step_cost = module.Method(args, cost) + def _instance_step(self, obj, *args): + pass + def _instance_step_cost(self, obj, *args): + pass + +def minimizer_factory(algo): + def decorator(fn): + if algo in _minimizers: + raise Exception('algo in use', algo) + else: + _minimizers[algo] = fn + return fn + return decorator + +@minimizer_factory('dummy') +def dummy_minimizer(): + def m(args, cost, parameters, gradients=None): + return DummyMinimizer(args, cost, parameters, gradients) + return m + +def make_minimizer(algo, **kwargs): + return _minimizers[algo](**kwargs) + diff -r b054271b2504 -r 798607a058bd pylearn/datasets/embeddings/percent.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/datasets/embeddings/percent.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,9 @@ +def percent(a, b): + """ + Return percentage string of a and b, e.g.: + "1 of 10 (10%)" + """ + assert a <= b + assert a >= 0 + assert b > 0 + return "%s of %s (%.2f%%)" % (a, b, 100.*a/b) diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/__init__.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,1 @@ +from api0 import sqlite_file_db, sqlite_memory_db diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/api0.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/api0.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,511 @@ +from sqlalchemy import create_engine, desc +from sqlalchemy.orm import sessionmaker +from sqlalchemy import Table, Column, MetaData, ForeignKey +from sqlalchemy import Integer, String, Float, Boolean, DateTime, Text, Binary +from sqlalchemy.orm import mapper, relation, backref, eagerload +from sqlalchemy.sql import operators, select +from sql_commands import crazy_sql_command + +class Todo(Exception): """Replace this with some working code!""" + +class DbHandle (object): + """ + This class also provides filtering shortcuts that hide the names of the + DbHandle internal databases. + + Attributes: + dict_table + pair_table + link_table + + + + dict_table + + An SqlAlchemy-mapped class corresponding to database table with the + following schema: + + Column('id', Integer, primary_key=True) + Column('create', DateTime) + Column('write', DateTime) + Column('read', DateTime) + + #TODO: reconsider create/read/write + + pair_table + + An SqlAlchemy-mapped class corresponding to database table with the + following schema: + + Column('id', Integer, primary_key=True) + Column('name', String(128)) + Column('ntype', Boolean) + Column('fval', Double) + Column('sval', Text)) + Column('bval', Blob) + + #TODO: Consider difference between text and binary + #TODO: Consider adding a 'type' column + #TODO: Consider union? + #TODO: Are there stanard ways of doing this kind of thing? + + link_table + + An SqlAlchemy-mapped class corresponding to database table with the + following schema: + + Column('dict_id', Integer, ForeignKey('%s.id' % t_trial), primary_key=True), + Column('keyval_id', Integer, ForeignKey('%s.id' % t_keyval), primary_key=True)) + + """ + + e_bad_table = 'incompatible columns in table' + + def __init__(h_self, Session, engine, dict_table, pair_table, link_table): + h_self._engine = engine; + h_self._dict_table = dict_table + h_self._pair_table = pair_table + h_self._link_table = link_table + + #TODO: replace this crude algorithm (ticket #17) + if ['id', 'create', 'write', 'read'] != [c.name for c in dict_table.c]: + raise ValueError(h_self.e_bad_table, dict_table) + if ['id', 'name', 'ntype', 'fval', 'sval', 'bval'] != [c.name for c in pair_table.c]: + raise ValueError(h_self.e_bad_table, pair_table) + if ['dict_id', 'pair_id'] != [c.name for c in link_table.c]: + raise ValueError(h_self.e_bad_table, pair_table) + + h_self._session_fn = Session + h_self._session = Session() + + class KeyVal (object): + """KeyVal interfaces between python types and the database. + + It encapsulates heuristics for type conversion. + """ + def __init__(k_self, name, val): + k_self.name = name + k_self.val = val + def __repr__(k_self): + return "" % (k_self.id, k_self.name, repr(k_self.val)) + def __get_val(k_self): + val = None + if k_self.fval is not None: val = [int, float][k_self.ntype](k_self.fval) + if k_self.bval is not None: val = eval(str(k_self.bval)) + if k_self.sval is not None: val = k_self.sval + return val + def __set_val(k_self, val): + if isinstance(val, (str,unicode)): + k_self.fval = None + k_self.bval = None + k_self.sval = val + else: + k_self.sval = None + try: + f = float(val) + except (TypeError, ValueError): + f = None + if f is None: #binary data + k_self.bval = repr(val) + assert eval(k_self.bval) == val + k_self.fval = None + k_self.ntype = None + else: + k_self.bval = None + k_self.fval = f + k_self.ntype = isinstance(val,float) + val = property(__get_val, __set_val) + + mapper(KeyVal, pair_table) + + class Dict (object): + """ + Instances are dict-like objects with additional features for + communicating with an active database. + + This class will be mapped by SqlAlchemy to the dict_table. + + Attributes: + handle - reference to L{DbHandle} (creator) + + """ + def __init__(d_self): + s = h_self._session + s.save(d_self) + s.commit() + + _forbidden_keys = set(['session']) + + # + # dictionary interface + # + + def __contains__(d_self, key): + for a in d_self._attrs: + if a.name == key: + return True + return False + + def __eq__(self, other): + return dict(self) == dict(other) + def __neq__(self, other): + return dict(self) != dict(other) + + def __getitem__(d_self, key): + for a in d_self._attrs: + if a.name == key: + return a.val + raise KeyError(key) + + def __setitem__(d_self, key, val): + s = h_self._session + d_self._set_in_session(key, val, s) + s.update(d_self) #session update, not dict-like update + s.commit() + + def __delitem__(d_self, key): + s = h_self._session + #find the item to delete in d_self._attrs + to_del = None + for i,a in enumerate(d_self._attrs): + if a.name == key: + assert to_del is None + to_del = (i,a) + if to_del is None: + raise KeyError(key) + else: + i, a = to_del + s.delete(a) + del d_self._attrs[i] + s.commit() + s.update(d_self) + + def items(d_self): + return [(kv.name, kv.val) for kv in d_self._attrs] + + def keys(d_self): + return [kv.name for kv in d_self._attrs] + + def values(d_self): + return [kv.val for kv in d_self._attrs] + + def update(d_self, dct, **kwargs): + """Like dict.update(), set keys from kwargs""" + s = h_self._session + for k, v in dct.items(): + d_self._set_in_session(k, v, s) + for k, v in kwargs.items(): + d_self._set_in_session(k, v, s) + s.update(d_self) + s.commit() + + def get(d_self, key, default): + try: + return d_self[key] + except KeyError: + return default + + # + # database stuff + # + + def refresh(d_self, session=None): + """Sync key-value pairs from database to self + + @param session: use the given session, and do not commit. + + """ + if session is None: + session = h_self._session + session.refresh(d_self) + session.commit() + else: + session.refresh(self.dbrow) + + def delete(d_self, session=None): + """Delete this dictionary from the database + + @param session: use the given session, and do not commit. + """ + if session is None: + session = h_self._session + session.delete(d_self) + session.commit() + else: + session.delete(d_self) + + # helper routine by update() and __setitem__ + def _set_in_session(d_self, key, val, session): + """Modify an existing key or create a key to hold val""" + if key in d_self._forbidden_keys: + raise KeyError(key) + created = None + for i,a in enumerate(d_self._attrs): + if a.name == key: + assert created == None + created = h_self._KeyVal(key, val) + d_self._attrs[i] = created + if not created: + created = h_self._KeyVal(key, val) + d_self._attrs.append(created) + session.save(created) + + mapper(Dict, dict_table, + properties = { + '_attrs': relation(KeyVal, + secondary=link_table, + cascade="all, delete-orphan") + }) + + class _Query (object): + """ + Attributes: + _query - SqlAlchemy.Query object + """ + + def __init__(q_self, query): + q_self._query = query + + def __iter__(q_self): + return q_self.all().__iter__() + + def __getitem__(q_self, item): + return q_self._query.__getitem__(item) + + def filter_by(q_self, **kwargs): + """Return a Query object that restricts to dictionaries containing + the given kwargs""" + + #Note: when we add new types to the key columns, add them here + q = q_self._query + T = h_self._Dict + for kw, arg in kwargs.items(): + if isinstance(arg, (str,unicode)): + q = q.filter(T._attrs.any(name=kw, sval=arg)) + else: + try: + f = float(arg) + except (TypeError, ValueError): + f = None + if f is None: + q = q.filter(T._attrs.any(name=kw, bval=repr(arg))) + else: + q = q.filter(T._attrs.any(name=kw, fval=f)) + + return h_self._Query(q) + + def all(q_self): + """Return an iterator over all matching dictionaries. + + See L{SqlAlchemy.Query} + """ + return q_self._query.all() + + def count(q_self): + """Return the number of matching dictionaries. + + See L{SqlAlchemy.Query} + """ + return q_self._query.count() + + def first(q_self): + """Return some matching dictionary, or None + See L{SqlAlchemy.Query} + """ + return q_self._query.first() + + def all_ordered_by(q_self, key, desc=False): + """Return query results, sorted. + + @type key: string or tuple of string or list of string + @param: keys by which to sort the results. + + @rtype: list of L{DbHandle._Dict} instances + @return: query results, sorted by given keys + """ + + # order_by is not easy to do in SQL based on the data structures we're + # using. Considering we support different data types, it may not be + # possible at all. + # + # It would be easy if 'pivot' or 'crosstab' were provided as part of the + # underlying API, but they are not. For example, read this: + # http://www.simple-talk.com/sql/t-sql-programming/creating-cross-tab-queries-and-pivot-tables-in-sql/ + + # load query results + results = list(q_self.all()) + + if isinstance(key, (tuple, list)): + val_results = [([d[k] for k in key], d) for d in results] + else: + val_results = [(d[key], d) for d in results] + + val_results.sort() #interesting: there is an optional key parameter + if desc: + val_results.reverse() + return [vr[-1] for vr in val_results] + + h_self._KeyVal = KeyVal + h_self._Dict = Dict + h_self._Query = _Query + + def __iter__(h_self): + return h_self.query().__iter__() + + def insert_kwargs(h_self, **dct): + """ + @rtype: DbHandle with reference to self + @return: a DbHandle initialized as a copy of dct + + @type dct: dict-like instance whose keys are strings, and values are + either strings, integers, floats + + @param dct: dictionary to insert + + """ + rval = h_self._Dict() + if dct: rval.update(dct) + return rval + def insert(h_self, dct): + """ + @rtype: DbHandle with reference to self + @return: a DbHandle initialized as a copy of dct + + @type dct: dict-like instance whose keys are strings, and values are + either strings, integers, floats + + @param dct: dictionary to insert + + """ + rval = h_self._Dict() + if dct: rval.update(dct) + return rval + + def query(h_self, **kwargs): + """Construct an SqlAlchemy query, which can be subsequently filtered + using the instance methods of DbQuery""" + + return h_self._Query(h_self._session.query(h_self._Dict)\ + .options(eagerload('_attrs')))\ + .filter_by(**kwargs) + + def createView(h_self, view): + + s = h_self._session; + cols = []; + + for col in view.columns: + if col.name is "id": + continue; + elif isinstance(col.type, (Integer,Float)): + cols.append([col.name,'fval']); + elif isinstance(col.type,String): + cols.append([col.name,'sval']); + elif isinstance(col.type,Binary): + cols.append([col.name,'bval']); + else: + assert "Error: wrong column type in view",view.name; + + # generate raw sql command string + viewsql = crazy_sql_command(view.name, cols, \ + h_self._pair_table.name, \ + h_self._link_table.name); + + #print 'Creating sql view with command:\n', viewsql; + + h_self._engine.execute(viewsql); + s.commit(); + + class MappedClass(object): + pass + + mapper(MappedClass, view) + + return MappedClass + + def session(h_self): + return h_self._session_fn() + + +def db_from_engine(engine, + table_prefix='DbHandle_default_', + trial_suffix='trial', + keyval_suffix='keyval', + link_suffix='link'): + """Create a DbHandle instance + + @type engine: sqlalchemy engine (e.g. from create_engine) + @param engine: connect to this database for transactions + + @type table_prefix: string + @type trial_suffix: string + @type keyval_suffix: string + @type link_suffix: string + + @rtype: DbHandle instance + + @note: The returned DbHandle will use three tables to implement the + many-to-many pattern that it needs: + - I{table_prefix + trial_suffix}, + - I{table_prefix + keyval_suffix} + - I{table_prefix + link_suffix} + + """ + Session = sessionmaker(autoflush=True, autocommit=False) + + metadata = MetaData() + + t_trial = Table(table_prefix+trial_suffix, metadata, + Column('id', Integer, primary_key=True), + Column('create', DateTime), + Column('write', DateTime), + Column('read', DateTime)) + + t_keyval = Table(table_prefix+keyval_suffix, metadata, + Column('id', Integer, primary_key=True), + Column('name', String(128), nullable=False), #name of attribute + Column('ntype', Boolean), + Column('fval', Float(53)), + Column('sval', Text), + Column('bval', Binary)) + + t_link = Table(table_prefix+link_suffix, metadata, + Column('dict_id', Integer, ForeignKey('%s.id' % t_trial), + primary_key=True), + Column('pair_id', Integer, ForeignKey('%s.id' % t_keyval), + primary_key=True)) + + metadata.bind = engine + metadata.create_all() # no-op when tables already exist + #warning: tables can exist, but have incorrect schema + # see bug mentioned in DbHandle constructor + + return DbHandle(Session, engine, t_trial, t_keyval, t_link) + +def sqlite_memory_db(echo=False, **kwargs): + """Return a DbHandle backed by a memory-based database""" + engine = create_engine('sqlite:///:memory:', echo=False) + return db_from_engine(engine, **kwargs) + +def sqlite_file_db(filename, echo=False, **kwargs): + """Return a DbHandle backed by a file-based database""" + engine = create_engine('sqlite:///%s' % filename, echo=False) + return db_from_engine(engine, **kwargs) + +_db_host = 'jais.iro.umontreal.ca' +_pwd='potatomasher'; + +def postgres_db(user, password, host, database, echo=False, **kwargs): + """Create an engine to access a postgres_dbhandle + """ + db_str ='postgres://%(user)s:%(password)s@%(host)s/%(database)s' % locals() + + # should force the app release extra connections releasing + # connections should let us schedule more jobs, since each one + # operates autonomously most of the time, just checking the db + # rarely. TODO: optimize this for large numbers of jobs + pool_size = 0; + engine = create_engine(db_str, pool_size=pool_size, echo=echo) + + return db_from_engine(engine, **kwargs) + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/crap.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/crap.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,256 @@ + +from __future__ import absolute_import + +import os, sys, socket, datetime, copy, tempfile, shutil +from .dconfig import Config, load +from .api0 import sqlite_memory_db +from .api0 import postgres_serial + + +class Job(object): + @classmethod + def is_a_default_option(cls, key, val): + if callable(val): + return False + if key.startswith('_'): + return False + if key in ('job_module', 'job_symbol'): + return False + return True + + @classmethod + def options(cls): + opts = [attr for attr in dir(cls) + if cls.is_a_default_option(attr, getattr(cls, attr))] + opts.sort() + return opts + + + def __init__(self, config): + for key, val in config.items(): + if self.is_a_default_option(key, val): + if hasattr(self, key): + setattr(self, key, val) + else: + raise AttributeError("Job has no such option:", key) + + @classmethod + def print_defaults(cls): + for attr in cls.options(): + print attr, '=', getattr(cls, attr) + + def print_config(self, outfile=sys.stdout): + for attr in self.options(): + print >> outfile, attr, '=', getattr(self, attr) + + def save_config(self, outfile=sys.stdout): + """Like print_config, but with parsable format rather than human-readable format""" + kwargs = dict([(attr,getattr(self, attr)) for attr in self.options()]) + Config(**kwargs).save_file(outfile) + +def run_job(fn=None, cwd=None, + + config_path = 'job_config.py', + result_path = 'job_result.py', + stdout_path = 'stdout', + stderr_path = 'stderr', + work_dir = 'workdir' + ): + cwd = os.getcwd() if cwd is None else cwd + +def _pop_cwd(): + try: + cwd = sys.argv.pop(0) + if not cwd.startswith('/'): + cwd = os.path.join(os.getcwd(), cwd) + except IndexError: + cwd = os.getcwd() + return cwd + + +class RunJob(object): + """ + """ + + path_perf = 'job_perf.py' + path_results = 'job_results.py' + path_config = 'job_config.py' + path_fullconfig = 'job_fullconfig.py' + path_stdout = 'stdout' + path_stderr = 'stderr' + path_workdir = 'workdir' + + def __init__(self, exename): + pass + + def _load_config(self, cwd = None): + cwd = _pop_cwd() if cwd is None else cwd + config = load(os.path.join(cwd, self.path_config)) + return config + + def _load_job_class(self, config=None): + config = self._load_config() if config is None else config + job_module_name = config.job_module + job_symbol = config.job_symbol + #level=0 -> absolute imports + #fromlist=[None] -> evaluate the rightmost element of the module name, when it has + # dots (e.g., "A.Z" will return module Z) + job_module = __import__(job_module_name, fromlist=[None], level=0) + try: + job_class = getattr(job_module, job_symbol) + except: + print >> sys.stderr, "failed to load job class:", job_module_name, job_symbol + raise + return job_class + + def print_config(self): + config = self._load_config() + job_class = self._load_job_class(config) + job = job_class(config) + job.print_config() + + def run(self): + cwd = _pop_cwd() + config = self._load_config(cwd) + job_class = self._load_job_class(config) + job = job_class(config) + + job.save_config(open(os.path.join(cwd, self.path_fullconfig),'w')) + + perf = Config() + perf.host_name = socket.gethostname() + perf.start_time = str(datetime.datetime.now()) + perf.save(os.path.join(cwd, self.path_perf)) + + stdout_orig = sys.stdout + stderr_orig = sys.stderr + + try: + sys.stdout = open(os.path.join(cwd, self.path_stdout), 'w') + sys.stderr = open(os.path.join(cwd, self.path_stderr), 'w') + + # + # mess around with the working directory + # + + wd = os.path.join(cwd, self.path_workdir) + try: + os.mkdir(wd) + except OSError, e: + print >> sys.stderr, "trouble making wordking directory:" + print >> sys.stderr, e + print >> sys.stderr, "ignoring error and proceeding anyway" + try: + os.chdir(wd) + except: + pass + + print >> sys.stderr, "cwd:", os.getcwd() + + # + # run the job... + # + + #TODO load the results file before running the job, to resume job + results = Config() + + job_rval = job.run(results) + + #TODO: use the return value to decide whether the job should be resumed or not, + # False means run is done + # True means the job has yielded, but should be continued + results.save(os.path.join(cwd, self.path_results)) + + finally: + #put back stderr and stdout + sys.stdout = stdout_orig + sys.stderr = stderr_orig + perf.end_time = str(datetime.datetime.now()) + perf.save(os.path.join(cwd, self.path_perf)) + + def defaults(self): + job_class = self._load_job_class() + job_class.print_defaults() + +def standalone_run_job(): + exe = RunJob(sys.argv.pop(0)) + try: + cmd = sys.argv.pop(0) + fn = getattr(exe, cmd) + except IndexError: + fn = getattr(exe, 'run') + except AttributeError: + print >> sys.stderr, "command not supported", cmd + + fn() + +def build_db(cwd, db=None): + """WRITEME""" + db = sqlite_memory_db() if db is None else db + for e in os.listdir(cwd): + e = os.path.join(cwd, e) + try: + e_config = open(os.path.join(e, 'job_config.py')) + except: + e_config = None + + try: + e_sentinel = open(os.path.join(e, '__jobdir__')) + except: + e_sentinel = None + + if not (e_config or e_sentinel): + continue #this is not a job dir + + if e_config: + e_config.close() + config = load(os.path.join(e, 'job_config.py')) + kwargs = copy.copy(config.__dict__) + try: + results = load(os.path.join(e, 'job_results.py')) + kwargs.update(results.__dict__) + except: + pass + try: + perf = load(os.path.join(e, 'job_perf.py')) + kwargs.update(perf.__dict__) + except: + pass + + #TODO: this is a backward-compatibility hack for AISTATS*09 + if 'perf_jobdir' not in kwargs: + kwargs['perf_jobdir'] = e + if 'perf_workdir' not in kwargs: + kwargs['perf_workdir'] = os.path.join(e, 'workdir') + + entry = db.insert(kwargs) + + if e_sentinel: + print >> sys.stderr, "NOT-IMPLEMENTED: RECURSION INTO SUBDIRECTORY", e + return db + + +class RunQuery(object): + + def __init__(self, exename): + pass + + def run(self): + cwd = _pop_cwd() + db = build_db(cwd) + for entry in db.query().all(): + print entry.items() + + +def standalone_query(): + exe = RunQuery(sys.argv.pop(0)) + try: + cmd = sys.argv.pop(0) + fn = getattr(exe, cmd) + except IndexError: + fn = getattr(exe, 'run') + except AttributeError: + print >> sys.stderr, "command not supported", cmd + + fn() + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/dbdict_run.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/dbdict_run.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,66 @@ +import sys +from .tools import DictProxyState, load_state_fn, run_state + +# N.B. +# don't put exotic imports here +# see what I did with def sql() below... since we shouldn't try to import that stuff unless we +# need it. + +_experiment_usage = """Usage: + dbdict-experiment + +Commands: + + help Print help. (You're looking at it.) + + cmdline Obtain experiment configuration by evaluating the commandline + sql Obtain experiment configuration by querying an sql database + +Help on individual commands might be available by typing 'dbdict-experiment help' + +""" + #dbdict Obtain experiment configuration by loading a dbdict file + +def _dispatch_cmd(self, stack=sys.argv): + try: + cmd = stack.pop(0) + except IndexError: + cmd = 'help' + try: + fn = getattr(self, cmd) + except AttributeError: + print >> sys.stderr, "command not supported", cmd + fn = self.help + fn() + +class RunCmdline(object): + def __init__(self): + try: + dct = eval('dict(' + sys.argv.pop(0) + ')') + except Exception, e: + print >> sys.stderr, "Exception:", e + self.help() + return + + run_state(DictProxyState(dct)) + print dct + + def help(self): + print >> sys.stderr, "Usage: dbdict-experiment cmdline " + + +class RunExperiment(object): + """This class handles the behaviour of the dbdict-run script.""" + def __init__(self): + exe = sys.argv.pop(0) + #print 'ARGV', sys.argv + _dispatch_cmd(self) + + def sql(self): + from .dbdict_run_sql import run_sql + return run_sql() + + cmdline = RunCmdline + + def help(self): + print _experiment_usage # string not inlined so as not to fool VIM's folding mechanism diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/dbdict_run_sql.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/dbdict_run_sql.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,434 @@ +"""Code related to using the api0 dictionary table to run experiments. + +This module has been tested with postgres. + +""" + +import sys, os, socket, tempfile, shutil, copy, time +import numpy # for the random wait time in book_unstarted_trial + +import sqlalchemy +from sqlalchemy import create_engine, desc +from sqlalchemy.orm import eagerload +import psycopg2, psycopg2.extensions + +from .api0 import db_from_engine, postgres_db +from .tools import run_state, DictProxyState, COMPLETE, INCOMPLETE, SYMBOL, MODULE # tools +from .dconfig import save_items + +# _TEST CONCURRENCY +# To ensure that concurrency is handled properly by the consumer (book_dct) +# set this flag to True and (manually) run the following test. +# +# Launch two processes side by side, starting one of them a few seconds after the other. +# There is an extra sleep(10) that will delay each process's job dequeue. +# +# You should see that the process that started first gets the job, +# and the process that started second tries to get the same job, +# fails, and then gets another one instead. +_TEST_CONCURRENCY = False + +_help = """Usage: +dbdict-run sql postgres://:@// + + user - postgres username + pass - password + hostname - the network address of the host on which a postgres server is + running (on port ??) + database - a database served by the postgres server on + api0-table - the name (actually, table_prefix) associated with tables, + created by dbdict.api0. + + experiment-root - a local or network path. Network paths begin with ssh:// + E.g. /tmp/blah + ssh://mammouth:blah + ssh://foo@linux.org:/tmp/blah + +Experiment-root is used to store the file results of experiments. If a job with a given +creates file 'a.txt', and directory 'b' with file 'foo.py', then these will be rsync'ed to the +experiment-root when job has finished running. They will be found here: + + ////workdir/a.txt + ////workdir/b/foo.py + +Files 'stdout', 'stderr', and 'state.py' will be created. + + ////stdout - opened for append + ////stderr - opened for append + ////state.py - overwritten with database version + +If a job is restarted or resumed, then those files are rsync'ed back to the current working +directory, and stdout and stderr are re-opened for appending. When a resumed job stops for +the second (or more) time, the cwd is rsync'ed back to the experiment-root. In this way, the +experiment-root accumulates the results of experiments that run. + +""" + +STATUS = 'dbdict_sql_status' +PRIORITY = 'dbdict_sql_priority' +HOST = 'dbdict_sql_hostname' +HOST_WORKDIR = 'dbdict_sql_host_workdir' +PUSH_ERROR = 'dbdict_sql_push_error' + +START = 0 +"""dbdict_status == START means a experiment is ready to run""" + +RUNNING = 1 +"""dbdict_status == RUNNING means a experiment is running on dbdict_hostname""" + +DONE = 2 +"""dbdict_status == DONE means a experiment has completed (not necessarily successfully)""" + +RESTART_PRIORITY = 2.0 +"""Stopped experiments are marked with this priority""" + + +def postgres_serial(user, password, host, database, **kwargs): + """Return a DbHandle instance that communicates with a postgres database at transaction + isolation_level 'SERIALIZABLE'. + + :param user: a username in the database + :param password: the password for the username + :param host: the network address of the host on which the postgres server is running + :param database: a database served by the postgres server + + """ + this = postgres_serial + + if not hasattr(this,'engine'): + def connect(): + c = psycopg2.connect(user=user, password=password, database=database, host=host) + c.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) + return c + pool_size = 0 + this.engine = create_engine('postgres://' + ,creator=connect + ,pool_size=0 # should force the app release connections + ) + + db = db_from_engine(this.engine, **kwargs) + db._is_serialized_session_db = True + return db + + +def add_experiments_to_db(exp_cls, jobs, db, verbose=0, add_dups=False, type_check=None): + """Add experiments paramatrized by exp_cls and jobs[i] to database db. + + Default behaviour is to ignore jobs which are already in the database. + + If type_check is a class (instead of None) then it will be used as a type declaration for + all the elements in each job dictionary. For each key,value pair in the dictionary, there + must exist an attribute,value pair in the class meeting the following criteria: + the attribute and the key are equal, and the types of the values are equal. + + :param exp_cls: The Experiment class to run these experiments. + :param jobs: The parameters of experiments to run. + :type jobs: an iterable object over dictionaries + :param verbose: print which jobs are added and which are skipped + :param add_dups: False will ignore a job if it matches (on all items()) with a db entry. + :type add_dups: Bool + + :returns: list of (Bool,job[i]) in which the flags mean the corresponding job actually was + inserted. + + """ + rval = [] + for job in jobs: + job = copy.copy(job) + do_insert = add_dups or (None is db.query(**job).first()) + + if do_insert: + if type_check: + for k,v in job.items(): + if type(v) != getattr(type_check, k): + raise TypeError('Experiment contains value with wrong type', + ((k,v), getattr(type_check, k))) + + job[STATUS] = START + job[SYMBOL] = exp_cls.__name__ + job[MODULE] = exp_cls.__module__ + job[PRIORITY] = 1.0 + if verbose: + print 'ADDING ', job + db.insert(job) + rval.append((True, job)) + else: + if verbose: + print 'SKIPPING', job + rval.append((False, job)) + + +def book_dct_postgres_serial(db, retry_max_sleep=10.0, verbose=0): + """Find a trial in the lisa_db with status START. + + A trial will be returned with dbdict_status=RUNNING. + + Returns None if no such trial exists in DB. + + This function uses a serial access to the lisadb to guarantee that no other + process will retrieve the same dct. It is designed to facilitate writing + a "consumer" for a Producer-Consumer pattern based on the database. + + """ + print >> sys.stderr, """#TODO: use the priority field, not the status.""" + print >> sys.stderr, """#TODO: ignore entries with key PUSH_ERROR.""" + + s = db._session + + # NB. we need the query and attribute update to be in the same transaction + assert s.autocommit == False + + dcts_seen = set([]) + keep_trying = True + + dct = None + while (dct is None) and keep_trying: + #build a query + q = s.query(db._Dict) + q = q.options(eagerload('_attrs')) #hard-coded in api0 + q = q.filter(db._Dict._attrs.any(name=STATUS, fval=START)) + + #try to reserve a dct + try: + #first() may raise psycopg2.ProgrammingError + dct = q.first() + + if dct is not None: + assert (dct not in dcts_seen) + if verbose: print 'book_unstarted_dct retrieved, ', dct + dct._set_in_session(STATUS, RUNNING, s) + if 1: + if _TEST_CONCURRENCY: + print >> sys.stderr, 'SLEEPING BEFORE BOOKING' + time.sleep(10) + + #commit() may raise psycopg2.ProgrammingError + s.commit() + else: + print >> sys.stderr, 'DEBUG MODE: NOT RESERVING JOB!', dct + #if we get this far, the job is ours! + else: + # no jobs are left + keep_trying = False + except (psycopg2.OperationalError, + sqlalchemy.exceptions.ProgrammingError), e: + #either the first() or the commit() raised + s.rollback() # docs say to do this (or close) after commit raises exception + if verbose: print 'caught exception', e + if dct: + # first() succeeded, commit() failed + dcts_seen.add(dct) + dct = None + wait = numpy.random.rand(1)*retry_max_sleep + if verbose: print 'another process stole our dct. Waiting %f secs' % wait + time.sleep(wait) + return dct + +def book_dct(db): + print >> sys.stderr, """#TODO: use the priority field, not the status.""" + print >> sys.stderr, """#TODO: ignore entries with key self.push_error.""" + + return db.query(dbdict_status=START).first() + +def parse_dbstring(dbstring): + postgres = 'postgres://' + assert dbstring.startswith(postgres) + dbstring = dbstring[len(postgres):] + + #username_and_password + colon_pos = dbstring.find('@') + username_and_password = dbstring[:colon_pos] + dbstring = dbstring[colon_pos+1:] + + colon_pos = username_and_password.find(':') + if -1 == colon_pos: + username = username_and_password + password = None + else: + username = username_and_password[:colon_pos] + password = username_and_password[colon_pos+1:] + + #hostname + colon_pos = dbstring.find('/') + hostname = dbstring[:colon_pos] + dbstring = dbstring[colon_pos+1:] + + #dbname + colon_pos = dbstring.find('/') + dbname = dbstring[:colon_pos] + dbstring = dbstring[colon_pos+1:] + + #tablename + tablename = dbstring + + if password is None: + password = open(os.getenv('HOME')+'/.dbdict_%s'%dbname).readline()[:-1] + if False: + print 'USERNAME', username + print 'PASS', password + print 'HOST', hostname + print 'DB', dbname + print 'TABLE', tablename + + return username, password, hostname, dbname, tablename + +class ExperimentLocation(object): + def __init__(self, root, dbname, tablename, id): + ssh_prefix='ssh://' + if root.startswith(ssh_prefix): + root = root[len(ssh_prefix):] + #at_pos = root.find('@') + colon_pos = root.find(':') + self.host, self.path = root[:colon_pos], root[colon_pos+1:] + else: + self.host, self.path = '', root + self.dbname = dbname + self.tablename = tablename + self.id = id + + def rsync(self, direction): + """The directory at which experiment-related files are stored. + + :returns: ":", of the sort used by ssh and rsync. + """ + path = os.path.join( + ':'.join([self.host, self.path]), + self.dbname, + self.tablename, + self.id) + + if direction == 'push': + rsync_cmd = 'rsync -r * "%s"' % path + elif direction == 'pull': + rsync_cmd = 'rsync -r "%s/*" .' % path + else: + raise Exception('invalid direction', direction) + + rsync_rval = os.system(rsync_cmd) + if rsync_rval != 0: + raise Exception('rsync failure', (rsync_rval, rsync_cmd)) + + def pull(self): + return self.rsync('pull') + + def push(self): + return self.rsync('push') + + def touch(self): + host = self.host + path = os.path.join(self.path, self.dbname, self.tablename, self.id) + ssh_cmd = ('ssh %(host)s "mkdir -p \'%(path)s\' && cd \'%(path)s\' ' + '&& touch stdout stderr && mkdir -p workdir"' % locals()) + ssh_rval = os.system(ssh_cmd) + if 0 != ssh_rval: + raise Exception('ssh failure', (ssh_rval, ssh_cmd)) + + def delete(self): + #something like ssh %s 'rm -Rf %s' should work, but it's pretty scary... + raise NotImplementedError() + +def run_sql(): + try: + username, password, hostname, dbname, tablename = parse_dbstring(sys.argv.pop(0)) + except Exception, e: + print >> sys.stderr, e + print >> sys.stderr, _help + raise + + #set experiment_root + try: + exproot = sys.argv.pop(0) + except: + exproot = os.getcwd() + + #TODO: THIS IS A GOOD IDEA RIGHT? + # It makes module-lookup work based on cwd-relative paths + # But possibly has really annoying side effects? Is there a cleaner + # way to change the import path just for loading the experiment class? + sys.path.insert(0, os.getcwd()) + + #TODO: refactor this so that we can use any kind of database (not just postgres) + + #a serialized session is necessary for jobs not be double-booked by listeners running + #in parallel on the cluster. + db = postgres_serial(user=username, + password=password, + host=hostname, + database=dbname, + table_prefix=tablename) + + while True: + dct = book_dct_postgres_serial(db, verbose=1) + if dct is None: + break + + try: + # + # chdir to a temp folder + # + workdir = tempfile.mkdtemp() + print >> sys.stderr, "INFO RUNNING ID %i IN %s" % (dct.id, workdir) + os.chdir(workdir) + + # not sure where else to put this... + dct[HOST] = socket.gethostname() + dct[HOST_WORKDIR] = workdir + + exploc = ExperimentLocation(exproot, dbname, tablename, str(dct.id)) + + # + # pull cwd contents + # + exploc.touch() + exploc.pull() + + # + # run the experiment + # + old_stdout, old_stderr = sys.stdout, sys.stderr + sys.stdout, sys.stderr = open('stdout', 'a+'), open('stderr', 'a+') + assert RUNNING == dct[STATUS] #set by get_dct + os.chdir(os.path.join(workdir, 'workdir')) + try: + run_rval = run_state(DictProxyState(dct)) + except Exception, e: + run_rval = COMPLETE + print >> sys.stderr, 'Exception:', e + print >> old_stderr, '#TODO: print a bigger traceback to stderr' + sys.stdout, sys.stderr = old_stdout, old_stderr + + # + # push the results back to the experiment_root + # + try: + os.chdir(workdir) + #pickle the state #TODO: write it human-readable + #cPickle.dump(dict((k,v) for k,v in dct.items()), open('state.pickle','w')) + save_items(dct.items(), open('state.py', 'w')) + exploc.push() + except Exception, e: + dct[PUSH_ERROR] = str(e) + raise + + # Cleanup the tempdir + # TODO: put this in a 'finally' block? + # + shutil.rmtree(workdir, ignore_errors=True) + + except: + dct[STATUS] = DONE + dct[PRIORITY] = None + raise + + if run_rval is INCOMPLETE: + #mark the job as needing restarting + dct[STATUS] = START + dct[PRIORITY] = RESTART_PRIORITY + else: + #mark the job as being done + dct[STATUS] = DONE + dct[PRIORITY] = None + + break # don't actually loop + + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/dconfig.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/dconfig.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,145 @@ +import sys +# +# +# Utility +# +# + +class Config (object): + """A class to store experiment configurations. + + Configuration variables are stored in class instance __dict__. This class + ensures that keys are alphanumeric strings, and that values can be rebuilt + from their representations. + + It can be serialized to/from a python file. + + """ + def __init__(self, __dict__=None, **kwargs): + if __dict__: + Config.__checkkeys__(*__dict__.keys()) + self.__dict__.update(__dict__) + if kwargs: + Config.__checkkeys__(*kwargs.keys()) + self.__dict__.update(kwargs) + + def __setattr__(self, attr, value): + Config.__checkkeys__(attr) + self.__dict__[attr] = value + + def __hash__(self): + """Compute a hash string from a dict, based on items(). + + @type dct: dict of hashable keys and values. + @param dct: compute the hash of this dict + + @rtype: string + + """ + items = list(self.items()) + items.sort() + return hash(repr(items)) + + def keys(self): + return self.__dict__.keys() + + def items(self): + return self.__dict__.items() + + def update(self, dct): + Config.__checkkeys__(*dct.keys()) + self.__dict__.update(dct) + + def save_file(self, f=sys.stdout): + items = self.items() + items.sort() + save_dictlike(items, f) + + def save(self, filename): + """Write a python file as a way of serializing a dictionary + + @type filename: string + @param filename: filename to open (overwrite) to save dictionary contents + + @return None + + """ + f = open(filename, 'w') + self.save_file(f) + f.close() + + def update_fromfile(self, filename): + """Read the local variables from a python file + + @type filename: string, filename suitable for __import__() + @param filename: a file whose module variables will be returned as a + dictionary + + @rtype: dict + @return: the local variables of the imported filename + + @note: + This implementation silently ignores all module symbols that don't meet + the standards enforced by L{Config.__checkkeys__}. This is meant to + ignore module special variables. + + """ + #m = __import__(filename) #major security problem, or feature? + f = open(filename) + for line in f: + if line.startswith('#'): + continue + line = line[:-1] #trim the '\n' + tokens = line.split(' ') + try: + key = tokens[0] + assert '=' == tokens[1] + except: + raise Exception('trouble parsing line:', line) + repr_val = ' '.join(tokens[2:]) + val = eval(repr_val) + setattr(self, key, val) + + @staticmethod + def __checkkeys__(*args): + conf = Config() + for k in args: + #must be string + if type(k) != str: + raise KeyError(k) + #mustn't be part of Config class interface + if hasattr(conf, k): + raise KeyError(k) + #all alphanumeric + for c in k: + if c not in ('abcdefghijklmnopqrstuvwxyz' + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + '_0123456789'): + raise KeyError(k) + #no symbols that look like reserved symbols + if k[:2]=='__' and k[-2:]=='__' and len(k)>4: + raise KeyError(k) + +def load(*filenames): + rval = Config() + for filename in filenames: + rval.update_fromfile(filename) + return rval + +def save_items(items, f): + """Write a python file as a way of serializing a dictionary + + @type f: writable file-like object + @param f: the attributes of this object will be written here + + @return None + + """ + print >> f, "#File generated by dbdict.dconfig.save_items" + for key, val in items: + Config.__checkkeys__(key) + # should never raise... illegal keys are not + # supposed to ever get into the dictionary + repr_val = repr(val) + assert val == eval(repr_val) + print >> f, key, '=', repr_val diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/design.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/design.txt Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,316 @@ + +TODO +==== + +- Class interface for jobs (SHAPING UP) + +- How exactly will state work (STILL A MYSTERY sql vs. file vs. script) + - How to insert jobs + - How to retrieve jobs + +- How will the driver work (rsync / file conventions) + + + +Components +========== + +DbDict +------ + +A DbDict is a restricted dictionary. + +It's keys are restricted to simple types, simple enough to be stored into a +database. Strings, integers, floating-points. + + +Experiment +---------- + +An Experiment is a python class with methods like start(), run(), stop(). A +user should make an Experiment class whose run() method does some computations +like training a neural network and measuring test errors, training an SVM, +doing some feature extraction, etc. + +An Experiment's arguments are stored in a DbDict. + +The results of an experiment can be stored in two places: in a DbDict (for +easy SQL searching later on) and in files. + + +ExperimentLauncher +------------------ + +Experiments are not standalone executables, they need to be run by some +additional code. There are several ways to launch an experiment: + +- running on your local machine vs. cluster vs. condor vs. mammouth + +- loading the arguments from the commandline vs. file vs. SQL query + +- running alone, or running in parallel with other experiments. + +The ExperimentLauncher is an executable python program that can run +experiments in these different ways. + + + +Workflow (How to run experiments) +================================= + +First, you'll need a Python program to compute what you want. Think about how +long it will take and whether you will want to be able to resume after being +stopped. + +If you want to run a simple or short experiment that is not restartable, +simply extend dbdict.Experiment like this: + +.. code-block:: python + + class MyExperiment(dbdict.Experiment): + + def __init__(self, dct): + self.dct = dct + + def run(self): + + # compute something + result = self.dct.a * self.dct.b + + # store it + self.dct.result = result + + # write some other results to a file + f = open('some_file', 'w') + print >> f, "hello from the job?" + + # return False or None to indicate the job is done + +If you want to run a more elaborate experiment, then extend dbdict.Experiment +like this: + +.. code-block:: python + + class MyRestartableExperiment(dbdict.Experiment): + + def __init__(self, state): + """Called once per lifetime of the class instance. Can be used to + create new jobs and save them to the database. This function will not + be called when a Job is retrieved from the database. + + Parent creates keys: dbdict_id, dbdict_module, dbdict_symbol, dbdict_state. + + dbdict_state = NEW (possibilities: NEW, RUNNING, DONE) + + """ + + def start(self): + """Called once per lifetime of the compute job. + + This is a good place to initialize internal variables. + + After this function returns, either stop() or run() will be called. + + dbdict_state -> RUNNING + + """ + + def resume(self): + """Called to resume computations on a previously stop()'ed job. The + os.getcwd() is just like it was after some previous stop() command. + + This is a good place to load internal variables from os.getcwd(). + + dbdict_state -> RUNNING + + """ + + def run(self, channel): + """Called after start() or resume(). + + channel() may return different things at different times. + None - run should continue. + 'stop' - the job should save state as soon as possible because + the process may soon be terminated + + When this function returns, dbdict_state -> DONE. + """ + + +Having written your Experiment subclass, you can now test it directly, or +using the ExperimentLauncher. + +.. code-block:: python + + if __name__ == '__main__': + + class Config(object): + a = 3 + b = 50 + + experiment = MyExperiment(Config()) + experiment.run() + +.. code-block:: bash + + dbdict-run my_file.MyExperiment 'dict(a=3, b=50)' + + +To run a batch of experiments on a cluster using 3 concurrent processes using +the SQL-based job list, you have to insert the jobs into a database table, and +then launch 10 instances of dbdict-run. + +.. code-block:: python + + from dbdict.tools import insert, view_table + + def add_jobs(): + experiment_module = 'my_repo.some_file' #match you module name + experiment_symbol = 'MyExperiment' #match your class name + experiment_name = 'showing some stuff' + for a in 3, 4, 5: + for b in 0.0, 0.1, 3.0: + insert(locals(), ignore_duplicates=False) + add_jobs() + + dbdict_run_table('my_launcher_view', experiment_name='showing_some_stuff') + + + +.. code-block:: bash + + dbidispatch --condor dbdict-run sql my_launcher_view + + + +Component Documentation +======================= + +DbDict +------ + + +Experiment +---------- + +class Experiment(object): + + new_stdout == 'job_stdout' + new_stderr == 'job_stderr' + + def remap_stdout(self): + """ + Called before start and resume. + + Parent-class behaviour is to replace sys.stdout with open(self.new_stdout, 'w+'). + + """ + + def remap_stderr(self): + """ + Called before start and resume. + + """ + + def tempdir(self): + """ + Return the recommended filesystem location for temporary files. + + The idea is that this will be a fast, local disk partition, suitable + for temporary storage. + + Files here will not generally be available at the time of resume(). + + The return value of this function may be controlled by one or more + environment variables. + + Will return $DBDICT_EXPERIMENT_TEMPDIR if present. + Failing that, will return $TMP/username-dbdict/hash(self) + Failing that, will return /tmp/username-dbdict/hash(self) + + .. note:: + Maybe we should use Python stdlib's tempdir mechanism. + + """ + + +Environment Variables +===================== + + +TMP - sometimes used by Experiment.tempdir() + +DBDICT_EXPERIMENT_TEMPDIR - used by Experiment.tempdir() + + +JOB DRIVER +---------- + +When a job is started, save: + +- workdir + +- dir + +- network (e.g. mammouth vs. diro... where are jobdir and workdir? + Alternatively, automatically rsync finished jobs' workdir and jobdir to diro.) + + +dbdict-run-job +-------------- + +prepare job dir +~~~~~~~~~~~~~~~~ + +On iro, cd to fringant2/.../job_id/ + +when restarting on mammouth, cd to somewhere and rsync -r fringant2/.../job_id job_id + +after stopping on mammouth rsync -r job_id fringant2/.../job_id + +.. note: + What to do on error in rsync? Where to report problems? + +.. note: + What to do with stderr, stdout? + +job queue table +~~~~~~~~~~~~~~~ +New table: id job_id priority + +This is the table that dbdict-run-job will pop jobs from. + + +How to run experiments using DbDict +----------------------------------- + +- Inherit from tools.Job. This is where you put the logic that takes some + parameters, and computes some results associated with those parameters. + +- Write a loop that creates a bunch of Jobs, and then tests to see if they are + already in the db before inserting them. + +- Insert the new job ids and priority into the qu +- Save result files, and files needed for job resume to os.getcwd(). + +- Save non-result files to a TEMP directory, obtained by e.g. self.gettmpdir() + from inside a Job instance. + +- Load datasets through pylearn. Pylearn has internal configuration that allows + it to find files both on mammouth and DIRO networks. You just call + MNIST.head() or shapeset1.train_valid_test() and bingo. + +- Manage experiments in a postgres database. Which ones have been queued, which + ones are running, which are finished, simple results, etc. + +- To analyze results, select from PG and create a dedicated table in sqlite. + Make columns for all the dictionary keys from all the selected entries. + + + +When running classification experiments +--------------------------------------- + +Have convention of saving (train, valid, test) X (NLL, RATE). +Should the convention be enforced? How? + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/experiment.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/experiment.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,102 @@ + +#These should be +INT = type(0) +FLT = type(0.0) +STR = type('') + +COMPLETE = None #jobs can return this by returning nothing as well +INCOMPLETE = True #jobs can return this and be restarted + + +class Experiment(object): + + new_stdout = 'job_stdout' + new_stderr = 'job_stderr' + + def remap_stdout(self): + """ + Called before start and resume. + + Default behaviour is to replace sys.stdout with open(self.new_stdout, 'w+'). + + """ + if self.new_stdout: + sys.stdout = open(self.new_stdout, 'w+') + + def remap_stderr(self): + """ + Called before start and resume. + + Default behaviour is to replace sys.stderr with open(self.new_stderr, 'w+'). + + """ + if self.new_stderr: + sys.stderr = open(self.new_stderr, 'w+') + + def tempdir(self): + """ + Return the recommended filesystem location for temporary files. + + The idea is that this will be a fast, local disk partition, suitable + for temporary storage. + + Files here will not generally be available at the time of resume(). + + The return value of this function may be controlled by one or more + environment variables. + + Will return $DBDICT_EXPERIMENT_TEMPDIR if present. + Failing that, will return $TMP/username-dbdict/hash(self) + Failing that, will return /tmp/username-dbdict/hash(self) + + .. note:: + Maybe we should use Python stdlib's tempdir mechanism. + + """ + + print >> sys.stderr, "TODO: get tempdir correctly" + return '/tmp/dbdict-experiment' + + + def __init__(self, state): + """Called once per lifetime of the class instance. Can be used to + create new jobs and save them to the database. This function will not + be called when a Job is retrieved from the database. + + Parent creates keys: dbdict_id, dbdict_module, dbdict_symbol, dbdict_status. + + """ + + def start(self): + """Called once per lifetime of the compute job. + + This is a good place to initialize internal variables. + + After this function returns, either stop() or run() will be called. + + dbdict_status -> RUNNING + + """ + + def resume(self): + """Called to resume computations on a previously stop()'ed job. The + os.getcwd() is just like it was after some previous stop() command. + + This is a good place to load internal variables from os.getcwd(). + + dbdict_status -> RUNNING + + """ + return self.start() + + def run(self, channel): + """Called after start() or resume(). + + channel() may return different things at different times. + None - run should continue. + 'stop' - the job should save state as soon as possible because + the process may soon be terminated + + When this function returns, dbdict_status -> DONE. + """ + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/sample_create_jobs.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/sample_create_jobs.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,135 @@ +import sys, subprocess + +from .dconfig import Config +from .tools import Job # tools.py + +# +# Models +# + +class OldSampleJob(Job): + """ Sample Job object. Cut and paste this for your own experiments. """ + + # default parameters for the job + # config file parameters must be from this list. + a = 0 + b = 0 + c = 'a' + d = 0.0 + + def run(self, results, dry_run=False): + # attributes that you set in 'results' will be saved to job_results.py, next to + # job_config.py when this function returns. + results.f = self.a * self.b + + # The current working directory (cwd) is writable and readable + # it will also be left alone after the job terminates + f = open('some_file', 'w') + print >> f, "hello from the job?" + + return True #restart this job + + + +class SampleJobState(): + + # default parameters for the job + # config file parameters must be from this list. + a = 0 + b = 0 + c = 'a' + d = 0.0 + + +sample_job_table = Table('my_table_for_testing', metadata, + Column('a', Integer), + Column('b', Float(53))) + +metadata.create_all() + +mapper(SampleJobState, sample_job_table) + +s = SampleJobState() +s.a = 5 +s.b = 8.2 + +if Session().query(SampleJobState).filter_by(a=5, b=8.2).any(): + break; +else: + Session().save(s).commit() + + +class SampleJob(Job): + """ Sample Job object. Cut and paste this for your own experiments. """ + + def __init__(self, state): + pass + + def start(self): + pass + + def resume(self): + pass + + def run(self, switch = lambda : 'continue'): + # attributes that you set in 'results' will be saved to job_results.py, next to + # job_config.py when this function returns. + params.f = params.a * params.b + + + # The current working directory (cwd) is writable and readable + # it will also be left alone after the job terminates + f = open('some_file', 'w') + print >> f, "hello from the job?" + while True: + time.sleep(5) + if switch() == 'stop': + break + + return True #continue running if possible... + + def stop(self): + pass + + + def run(self, state, dry_run=False): + +def some_jobs(a = 0, + b = 2): + job_module = 'dbdict.sample_create_jobs' + job_symbol = 'SampleJob' + for c in ('a', 'b', 'c'): + for d in [0.0, 9.9]: + yield locals() + +def create(generator): + """Create a set of job directories""" + jobs = [] + dispatch = sys.stdout #open('jobs','w') + + for config in generator: + #print ' config', config + configdir = 'job_%016x'% abs(hash(config)) + jobs.append(configdir) + create_dirs = True + dry_run = 0 + if not dry_run: + if create_dirs and subprocess.call(('mkdir', configdir)): + print >> sys.stderr, 'Error creating directory: ', configdir + else: + #no problem creating the directory + config.save(configdir + '/job_config.py') + print >> dispatch, "dbdict-run-job run", configdir + else: + #print configdir + pass + dispatch.close() + +if __name__ == '__main__': + create(some_jobs()) + + j = SampleJob(SampleJobState()) + j.start() + j.run() + + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/scratch.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/scratch.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,451 @@ +import threading, time, commands, os, sys, math, random, datetime + +import psycopg2, psycopg2.extensions +from sqlalchemy import create_engine, desc +from sqlalchemy.orm import sessionmaker +from sqlalchemy import Table, Column, MetaData, ForeignKey +from sqlalchemy import Integer, String, Float, DateTime +from sqlalchemy.orm import mapper, relation, backref, eagerload +from sqlalchemy.sql import operators, select + + +########## +# +# Connection stuff +# +# + +if 0: + _db_host = 'jais.iro.umontreal.ca' + + def _pwd(): + if not hasattr(_pwd,'rval'): + pw_cmd = 'ssh bergstrj@grieg.iro.umontreal.ca cat .lisa_db' + rval = commands.getoutput(pw_cmd) + return rval + + def engine(): + """Create an engine to access lisa_db on gershwin + + This function caches the return value between calls. + """ + if not hasattr(engine,'rval'): + pw = _pwd() + db_str ='postgres://bergstrj:%s@%s/lisa_db' % (pw,_db_host) + echo = False #spews pseudo-sql to stdout + engine.rval = create_engine(db_str + ,pool_size=1 # should force the app release extra connections + # releasing connections should let us schedule more jobs, since each one operates + # autonomously most of the time, just checking the db rarely. + # TODO: optimize this for large numbers of jobs + ,echo=echo + ) + return engine.rval + + def engine_serializable(): + """Create an engine to access lisa_db on gershwin, which uses serializable + transaction mode. + + This function caches the return value between calls. + """ + + this = engine_serializable + + if not hasattr(this,'rval'): + pw = _pwd() + def connect(): + c = psycopg2.connect(user='bergstrj', + password=pw, + database='lisa_db', + host='gershwin.iro.umontreal.ca') + c.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) + return c + pool_size=0 + this.rval = create_engine('postgres://' + ,creator=connect + ,pool_size=0 # should force the app release connections + ) + return this.rval + + + Session = sessionmaker(bind=engine(), autoflush=True, transactional=True) + SessionSerial = sessionmaker(bind=engine_serializable(), + autoflush=True, transactional=True) + +else: + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + engine = create_engine('sqlite:///:memory:', echo=False) + Session = sessionmaker(bind=engine, autoflush=True, transactional=True) + SessionSerial = Session + + +################ +# +# Database setup +# +table_prefix='bergstrj_scratch_test_' +metadata = MetaData() + +def table_with_id(name, *args): + return Table(name, metadata, + Column('id', Integer, primary_key=True), + *args) + +t_trial = table_with_id(table_prefix+'trial', + Column('desc', String(256)), #comment: why running this trial? + Column('priority', Float(53)), #aka Double + Column('start', DateTime), + Column('finish', DateTime), + Column('host', String(256))) + +t_keyval = table_with_id(table_prefix+'keyval', + Column('name', String(32), nullable=False), #name of attribute + Column('fval', Float(53)), #aka Double + Column('ival', Integer), + Column('sval', String(256))) #TODO: store text (strings of unbounded length) + +t_trial_keyval = table_with_id(table_prefix+'trial_keyval', + Column('trial_id', Integer, ForeignKey('%s.id' % t_trial)), + Column('keyval_id', Integer, ForeignKey('%s.id' % t_keyval))) + +class _KeyVal(object): + @staticmethod + def cache(name, val, session, create=True): + #TODO: consider using a local cache to remove the need to query db + # this takes advantage of common usage, which is to only add KeyVal + # pairs. + #check if it is in the DB + q = session.query(_KeyVal) + if isinstance(val, float): + q = q.filter_by(name=name, fval=val) + elif isinstance(val, int): + q = q.filter_by(name=name, ival=val) + elif isinstance(val, str): + q = q.filter_by(name=name, sval=val) + else: + raise TypeError(val) + rval = q.first() + if rval is None and create: + rval = _KeyVal(name, val) + session.save_or_update(rval) + return rval + + def __init__(self, name, val): + self.name = name + self.val = val + def __get_val(self): + val = None + if self.fval is not None: val = self.fval + if self.ival is not None: val = self.ival + if self.sval is not None: val = self.sval + return val + def __set_val(self, val): + if isinstance(val, float): + self.fval = val + self.ival = None + self.sval = None + elif isinstance(val, int): + self.fval = None + self.ival = val + self.sval = None + elif isinstance(val, str): + self.fval = None + self.ival = None + self.sval = val + else: + raise TypeError(val) + val = property(__get_val, __set_val) + def __repr__(self): + return "" % (self.id, self.name, repr(self.val)) +mapper(_KeyVal, t_keyval) + + +################### +# +# Job interface +# + +class Trial(object): + _native_cols = 'desc', 'priority', 'start', 'finish', 'host' + + #TODO: remove these forbidden keynames, and let all keynames work properly + _forbidden_keynames = set(['filter', 'desc', + 'priority', 'start', 'finish', 'host', + 'create', 'max_sleep', 'max_retry', 'session', 'c', + 'abort', 'complete']) + + #TODO: unittest cases to verify that having these kinds of keys is OK + + class ReserveError(Exception): """reserve failed""" + + @staticmethod + def filter(session=None, **kwargs): + """Construct a query for Trials. + + @param kwargs: each (kw,arg) pair in kwargs, will restrict the list of + Trials to those such that the 'kw' attr has been associated with the job, + and it has value 'arg' + + @return SqlAlchemy query object + + @note: will raise TypeError if any arg in kwargs has a type other than + float, int or string. + + """ + if session is None: + session = Session() + q = session.query(Trial) + for col in Trial._native_cols: + if col in kwargs: + q = q.filter_by(**{col:kwargs[col]}) + del kwargs[col] + for kw, arg in kwargs.items(): + if isinstance(arg, float): + q = q.filter(Trial._attrs.any(name=kw, fval=arg)) + elif isinstance(arg, int): + q = q.filter(Trial._attrs.any(name=kw, ival=arg)) + elif isinstance(arg, str): + q = q.filter(Trial._attrs.any(name=kw, sval=arg)) + else: + raise TypeError(arg) + return q + + @staticmethod + def reserve_unique(query_fn, max_retry=10, max_sleep=5.0): + """Reserve an un-reserved job. + + @param query_fn: build the query for the trial to reserve (see + L{reserve_unique_kw} for example usage). + + @param max_retry: try this many times to reserve a job before raising an exception + + @param max_sleep: L{time.sleep} up to this many seconds between retry attempts + + @return a trial which was reserved (uniquely) by this function call. If + no matching jobs remain, return None. + + """ + s = SessionSerial() + retry = max_retry + trial = None + + while (trial is None) and retry: + + q = query_fn(s.query(Trial)) + q = q.options(eagerload('_attrs')) #TODO is this a good idea? + + trial = q.first() + if trial is None: + return None # no jobs match the query + else: + try: + trial.reserve(session=s) + except Trial.ReserveError, e: + s.rollback() + waittime = random.random() * max_sleep + if debug: print 'another process stole our trial. Waiting %f secs' % wait + time.sleep(waittime) + retry -= 1 + if trial: + s.expunge(trial) + s.close() + return trial + + @staticmethod + def reserve_unique_kw(max_retry=10, max_sleep=5.0, **kwargs): + """Call reserve_unique with a query function that matches jobs with + attributes (and values) given by kwargs. Results are sorted by + priority. + + """ + assert 'start' not in kwargs + assert 'query_fn' not in kwargs + def query_fn(q): + q = q.filter_by(start=None, + **kwargs).order_by(desc(Trial.c.priority)) + return q + + + return Trial.reserve_unique(query_fn, max_retry, max_sleep) + + def __init__(self, desc=None, priority=None, start=None, finish=None, host=None, **kwargs): + self.desc = desc + self.priority = priority + self.start = start + self.finish = finish + self.host = host + self.attrs.update(kwargs) + def __repr__(self): + return "" \ + %(str(self.id), self.desc, self.priority, self.host, + self.start, self.finish) + + def _get_attrs(self): + #This bit of code makes it so that you can type something like: + # + # trial.attrs.score = 50 + # + # It will use the self._attrs list of _KeyVal instances as a backend, + # because these pseudo-attributes (here 'score') are not stored in any + # object's __dict__. + class AttrCatcher(object): + #TODO: make these methods faster with better data structures + def __getattr__(attr_self, attr): + attrlist = self._attrs + for i,a in enumerate(attrlist): + if a.name == attr: + return a.val + raise AttributeError(attr) + def __setattr__(attr_self, attr, val): + n = 0 + s = Session() + assert attr not in Trial._forbidden_keynames + for i,a in enumerate(self._attrs): + if a.name == attr: + attrlist[i] = _KeyVal.cache(attr,val, s) + n += 1 + assert n <= 1 + if n == 0: + self._attrs.append(_KeyVal.cache(attr,val, s)) + s.commit() + def __iter__(_self): + def gen(): + return self._attrs.__iter__() + return gen() + def update(attr_self, dct): + for k,v in dct.items(): + setattr(attr_self, k, v) + + #we can't add attributes to self, so just do this... + return AttrCatcher() #seriously, allocate a new one each time + attrs = property(_get_attrs, doc = ("Provide attribute-like access to the" + " key-value pairs associated with this trial")) + def reserve(self, session): #session should have serialized isolation mode + """Reserve the job for the current process, to the exclusion of all + other processes. In other words, lock it.""" + + serial_self = session.query(Trial).get(self.id) + if serial_self.start is not None: + raise Trial.ReserveError(self.host) + serial_self.start = datetime.datetime.now() + serial_self.host = 'asdf' #TODO: get hostname + try: + session.commit() + except Exception, e: + # different db backends raise different exceptions when a + # commit fails, so here we just treat all problems the same + #s.rollback() # doc says rollback or close after commit exception + session.close() + raise Trial.ReserveError(self.host) + + #Session().refresh(self) #load changes to serial_self into self + + def abort(self): + """Reset job to be available for reservation. + + @return None + + @note: Raises exception if job is finished + + """ + if self.finish is not None: + raise Exception('wtf?') + self.start = None + self.host = None + s = Session() + s.save_or_update(self) + s.commit() + + def complete(self, **kwargs): + """Mark job self as finished and update attrs with kwargs.""" + self.attrs.update(kwargs) + self.finish = datetime.datetime.now() + s = Session() + s.save_or_update(self) + s.commit() + +mapper(Trial, t_trial, + properties = { + '_attrs': relation(_KeyVal, + secondary=t_trial_keyval, + cascade="delete-orphan") + }) + +metadata.create_all(engine) # does nothing when tables already exist + + + + +############ +# +# Test Stuff +# +# + +if __name__ == '__main__': + s = Session() + + def add_some_jobs(): + dvalid, dtest = 'dvalid', 'dtest file' + desc = 'debugging' + + def blah(): + for lr in [0.001, 0.01]: + for scale in [0.0001 * math.sqrt(10.0)**i for i in range(4)]: + for rng_seed in [4, 5, 6]: + for priority in [None, 1, 2]: + yield dict(locals()) + + for kwargs in blah(): + t = Trial(desc=desc, dvalid=dvalid, dtest=dtest, **kwargs) + s.save(t) + + def work(jobtype): + try: + jid = reserve(jobtype) + except StopIteration: + return + + def blah(*args): + print 'blah: ', args + dct = get_next() + + add_some_jobs() + + if 0: + for t in s.query(Trial): + print 'saved:', t, [a.name for a in list(t.attrs)] + + def yield_unique_jobs(): + while True: + rval = Trial.reserve_unique_kw() + if rval is None: + break + else: + yield rval + + if 0: + for job in yield_unique_jobs(): + print 'yielded job', job + job.complete(score=random.random()) + + if 0: + for t in s.query(Trial): + print 'final:', t, [a.name for a in list(t.attrs)] + + q = s.query(Trial) + #q = q.order_by(Trial.attrs) + q = q.filter(Trial._attrs.any(name='rng_seed', ival=5)) + #q = q.filter(Trial._attrs.all(name='lr').order_by(_KeyVal.c.fval)) + + print dir(q) + + + print q + for t in q.all(): + print 'final:', t, [a.name for a in list(t.attrs)] + t.homebrew = 6 + + + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/sql_commands.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/sql_commands.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,36 @@ + +def crazy_sql_command(viewname, cols, keytab, linktab, id_col='id', dict_id='dict_id', pair_id='pair_id'): + + #create or replace view expview as select * from (select id as v1_id, value as nhid from + #test0 where name='nhid') nhid LEFT OUTER JOIN (select id as v2_id, value as lrate from + #test0 where name='lrate') lrate on nhid.v1_id = lrate.v2_id; + + col_queries = [] + colname0 = None + for i, (colname, table_col) in enumerate(cols): + if i == 0: + q = """(select %(dict_id)s as v%(i)s_id, %(table_col)s as %(colname)s + from \"%(keytab)s\", \"%(linktab)s\" + where name='%(colname)s' + and \"%(keytab)s\".%(id_col)s = \"%(linktab)s\".%(pair_id)s) + %(colname)s """ % locals() + colname0 = colname + else: + q = """ LEFT OUTER JOIN (select %(dict_id)s as v%(i)s_id, %(table_col)s as %(colname)s + from \"%(keytab)s\", \"%(linktab)s\" + where name='%(colname)s' + and \"%(keytab)s\".%(id_col)s = \"%(linktab)s\".%(pair_id)s) + %(colname)s + on %(colname0)s.v0_id = %(colname)s.v%(i)s_id""" % locals() + col_queries.append(q) + + header = " create or replace view %s as select %s.v0_id as id, %s from " \ + % (viewname, colname0, (", ".join([c[0] for c in cols]))) + + rval = header + "\n".join(col_queries) + + return rval + + +#print crazy_sql_command('test0', 'expview', (('nhid', 'value'), ('lrate', 'value'), ('a','value'))) + diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/test_api0.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/test_api0.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,351 @@ +from api0 import * +import threading, time, commands, os, sys, math, random, datetime + +import psycopg2, psycopg2.extensions +from sqlalchemy import create_engine, desc +from sqlalchemy.orm import sessionmaker +from sqlalchemy import Table, Column, MetaData, ForeignKey +from sqlalchemy import Integer, String, Float, DateTime, Text, Binary +from sqlalchemy.orm import mapper, relation, backref, eagerload +from sqlalchemy.sql import operators, select + +import unittest + +class T(unittest.TestCase): + + def test_bad_dict_table(self): + """Make sure our crude version of schema checking kinda works""" + engine = create_engine('sqlite:///:memory:', echo=False) + Session = sessionmaker(bind=engine, autoflush=True, transactional=True) + + table_prefix='bergstrj_scratch_test_' + metadata = MetaData() + t_trial = Table(table_prefix+'trial', metadata, + Column('id', Integer, primary_key=True), + Column('desc', String(256)), #comment: why running this trial? + Column('priority', Float(53)), #aka Double + Column('start', DateTime), + Column('finish', DateTime), + Column('host', String(256))) + metadata.create_all(engine) + + try: + h = DbHandle(None, t_trial, None, None) + except ValueError, e: + if e[0] == DbHandle.e_bad_table: + return + self.fail() + + + def go(self): + """Create tables and session_maker""" + engine = create_engine('sqlite:///:memory:', echo=False) + Session = sessionmaker(autoflush=True, transactional=True) + + table_prefix='bergstrj_scratch_test_' + metadata = MetaData() + + t_trial = Table(table_prefix+'trial', metadata, + Column('id', Integer, primary_key=True), + Column('create', DateTime), + Column('write', DateTime), + Column('read', DateTime)) + + t_keyval = Table(table_prefix+'keyval', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(32), nullable=False), #name of attribute + Column('ntype', Integer), + Column('fval', Float(53)), #aka Double + Column('sval', Text), #aka Double + Column('bval', Binary)) #TODO: store text (strings of unbounded length) + + t_trial_keyval = Table(table_prefix+'trial_keyval', metadata, + Column('dict_id', Integer, ForeignKey('%s.id' % t_trial), + primary_key=True), + Column('pair_id', Integer, ForeignKey('%s.id' % t_keyval), + primary_key=True)) + + metadata.bind = engine + metadata.create_all() # does nothing when tables already exist + + self.engine = engine + return Session, t_trial, t_keyval, t_trial_keyval + + + def test_insert_save(self): + + Session, t_dict, t_pair, t_link = self.go() + + db = DbHandle(*self.go()) + + def jobs(): + dvalid, dtest = 'dvalid', 'dtest file' + desc = 'debugging' + for lr in [0.001]: + for scale in [0.0001 * math.sqrt(10.0)**i for i in range(4)]: + for rng_seed in [4, 5, 6]: + for priority in [None, 1]: + yield dict(locals()) + + jlist = list(jobs()) + assert len(jlist) == 1*4*3*2 + for i, dct in enumerate(jobs()): + t = db.insert(**dct) + + #make sure that they really got inserted into the db + orig_keycount = db._session.query(db._KeyVal).count() + self.failUnless(orig_keycount > 0, orig_keycount) + + orig_dctcount = Session().query(db._Dict).count() + self.failUnless(orig_dctcount ==len(jlist), orig_dctcount) + + orig_keycount = Session().query(db._KeyVal).count() + self.failUnless(orig_keycount > 0, orig_keycount) + + #queries + q0list = list(db.query().all()) + q1list = list(db.query()) + q2list = list(db) + + self.failUnless(q0list == q1list, (q0list,q1list)) + self.failUnless(q0list == q2list, (q0list,q1list)) + + self.failUnless(len(q0list) == len(jlist)) + + for i, (j, q) in enumerate(zip(jlist, q0list)): + jitems = list(j.items()) + qitems = list(q.items()) + jitems.sort() + qitems.sort() + if jitems != qitems: + print i + print jitems + print qitems + self.failUnless(jitems == qitems, (jitems, qitems)) + + def test_query_0(self): + Session, t_dict, t_pair, t_link = self.go() + + db = DbHandle(*self.go()) + + def jobs(): + dvalid, dtest = 'dvalid', 'dtest file' + desc = 'debugging' + for lr in [0.001]: + for scale in [0.0001 * math.sqrt(10.0)**i for i in range(4)]: + for rng_seed in [4, 5, 6]: + for priority in [None, 1]: + yield dict(locals()) + + jlist = list(jobs()) + assert len(jlist) == 1*4*3*2 + for i, dct in enumerate(jobs()): + t = db.insert(**dct) + + qlist = list(db.query(rng_seed=5)) + self.failUnless(len(qlist) == len(jlist)/3) + + jlist5 = [j for j in jlist if j['rng_seed'] == 5] + + for i, (j, q) in enumerate(zip(jlist5, qlist)): + jitems = list(j.items()) + qitems = list(q.items()) + jitems.sort() + qitems.sort() + if jitems != qitems: + print i + print jitems + print qitems + self.failUnless(jitems == qitems, (jitems, qitems)) + + def test_delete_keywise(self): + Session, t_dict, t_pair, t_link = self.go() + + db = DbHandle(*self.go()) + + def jobs(): + dvalid, dtest = 'dvalid', 'dtest file' + desc = 'debugging' + for lr in [0.001]: + for scale in [0.0001 * math.sqrt(10.0)**i for i in range(4)]: + for rng_seed in [4, 5, 6]: + for priority in [None, 1]: + yield dict(locals()) + + jlist = list(jobs()) + assert len(jlist) == 1*4*3*2 + for i, dct in enumerate(jobs()): + t = db.insert(**dct) + + orig_keycount = Session().query(db._KeyVal).count() + + del_count = Session().query(db._KeyVal).filter_by(name='rng_seed', + fval=5.0).count() + self.failUnless(del_count == 8, del_count) + + #delete all the rng_seed = 5 entries + qlist_before = list(db.query(rng_seed=5)) + for q in qlist_before: + del q['rng_seed'] + + #check that it's gone from our objects + for q in qlist_before: + self.failUnless('rng_seed' not in q) #via __contains__ + self.failUnless('rng_seed' not in q.keys()) #via keys() + exc=None + try: + r = q['rng_seed'] # via __getitem__ + print 'r,', r + except KeyError, e: + pass + + #check that it's gone from dictionaries in the database + qlist_after = list(db.query(rng_seed=5)) + self.failUnless(qlist_after == []) + + #check that exactly 8 keys were removed + new_keycount = Session().query(db._KeyVal).count() + self.failUnless(orig_keycount == new_keycount + 8, (orig_keycount, + new_keycount)) + + #check that no keys have rng_seed == 5 + gone_count = Session().query(db._KeyVal).filter_by(name='rng_seed', + fval=5.0).count() + self.failUnless(gone_count == 0, gone_count) + + + def test_delete_dictwise(self): + Session, t_dict, t_pair, t_link = self.go() + + db = DbHandle(*self.go()) + + def jobs(): + dvalid, dtest = 'dvalid', 'dtest file' + desc = 'debugging' + for lr in [0.001]: + for scale in [0.0001 * math.sqrt(10.0)**i for i in range(4)]: + for rng_seed in [4, 5, 6]: + for priority in [None, 1]: + yield dict(locals()) + + jlist = list(jobs()) + assert len(jlist) == 1*4*3*2 + for i, dct in enumerate(jobs()): + t = db.insert(**dct) + + orig_keycount = Session().query(db._KeyVal).count() + orig_dctcount = Session().query(db._Dict).count() + self.failUnless(orig_dctcount == len(jlist)) + + #delete all the rng_seed = 5 dictionaries + qlist_before = list(db.query(rng_seed=5)) + for q in qlist_before: + q.delete() + + #check that the right number has been removed + post_dctcount = Session().query(db._Dict).count() + self.failUnless(post_dctcount == len(jlist)-8) + + #check that the remaining ones are correct + for a, b, in zip( + [j for j in jlist if j['rng_seed'] != 5], + Session().query(db._Dict).all()): + self.failUnless(a == b) + + #check that the keys have all been removed + n_keys_per_dict = 8 + new_keycount = Session().query(db._KeyVal).count() + self.failUnless(orig_keycount - 8 * n_keys_per_dict == new_keycount, (orig_keycount, + new_keycount)) + + + def test_setitem_0(self): + Session, t_dict, t_pair, t_link = self.go() + + db = DbHandle(*self.go()) + + b0 = 6.0 + b1 = 9.0 + + job = dict(a=0, b=b0, c='hello') + + dbjob = db.insert(**job) + + dbjob['b'] = b1 + + #check that the change is in db + qjob = Session().query(db._Dict).filter(db._Dict._attrs.any(name='b', + fval=b1)).first() + self.failIf(qjob is dbjob) + self.failUnless(qjob == dbjob) + + #check that the b:b0 key is gone + count = Session().query(db._KeyVal).filter_by(name='b', fval=b0).count() + self.failUnless(count == 0, count) + + #check that the b:b1 key is there + count = Session().query(db._KeyVal).filter_by(name='b', fval=b1).count() + self.failUnless(count == 1, count) + + def test_setitem_1(self): + """replace with different sql type""" + Session, t_dict, t_pair, t_link = self.go() + + db = DbHandle(*self.go()) + + b0 = 6.0 + b1 = 'asdf' # a different dtype + + job = dict(a=0, b=b0, c='hello') + + dbjob = db.insert(**job) + + dbjob['b'] = b1 + + #check that the change is in db + qjob = Session().query(db._Dict).filter(db._Dict._attrs.any(name='b', + sval=b1)).first() + self.failIf(qjob is dbjob) + self.failUnless(qjob == dbjob) + + #check that the b:b0 key is gone + count = Session().query(db._KeyVal).filter_by(name='b', fval=b0).count() + self.failUnless(count == 0, count) + + #check that the b:b1 key is there + count = Session().query(db._KeyVal).filter_by(name='b', sval=b1, + fval=None).count() + self.failUnless(count == 1, count) + + def test_setitem_2(self): + """replace with different number type""" + Session, t_dict, t_pair, t_link = self.go() + + db = DbHandle(*self.go()) + + b0 = 6.0 + b1 = 7 + + job = dict(a=0, b=b0, c='hello') + + dbjob = db.insert(**job) + + dbjob['b'] = b1 + + #check that the change is in db + qjob = Session().query(db._Dict).filter(db._Dict._attrs.any(name='b', + fval=b1)).first() + self.failIf(qjob is dbjob) + self.failUnless(qjob == dbjob) + + #check that the b:b0 key is gone + count = Session().query(db._KeyVal).filter_by(name='b', fval=b0,ntype=1).count() + self.failUnless(count == 0, count) + + #check that the b:b1 key is there + count = Session().query(db._KeyVal).filter_by(name='b', fval=b1,ntype=0).count() + self.failUnless(count == 1, count) + + +if __name__ == '__main__': + unittest.main() diff -r b054271b2504 -r 798607a058bd pylearn/dbdict/tools.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pylearn/dbdict/tools.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,72 @@ +import sys + +from .experiment import COMPLETE, INCOMPLETE + +MODULE = 'dbdict_module' +SYMBOL = 'dbdict_symbol' + +def dummy_channel(*args, **kwargs): + return None + +# +#this proxy object lets experiments use a dict like a state object +# +def DictProxyState(dct): + defaults_obj = [None] + class Proxy(object): + def subdict(s, prefix=''): + rval = {} + for k,v in dct.items(): + if k.startswith(prefix): + rval[k[len(prefix):]] = v + return rval + def use_defaults(s, obj): + defaults_obj[0] = obj + + def __getitem__(s,a): + try: + return dct[a] + except Exception, e: + try: + return getattr(defaults_obj[0], a) + except: + raise e + + def __setitem__(s,a,v): + dct[a] = v + + def __getattr__(s,a): + try: + return dct[a] + except KeyError: + return getattr(defaults_obj[0], a) + def __setattr__(s,a,v): + dct[a] = v + return Proxy() + +def load_state_fn(state): + + # + # load the experiment class + # + dbdict_module_name = getattr(state,MODULE) + dbdict_symbol = getattr(state, SYMBOL) + + try: + dbdict_module = __import__(dbdict_module_name, fromlist=[None], level=0) + dbdict_fn = getattr(dbdict_module, dbdict_symbol) + except: + print >> sys.stderr, "FAILED to load job symbol:", dbdict_module_name, dbdict_symbol + print >> sys.stderr, "PATH", sys.path + raise + + return dbdict_fn + + +def run_state(state, channel = dummy_channel): + fn = load_state_fn(state) + rval = fn(state, channel) + if rval not in (COMPLETE, INCOMPLETE): + print >> sys.stderr, "WARNING: INVALID job function return value" + return rval + diff -r b054271b2504 -r 798607a058bd pylearn/external/__init__.py diff -r b054271b2504 -r 798607a058bd pylearn/io/__init__.py diff -r b054271b2504 -r 798607a058bd setup.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.py Wed Nov 12 22:00:20 2008 -0500 @@ -0,0 +1,14 @@ +#!/bin/env python + +from ez_setup import use_setuptools +use_setuptools() +from setuptools import setup, find_packages, Extension, Library +setup(name="Pylearn", + version="0.1", + description="Pylearn", + long_description="""Machine learning toolkit""", + author="LISA", + author_email="pylearn-dev@googlegroups.com", + packages=find_packages(exclude='tests'), +) +