# HG changeset patch # User James Bergstra # Date 1229546370 18000 # Node ID 6a7f3d83c72b03b6e26b683c9e4c3e8ddfead057 # Parent 990fa151bb10fd1ab91da54f40838b43178c743b dbdict stuff diff -r 990fa151bb10 -r 6a7f3d83c72b pylearn/dbdict/__init__.py --- a/pylearn/dbdict/__init__.py Wed Dec 17 15:38:37 2008 -0500 +++ b/pylearn/dbdict/__init__.py Wed Dec 17 15:39:30 2008 -0500 @@ -1,1 +1,1 @@ -from api0 import sqlite_file_db, sqlite_memory_db +from api0 import sqlite_file_db, sqlite_memory_db, postgres_db diff -r 990fa151bb10 -r 6a7f3d83c72b pylearn/dbdict/api0.py --- a/pylearn/dbdict/api0.py Wed Dec 17 15:38:37 2008 -0500 +++ b/pylearn/dbdict/api0.py Wed Dec 17 15:39:30 2008 -0500 @@ -208,6 +208,9 @@ except KeyError: return default + def __str__(self): + return 'Dict'+ str(dict(self)) + # # database stuff # @@ -495,9 +498,6 @@ engine = create_engine('sqlite:///%s' % filename, echo=False) return db_from_engine(engine, **kwargs) -_db_host = 'jais.iro.umontreal.ca' -_pwd='potatomasher'; - def postgres_db(user, password, host, database, echo=False, **kwargs): """Create an engine to access a postgres_dbhandle """ diff -r 990fa151bb10 -r 6a7f3d83c72b pylearn/dbdict/newstuff.py --- a/pylearn/dbdict/newstuff.py Wed Dec 17 15:38:37 2008 -0500 +++ b/pylearn/dbdict/newstuff.py Wed Dec 17 15:39:30 2008 -0500 @@ -11,7 +11,7 @@ ### misc ################################################################################ -class DD(defaultdict): +class DD(dict): def __getattr__(self, attr): return self[attr] def __setattr__(self, attr, value): @@ -43,6 +43,7 @@ return obj def flatten(obj): + """nested dictionary -> flat dictionary with '.' notation """ d = {} def helper(d, prefix, obj): if isinstance(obj, (str, int, float)): @@ -60,9 +61,10 @@ return d def expand(d): - def dd(): - return DD(dd) - struct = dd() + """inverse of flatten()""" + #def dd(): + #return DD(dd) + struct = DD() for k, v in d.iteritems(): if k == '': raise NotImplementedError() @@ -70,7 +72,7 @@ keys = k.split('.') current = struct for k2 in keys[:-1]: - current = current[k2] + current = current.setdefault(k2, DD()) current[keys[-1]] = v #convert(v) return struct @@ -157,15 +159,19 @@ class Channel(object): - COMPLETE = None - INCOMPLETE = True + COMPLETE = property(lambda s:None, + doc=("Experiments should return this value to " + "indicate that they are done (if not done, return `Incomplete`")) + INCOMPLETE = property(lambda s:True, + doc=("Experiments should return this value to indicate that " + "they are not done (if done return `COMPLETE`)")) - START = 0 - """dbdict.status == START means a experiment is ready to run""" - RUNNING = 1 - """dbdict.status == RUNNING means a experiment is running on dbdict_hostname""" - DONE = 2 - """dbdict.status == DONE means a experiment has completed (not necessarily successfully)""" + START = property(lambda s: 0, + doc="dbdict.status == START means a experiment is ready to run") + RUNNING = property(lambda s: 1, + doc="dbdict.status == RUNNING means a experiment is running on dbdict_hostname") + DONE = property(lambda s: 2, + doc="dbdict.status == DONE means a experiment has completed (not necessarily successfully)") # Methods to be used by the experiment to communicate with the channel @@ -504,7 +510,7 @@ lr=0.03 """ state = expand(parse(*strings)) - state.dbdict.experiment = experiment + state.setdefault('dbdict', DD()).experiment = experiment experiment = resolve(experiment) workdir = options.workdir or format_d(state, sep=',', space = False) channel = StandardChannel(workdir, diff -r 990fa151bb10 -r 6a7f3d83c72b pylearn/dbdict/sql.py --- a/pylearn/dbdict/sql.py Wed Dec 17 15:38:37 2008 -0500 +++ b/pylearn/dbdict/sql.py Wed Dec 17 15:39:30 2008 -0500 @@ -1,16 +1,15 @@ -import sys +import sys, os, copy import sqlalchemy from sqlalchemy import create_engine, desc from sqlalchemy.orm import eagerload -import copy - import psycopg2, psycopg2.extensions -from api0 import db_from_engine, postgres_db +from api0 import db_from_engine, postgres_db, DbHandle +EXPERIMENT = 'dbdict.experiment' STATUS = 'dbdict.status' PRIORITY = 'dbdict.sql.priority' HOST = 'dbdict.sql.hostname' @@ -122,9 +121,15 @@ return db.query(dbdict_status=START).first() + +########### +# Connect +########### + def parse_dbstring(dbstring): postgres = 'postgres://' - assert dbstring.startswith(postgres) + if not dbstring.startswith(postgres): + raise ValueError('For now, dbdict dbstrings must start with postgres://', dbstring) dbstring = dbstring[len(postgres):] #username_and_password @@ -154,7 +159,8 @@ tablename = dbstring if password is None: - password = open(os.getenv('HOME')+'/.dbdict_%s'%dbname).readline()[:-1] + password = get_password(hostname, dbname) + if False: print 'USERNAME', username print 'PASS', password @@ -164,6 +170,50 @@ return username, password, hostname, dbname, tablename +def get_password(hostname, dbname): + """Return the current user's password for a given database + + :TODO: Replace this mechanism with a section in the pylearn configuration file + """ + password_path = os.getenv('HOME')+'/.dbdict_%s'%dbname + try: + password = open(password_path).readline()[:-1] #cut the trailing newline + except: + raise ValueError( 'Failed to read password for db "%s" from %s' % (dbname, password_path)) + return password + +def db(dbstring): + username, password, hostname, dbname, tablename = parse_dbstring(dbstring) + try: + return postgres_db(username, password, hostname, dbname, table_prefix=tablename) + except: + print 'pw', password + + +########### +# Queue +########### + +def insert_dict(jobdict, db, force_dup=False): + """Insert a new `job` dictionary into database `db`. + + :param force_dup: forces insertion even if an identical dictionary is already in the db + + """ + job = copy.copy(jobdict) + do_insert = force_dup or (None is db.query(**job).first()) + if do_insert: + job[STATUS] = START + job[PRIORITY] = 1.0 + return db.insert(job) + else: + return None + +def insert_job(experiment_fn, state, db, force_dup=False): + state = copy.copy(state) + state[EXPERIMENT] = experiment_fn.__module__ + '.' + experiment_fn.__name__ + return insert_dict(state, db, force_dup=force_dup) + def add_experiments_to_db(jobs, db, verbose=0, add_dups=False, type_check=None): """Add experiments paramatrized by jobs[i] to database db. @@ -209,6 +259,3 @@ rval.append((False, job)) - - -