--- a/pylearn/dbdict/__init__.py	Wed Dec 17 15:38:37 2008 -0500
+++ b/pylearn/dbdict/__init__.py	Wed Dec 17 15:39:30 2008 -0500
@@ -1,1 +1,1 @@
-from api0 import sqlite_file_db, sqlite_memory_db
+from api0 import sqlite_file_db, sqlite_memory_db, postgres_db

--- a/pylearn/dbdict/api0.py	Wed Dec 17 15:38:37 2008 -0500
+++ b/pylearn/dbdict/api0.py	Wed Dec 17 15:39:30 2008 -0500
@@ -208,6 +208,9 @@
                 except KeyError:
                     return default
 
+            def __str__(self):
+                return 'Dict'+ str(dict(self))
+
             #
             # database stuff
             #
@@ -495,9 +498,6 @@
     engine = create_engine('sqlite:///%s' % filename, echo=False)
     return db_from_engine(engine, **kwargs)
 
-_db_host = 'jais.iro.umontreal.ca'
-_pwd='potatomasher';
-
 def postgres_db(user, password, host, database, echo=False, **kwargs):
     """Create an engine to access a postgres_dbhandle
     """

--- a/pylearn/dbdict/newstuff.py	Wed Dec 17 15:38:37 2008 -0500
+++ b/pylearn/dbdict/newstuff.py	Wed Dec 17 15:39:30 2008 -0500
@@ -11,7 +11,7 @@
 ### misc
 ################################################################################
 
-class DD(defaultdict):
+class DD(dict):
     def __getattr__(self, attr):
         return self[attr]
     def __setattr__(self, attr, value):
@@ -43,6 +43,7 @@
         return obj
 
 def flatten(obj):
+    """nested dictionary -> flat dictionary with '.' notation """
     d = {}
     def helper(d, prefix, obj):
         if isinstance(obj, (str, int, float)):
@@ -60,9 +61,10 @@
     return d
 
 def expand(d):
-    def dd():
-        return DD(dd)
-    struct = dd()
+    """inverse of flatten()"""
+    #def dd():
+        #return DD(dd)
+    struct = DD()
     for k, v in d.iteritems():
         if k == '':
             raise NotImplementedError()
@@ -70,7 +72,7 @@
             keys = k.split('.')
         current = struct
         for k2 in keys[:-1]:
-            current = current[k2]
+            current = current.setdefault(k2, DD())
         current[keys[-1]] = v #convert(v)
     return struct
 
@@ -157,15 +159,19 @@
 
 class Channel(object):
 
-    COMPLETE = None
-    INCOMPLETE = True
+    COMPLETE = property(lambda s:None,
+            doc=("Experiments should return this value to "
+                "indicate that they are done (if not done, return `Incomplete`"))
+    INCOMPLETE = property(lambda s:True,
+            doc=("Experiments should return this value to indicate that "
+            "they are not done (if done return `COMPLETE`)"))
     
-    START = 0
-    """dbdict.status == START means a experiment is ready to run"""
-    RUNNING = 1
-    """dbdict.status == RUNNING means a experiment is running on dbdict_hostname"""
-    DONE = 2
-    """dbdict.status == DONE means a experiment has completed (not necessarily successfully)"""
+    START = property(lambda s: 0,
+            doc="dbdict.status == START means a experiment is ready to run")
+    RUNNING = property(lambda s: 1,
+            doc="dbdict.status == RUNNING means a experiment is running on dbdict_hostname")
+    DONE = property(lambda s: 2,
+            doc="dbdict.status == DONE means a experiment has completed (not necessarily successfully)")
 
     # Methods to be used by the experiment to communicate with the channel
 
@@ -504,7 +510,7 @@
             lr=0.03
     """
     state = expand(parse(*strings))
-    state.dbdict.experiment = experiment
+    state.setdefault('dbdict', DD()).experiment = experiment
     experiment = resolve(experiment)
     workdir = options.workdir or format_d(state, sep=',', space = False)
     channel = StandardChannel(workdir,

--- a/pylearn/dbdict/sql.py	Wed Dec 17 15:38:37 2008 -0500
+++ b/pylearn/dbdict/sql.py	Wed Dec 17 15:39:30 2008 -0500
@@ -1,16 +1,15 @@
 
-import sys
+import sys, os, copy
 
 import sqlalchemy
 from sqlalchemy import create_engine, desc
 from sqlalchemy.orm import eagerload
-import copy
-
 import psycopg2, psycopg2.extensions 
 
-from api0 import db_from_engine, postgres_db
+from api0 import db_from_engine, postgres_db, DbHandle
 
 
+EXPERIMENT = 'dbdict.experiment'
 STATUS = 'dbdict.status'
 PRIORITY = 'dbdict.sql.priority'
 HOST = 'dbdict.sql.hostname'
@@ -122,9 +121,15 @@
 
     return db.query(dbdict_status=START).first()
 
+
+###########
+# Connect
+###########
+
 def parse_dbstring(dbstring):
     postgres = 'postgres://'
-    assert dbstring.startswith(postgres)
+    if not dbstring.startswith(postgres):
+        raise ValueError('For now, dbdict dbstrings must start with postgres://', dbstring)
     dbstring = dbstring[len(postgres):]
 
     #username_and_password
@@ -154,7 +159,8 @@
     tablename = dbstring
 
     if password is None:
-        password = open(os.getenv('HOME')+'/.dbdict_%s'%dbname).readline()[:-1]
+        password = get_password(hostname, dbname)
+
     if False:
         print 'USERNAME', username
         print 'PASS', password
@@ -164,6 +170,50 @@
 
     return username, password, hostname, dbname, tablename
 
+def get_password(hostname, dbname):
+    """Return the current user's password for a given database
+
+    :TODO: Replace this mechanism with a section in the pylearn configuration file
+    """
+    password_path = os.getenv('HOME')+'/.dbdict_%s'%dbname
+    try:
+        password = open(password_path).readline()[:-1] #cut the trailing newline
+    except:
+        raise ValueError( 'Failed to read password for db "%s" from %s' % (dbname, password_path))
+    return password
+
+def db(dbstring):
+    username, password, hostname, dbname, tablename = parse_dbstring(dbstring)
+    try:
+        return postgres_db(username, password, hostname, dbname, table_prefix=tablename)
+    except:
+        print 'pw', password
+
+
+###########
+# Queue
+###########
+
+def insert_dict(jobdict, db, force_dup=False):
+    """Insert a new `job` dictionary into database `db`.
+
+    :param force_dup: forces insertion even if an identical dictionary is already in the db
+
+    """
+    job = copy.copy(jobdict)
+    do_insert = force_dup or (None is db.query(**job).first())
+    if do_insert:
+        job[STATUS] = START
+        job[PRIORITY] = 1.0
+        return db.insert(job)
+    else:
+        return None
+
+def insert_job(experiment_fn, state, db, force_dup=False):
+    state = copy.copy(state)
+    state[EXPERIMENT] = experiment_fn.__module__ + '.' + experiment_fn.__name__
+    return insert_dict(state, db, force_dup=force_dup)
+
 
 def add_experiments_to_db(jobs, db, verbose=0, add_dups=False, type_check=None):
     """Add experiments paramatrized by jobs[i] to database db.
@@ -209,6 +259,3 @@
             rval.append((False, job))
 
 
-
-
-