ift6266: utils/seriestables/series.py comparison

comparison utils/seriestables/series.py @ 218:4c137f16b013

Modifications pour stocker des timestamps/cpuclock avec chaque rangée créée, selon suggestion de Yoshua ce matin

author	fsavard
date	Wed, 10 Mar 2010 20:13:45 -0500
parents	a96fa4de06d2
children	e172ef73cdc5

comparison

equal deleted inserted replaced

-:a96fa4de06d2
+:4c137f16b013
 from tables import *
 import numpy
+import time
 '''
 The way these "IsDescription constructor" work is simple: write the
 code as if it were in a file, then exec()ute it, leaving us with
 a local-scoped LocalDescription which may be used to call createTable.
 It's a small hack, but it's necessary as the names of the columns
 are retrieved based on the variable name, which we can't programmatically set
 otherwise.
 '''
-def get_beginning_description_n_ints(int_names, int_width=64):
+def _get_description_timestamp_cpuclock_columns(store_timestamp, store_cpuclock, pos=0):
+toexec = ""
+if store_timestamp:
+toexec += "\ttimestamp = Time32Col(pos="+str(pos)+")\n"
+pos += 1
+if store_cpuclock:
+toexec += "\tcpuclock = Float64Col(pos="+str(pos)+")\n"
+pos += 1
+return toexec, pos
+def _get_description_n_ints(int_names, int_width=64, pos=0):
 """
 Begins construction of a class inheriting from IsDescription
 to construct an HDF5 table with index columns named with int_names.
 See Series().__init__ to see how those are used.
 """
 int_constructor = "Int64Col"
 if int_width == 32:
 int_constructor = "Int32Col"
-toexec = "class LocalDescription(IsDescription):\n"
+toexec = ""
-pos = 0
 for n in int_names:
 toexec += "\t" + n + " = " + int_constructor + "(pos=" + str(pos) + ")\n"
+pos += 1
-return toexec
+return toexec, pos
-def get_description_with_n_ints_n_floats(int_names, float_names, int_width=64, float_width=32):
+def _get_description_with_n_ints_n_floats(int_names, float_names,
+int_width=64, float_width=32,
+store_timestamp=True, store_cpuclock=True):
 """
 Constructs a class to be used when constructing a table with PyTables.
 This is useful to construct a series with an index with multiple levels.
 E.g. if you want to index your "validation error" with "epoch" first, then
 Names of the float (e.g. error) columns
 int_width : {'32', '64'}
 Type of ints.
 float_width : {'32', '64'}
 Type of floats.
+store_timestamp : bool
+See __init__ of Series
+store_cpuclock : bool
+See __init__ of Series
 Returns
 -------
 A class object, to pass to createTable()
 """
-toexec = get_beginning_description_n_ints(int_names, int_width=int_width)
+toexec = "class LocalDescription(IsDescription):\n"
+toexec_, pos = _get_description_timestamp_cpuclock_columns(store_timestamp, store_cpuclock)
+toexec += toexec_
+toexec_, pos = _get_description_n_ints(int_names, int_width=int_width, pos=pos)
+toexec += toexec_
 float_constructor = "Float32Col"
 if float_width == 64:
 float_constructor = "Float64Col"
-pos = len(int_names)
 for n in float_names:
 toexec += "\t" + n + " = " + float_constructor + "(pos=" + str(pos) + ")\n"
+pos += 1
 exec(toexec)
 return LocalDescription
 class Series():
-def __init__(self, table_name, hdf5_file, index_names=('epoch',), title="", hdf5_group='/'):
+def __init__(self, table_name, hdf5_file, index_names=('epoch',),
+title="", hdf5_group='/',
+store_timestamp=True, store_cpuclock=True):
 """Basic arguments each Series must get.
 Parameters
 ----------
 table_name : str
 Columns to use as index for elements in the series, other example would be ('epoch', 'minibatch'). This would then allow you to call append(index, element) with index made of two ints, one for epoch index, one for minibatch index in epoch.
 title : str
 Title to attach to this table as metadata. Can contain spaces and be longer then the table_name.
 hdf5_group : str
 Path of the group (kind of a file) in the HDF5 file under which to create the table.
+store_timestamp : bool
+Whether to create a column for timestamps and store them with each record.
+store_cpuclock : bool
+Whether to create a column for cpu clock and store it with each record.
 """
 self.table_name = table_name
 self.hdf5_file = hdf5_file
 self.index_names = index_names
 self.title = title
+self.store_timestamp = store_timestamp
+self.store_cpuclock = store_cpuclock
 def append(self, index, element):
 raise NotImplementedError
+def _timestamp_cpuclock(self, newrow):
+newrow["timestamp"] = time.time()
+newrow["cpuclock"] = time.clock()
 # To put in a series dictionary instead of a real series, to do nothing
 # when we don't want a given series to be saved.
 class DummySeries():
 def append(self, index, element):
 pass
 class ErrorSeries(Series):
-def __init__(self, error_name, table_name, hdf5_file, index_names=('epoch',), title="", hdf5_group='/'):
+def __init__(self, error_name, table_name,
-Series.__init__(self, table_name, hdf5_file, index_names, title)
+hdf5_file, index_names=('epoch',),
+title="", hdf5_group='/',
+store_timestamp=True, store_cpuclock=True):
+Series.__init__(self, table_name, hdf5_file, index_names, title, store_timestamp, store_cpuclock)
 self.error_name = error_name
 table_description = self._get_table_description()
 self._table = hdf5_file.createTable(hdf5_group, self.table_name, table_description, title=title)
 def _get_table_description(self):
-return get_description_with_n_ints_n_floats(self.index_names, (self.error_name,))
+return _get_description_with_n_ints_n_floats(self.index_names, (self.error_name,))
 def append(self, index, error):
 """
 Parameters
 ----------
 # Columns for index in table are based on index_names
 for col_name, value in zip(self.index_names, index):
 newrow[col_name] = value
 newrow[self.error_name] = error
+self._timestamp_cpuclock(newrow)
 newrow.append()
 self.hdf5_file.flush()
 # were appended, which should be a red flag.
 assert len(self._buffer) < self.reduce_every
 # Outside of class to fix an issue with exec in Python 2.6.
 # My sorries to the God of pretty code.
-def _BasicStatisticsSeries_construct_table_toexec(index_names):
+def _BasicStatisticsSeries_construct_table_toexec(index_names, store_timestamp, store_cpuclock):
-toexec = get_beginning_description_n_ints(index_names)
+toexec = "class LocalDescription(IsDescription):\n"
-bpos = len(index_names)
+toexec_, pos = _get_description_timestamp_cpuclock_columns(store_timestamp, store_cpuclock)
-toexec += "\tmean = Float32Col(pos=" + str(bpos) + ")\n"
+toexec += toexec_
-toexec += "\tmin = Float32Col(pos=" + str(bpos+1) + ")\n"
-toexec += "\tmax = Float32Col(pos=" + str(bpos+2) + ")\n"
+toexec_, pos = _get_description_n_ints(index_names, pos=pos)
-toexec += "\tstd = Float32Col(pos=" + str(bpos+3) + ")\n"
+toexec += toexec_
+toexec += "\tmean = Float32Col(pos=" + str(pos) + ")\n"
+toexec += "\tmin = Float32Col(pos=" + str(pos+1) + ")\n"
+toexec += "\tmax = Float32Col(pos=" + str(pos+2) + ")\n"
+toexec += "\tstd = Float32Col(pos=" + str(pos+3) + ")\n"
 # This creates "LocalDescription", which we may then use
 exec(toexec)
 return LocalDescription
 series_name : str
 Not optional here. Will be prepended with "Basic statistics for "
 stats_functions : dict, optional
 Dictionary with a function for each key "mean", "min", "max", "std". The function must take whatever is passed to append(...) and return a single number (float).
 """
-def __init__(self, table_name, hdf5_file, stats_functions=basic_stats_functions, index_names=('epoch',), title="", hdf5_group='/'):
+def __init__(self, table_name, hdf5_file,
-Series.__init__(self, table_name, hdf5_file, index_names, title)
+stats_functions=basic_stats_functions,
+index_names=('epoch',), title="", hdf5_group='/',
+store_timestamp=True, store_cpuclock=True):
+Series.__init__(self, table_name, hdf5_file, index_names, title, store_timestamp, store_cpuclock)
 self.hdf5_group = hdf5_group
 self.stats_functions = stats_functions
 self._construct_table()
 def _construct_table(self):
-table_description = _BasicStatisticsSeries_construct_table_toexec(self.index_names)
+table_description = _BasicStatisticsSeries_construct_table_toexec(self.index_names, self.store_timestamp, self.store_cpuclock)
 self._table = self.hdf5_file.createTable(self.hdf5_group, self.table_name, table_description)
 def append(self, index, array):
 """
 newrow["mean"] = self.stats_functions['mean'](array)
 newrow["min"] = self.stats_functions['min'](array)
 newrow["max"] = self.stats_functions['max'](array)
 newrow["std"] = self.stats_functions['std'](array)
+self._timestamp_cpuclock(newrow)
 newrow.append()
 self.hdf5_file.flush()

Mercurial > ift6266

comparison utils/seriestables/series.py @ 218:4c137f16b013