Mercurial > pylearn

changeset 1393:8ecc6da87350

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
allow to read header of dataset from a string. This is needed to allow using gzip file.
author Frederic Bastien <nouiz@nouiz.org>
date Thu, 06 Jan 2011 13:33:47 -0500
parents 2d3cbbb36178
children 48e8292e22e7
files pylearn/io/filetensor.py
diffstat 1 files changed, 13 insertions(+), 4 deletions(-) [+]
line wrap: on
line diff
--- a/pylearn/io/filetensor.py	Mon Dec 20 18:09:11 2010 -0500
+++ b/pylearn/io/filetensor.py	Thu Jan 06 13:33:47 2011 -0500
@@ -52,14 +52,17 @@
     s_array = numpy.fromstring(s, dtype='int32')
     return s_array.item()
 
-def _read_header(f, debug=False):
+def _read_header(f, debug=False, fromstring=False):
     """
     :returns: data type, element size, rank, shape, size
     """
     #what is the data type of this matrix?
     #magic_s = f.read(4)
     #magic = numpy.fromstring(magic_s, dtype='int32')
-    magic = _read_int32(f)
+    if fromstring:
+        magic = numpy.fromstring(f[0:4], dtype='int32').item()
+    else:
+        magic = _read_int32(f)
     magic_t, elsize = _magic_dtype[magic]
     if debug: 
         print 'header magic', magic, magic_t, elsize
@@ -67,11 +70,17 @@
         raise NotImplementedError('packed matrix not supported')
 
     #what is the rank of the tensor?
-    ndim = _read_int32(f)
+    if fromstring:
+        ndim = numpy.fromstring(f[4:8], dtype='int32').item()
+    else:
+        ndim = _read_int32(f)
     if debug: print 'header ndim', ndim
 
     #what are the dimensions of the tensor?
-    dim = numpy.fromfile(f, dtype='int32', count=max(ndim,3))[:ndim]
+    if fromstring:
+        dim = numpy.fromstring(f[8:8+max(ndim,3)*4], dtype='int32')[:ndim]
+    else:
+        dim = numpy.fromfile(f, dtype='int32', count=max(ndim,3))[:ndim]
     dim_size = _prod(dim)
     if debug: print 'header dim', dim, dim_size