Mercurial > pylearn

comparison pylearn/io/filetensor.py @ 1394:48e8292e22e7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
new version _read_header for filetensor that allow to work gzip file.
author Frederic Bastien <nouiz@nouiz.org>
date Fri, 07 Jan 2011 11:42:38 -0500
parents 8ecc6da87350
children e06c0ff46d2a
comparison
equal deleted inserted replaced
1393:8ecc6da87350 1394:48e8292e22e7
50 """unpack a 4-byte integer from the current position in file f""" 50 """unpack a 4-byte integer from the current position in file f"""
51 s = f.read(4) 51 s = f.read(4)
52 s_array = numpy.fromstring(s, dtype='int32') 52 s_array = numpy.fromstring(s, dtype='int32')
53 return s_array.item() 53 return s_array.item()
54 54
55 def _read_header(f, debug=False, fromstring=False): 55 def _read_header(f, debug=False, fromgzip=False):
56 """ 56 """
57 :returns: data type, element size, rank, shape, size 57 :returns: data type, element size, rank, shape, size
58 """ 58 """
59 #what is the data type of this matrix? 59 #what is the data type of this matrix?
60 #magic_s = f.read(4) 60 #magic_s = f.read(4)
61 #magic = numpy.fromstring(magic_s, dtype='int32') 61 #magic = numpy.fromstring(magic_s, dtype='int32')
62 if fromstring: 62 magic = _read_int32(f)
63 magic = numpy.fromstring(f[0:4], dtype='int32').item()
64 else:
65 magic = _read_int32(f)
66 magic_t, elsize = _magic_dtype[magic] 63 magic_t, elsize = _magic_dtype[magic]
67 if debug: 64 if debug:
68 print 'header magic', magic, magic_t, elsize 65 print 'header magic', magic, magic_t, elsize
69 if magic_t == 'packed matrix': 66 if magic_t == 'packed matrix':
70 raise NotImplementedError('packed matrix not supported') 67 raise NotImplementedError('packed matrix not supported')
71 68
72 #what is the rank of the tensor? 69 #what is the rank of the tensor?
73 if fromstring: 70 ndim = _read_int32(f)
74 ndim = numpy.fromstring(f[4:8], dtype='int32').item()
75 else:
76 ndim = _read_int32(f)
77 if debug: print 'header ndim', ndim 71 if debug: print 'header ndim', ndim
78 72
79 #what are the dimensions of the tensor? 73 #what are the dimensions of the tensor?
80 if fromstring: 74 if fromgzip:
81 dim = numpy.fromstring(f[8:8+max(ndim,3)*4], dtype='int32')[:ndim] 75 d = f.read(max(ndim,3)*4)
76 dim = numpy.fromstring(d, dtype='int32')[:ndim]
82 else: 77 else:
83 dim = numpy.fromfile(f, dtype='int32', count=max(ndim,3))[:ndim] 78 dim = numpy.fromfile(f, dtype='int32', count=max(ndim,3))[:ndim]
84 dim_size = _prod(dim) 79 dim_size = _prod(dim)
85 if debug: print 'header dim', dim, dim_size 80 if debug: print 'header dim', dim, dim_size
86 81