Mercurial > pylearn
annotate filetensor.py @ 323:44f94ffe28f7
version works with file and folder, returns null if not in hg. Also a function to get all modules, but seems to get junk at the same time
| author | Thierry Bertin-Mahieux <bertinmt@iro.umontreal.ca> |
|---|---|
| date | Thu, 12 Jun 2008 17:12:45 -0400 |
| parents | 82ba488b2c24 |
| children | 040cb796f4e0 |
| rev | line source |
|---|---|
| 33 | 1 """ |
| 2 Read and write the matrix file format described at | |
|
72
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
3 U{http://www.cs.nyu.edu/~ylclab/data/norb-v1.0/index.html} |
| 33 | 4 |
| 5 The format is for dense tensors: | |
| 6 | |
|
72
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
7 - magic number indicating type and endianness - 4bytes |
|
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
8 - rank of tensor - int32 |
|
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
9 - dimensions - int32, int32, int32, ... |
|
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
10 - <data> |
| 33 | 11 |
| 12 The number of dimensions and rank is slightly tricky: | |
|
72
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
13 - for scalar: rank=0, dimensions = [1, 1, 1] |
|
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
14 - for vector: rank=1, dimensions = [?, 1, 1] |
|
2b6656b2ef52
Changed docs slightly
Joseph Turian <turian@iro.umontreal.ca>
parents:
35
diff
changeset
|
15 - for matrix: rank=2, dimensions = [?, ?, 1] |
| 33 | 16 |
| 17 For rank >= 3, the number of dimensions matches the rank exactly. | |
| 18 | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
19 |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
20 @todo: add complex type support |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
21 |
| 33 | 22 """ |
| 23 import sys | |
| 24 import numpy | |
| 25 | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
26 def _prod(lst): |
| 33 | 27 p = 1 |
| 28 for l in lst: | |
| 29 p *= l | |
| 30 return p | |
| 31 | |
| 32 _magic_dtype = { | |
| 33 0x1E3D4C51 : ('float32', 4), | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
34 #0x1E3D4C52 : ('packed matrix', 0), #what is a packed matrix? |
| 33 | 35 0x1E3D4C53 : ('float64', 8), |
| 36 0x1E3D4C54 : ('int32', 4), | |
| 35 | 37 0x1E3D4C55 : ('uint8', 1), |
| 33 | 38 0x1E3D4C56 : ('int16', 2), |
| 39 } | |
| 40 _dtype_magic = { | |
| 41 'float32': 0x1E3D4C51, | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
42 #'packed matrix': 0x1E3D4C52, |
| 33 | 43 'float64': 0x1E3D4C53, |
| 44 'int32': 0x1E3D4C54, | |
| 35 | 45 'uint8': 0x1E3D4C55, |
| 33 | 46 'int16': 0x1E3D4C56 |
| 47 } | |
| 48 | |
| 49 # | |
| 50 # TODO: implement item selection: | |
| 51 # e.g. load('some mat', subtensor=(:6, 2:5)) | |
| 52 # | |
| 53 # This function should be memory efficient by: | |
| 54 # - allocating an output matrix at the beginning | |
| 55 # - seeking through the file, reading subtensors from multiple places | |
| 56 def read(f, subtensor=None, debug=False): | |
| 57 """Load all or part of file 'f' into a numpy ndarray | |
| 58 | |
| 59 If f is a string, it will be treated as a filename, and opened in read mode. | |
| 60 | |
| 61 If subtensor is not None, it should be like the argument to | |
| 62 numpy.ndarray.__getitem__. The following two expressions should return | |
| 63 equivalent ndarray objects, but the one on the left may be faster and more | |
| 64 memory efficient if the underlying file f is big. | |
| 65 | |
| 66 read(f, subtensor) <===> read(f)[*subtensor] | |
| 67 | |
| 68 Support for subtensors is currently spotty, so check the code to see if your | |
| 69 particular type of subtensor is supported. | |
| 70 | |
| 71 """ | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
72 def _read_int32(f): |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
73 s = f.read(4) |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
74 s_array = numpy.fromstring(s, dtype='int32') |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
75 return s_array.item() |
| 33 | 76 |
| 77 if isinstance(f, str): | |
| 78 if debug: print 'f', f | |
| 79 f = file(f, 'r') | |
| 80 | |
| 81 #what is the data type of this matrix? | |
| 82 #magic_s = f.read(4) | |
| 83 #magic = numpy.fromstring(magic_s, dtype='int32') | |
| 84 magic = _read_int32(f) | |
| 85 magic_t, elsize = _magic_dtype[magic] | |
| 86 if debug: | |
| 87 print 'header magic', magic, magic_t, elsize | |
| 88 if magic_t == 'packed matrix': | |
| 89 raise NotImplementedError('packed matrix not supported') | |
| 90 | |
| 91 #what is the rank of the tensor? | |
| 92 ndim = _read_int32(f) | |
| 93 if debug: print 'header ndim', ndim | |
| 94 | |
| 95 #what are the dimensions of the tensor? | |
| 96 dim = numpy.fromfile(f, dtype='int32', count=max(ndim,3))[:ndim] | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
97 dim_size = _prod(dim) |
| 33 | 98 if debug: print 'header dim', dim, dim_size |
| 99 | |
| 100 rval = None | |
| 101 if subtensor is None: | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
102 rval = numpy.fromfile(f, dtype=magic_t, count=_prod(dim)).reshape(dim) |
| 33 | 103 elif isinstance(subtensor, slice): |
| 104 if subtensor.step not in (None, 1): | |
| 105 raise NotImplementedError('slice with step', subtensor.step) | |
| 106 if subtensor.start not in (None, 0): | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
107 bytes_per_row = _prod(dim[1:]) * elsize |
| 33 | 108 raise NotImplementedError('slice with start', subtensor.start) |
| 109 dim[0] = min(dim[0], subtensor.stop) | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
110 rval = numpy.fromfile(f, dtype=magic_t, count=_prod(dim)).reshape(dim) |
| 33 | 111 else: |
| 112 raise NotImplementedError('subtensor access not written yet:', subtensor) | |
| 113 | |
| 114 return rval | |
| 115 | |
| 116 def write(f, mat): | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
117 """Write a numpy.ndarray to file. |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
118 |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
119 If 'f' is a string, then it will be interpreted as a filename. This filename |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
120 will be opened in 'w+' mode, and (automatically) closed at the end of the function. |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
121 """ |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
122 def _write_int32(f, i): |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
123 i_array = numpy.asarray(i, dtype='int32') |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
124 if 0: print 'writing int32', i, i_array |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
125 i_array.tofile(f) |
| 33 | 126 if isinstance(f, str): |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
127 f = file(f, 'w+') |
| 33 | 128 |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
129 try: |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
130 _write_int32(f, _dtype_magic[str(mat.dtype)]) |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
131 except KeyError: |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
132 raise TypeError('Invalid ndarray dtype for filetensor format', mat.dtype) |
|
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
133 |
| 33 | 134 _write_int32(f, len(mat.shape)) |
| 135 shape = mat.shape | |
| 136 if len(shape) < 3: | |
| 137 shape = list(shape) + [1] * (3 - len(shape)) | |
|
248
82ba488b2c24
polished filetensor a little
James Bergstra <bergstrj@iro.umontreal.ca>
parents:
72
diff
changeset
|
138 if 0: print 'writing shape =', shape |
| 33 | 139 for sh in shape: |
| 140 _write_int32(f, sh) | |
| 141 mat.tofile(f) | |
| 142 |