These are chat archives for dmlc/mxnet

10th
Aug 2017
Andre Bieler
@abieler
Aug 10 2017 07:06
Well, it is a big matrix (> 25 GB for 32bit floats), sure you have that much RAM available? (not disk space)
Daniel Haehn
@haehn
Aug 10 2017 17:23
yea i got 512 GB RAM
@abieler
its 16 bits
and also the error should be different, no?
print X_train.shape, Y_train.shape, X_train.nbytes
print X_val.shape, Y_val.shape, X_val.nbytes
print X_test.shape, Y_test.shape, X_test.nbytes
(212700, 6, 119, 119) (212700,) 36144536400
(70900, 6, 119, 119) (70900,) 12048178800
(70900, 6, 119, 119) (70900,) 12048178800

#
#
#
t0 = time.time()
batch_size = 100
train_iter = mx.io.NDArrayIter(data=X_train, label=Y_train, batch_size=batch_size)
val_iter = mx.io.NDArrayIter(data=X_val, label=Y_val, batch_size=batch_size)
test_iter = mx.io.NDArrayIter(data=X_test, label=Y_test, batch_size=batch_size)
print 'iterators configured', time.time()-t0, 'seconds'

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-6-713633301713> in <module>()
      4 t0 = time.time()
      5 batch_size = 100
----> 6 train_iter = mx.io.NDArrayIter(data=X_train, label=Y_train, batch_size=batch_size)
      7 val_iter = mx.io.NDArrayIter(data=X_val, label=Y_val, batch_size=batch_size)
      8 test_iter = mx.io.NDArrayIter(data=X_test, label=Y_test, batch_size=batch_size)

/home/dhaehn/D1/lib/python2.7/site-packages/mxnet/io.pyc in __init__(self, data, label, batch_size, shuffle, last_batch_handle, data_name, label_name)
    577         super(NDArrayIter, self).__init__(batch_size)
    578 
--> 579         self.data = _init_data(data, allow_empty=False, default_name=data_name)
    580         self.label = _init_data(label, allow_empty=True, default_name=label_name)
    581 

/home/dhaehn/D1/lib/python2.7/site-packages/mxnet/io.pyc in _init_data(data, allow_empty, default_name)
    485             except:
    486                 raise TypeError(("Invalid type '%s' for %s, "  % (type(v), k)) + \
--> 487                     "should be NDArray or numpy.ndarray")
    488 
    489     return list(data.items())

TypeError: Invalid type '<type 'numpy.ndarray'>' for data, should be NDArray or numpy.ndarray
$ free -h
              total        used        free      shared  buff/cache   available
Mem:           503G        118G        156G        695M        228G        384G
Swap:          2.0G        1.1G        955M