Add script for downloading MNIST data. Reorganize directories

tpsjr7 · Sep 20, 2013 · d15bd59 · d15bd59
1 parent aa41074
commit d15bd59
Show file tree

Hide file tree

Showing 10 changed files with 82 additions and 0 deletions.
diff --git a/Destin/DavisDestin/mnist/.gitignore b/Destin/DavisDestin/mnist/.gitignore
@@ -0,0 +1 @@
+data/
diff --git a/Destin/DavisDestin/config_16 → Destin/DavisDestin/mnist/config_16 b/Destin/DavisDestin/config_16 → Destin/DavisDestin/mnist/config_16
diff --git a/Destin/DavisDestin/config_32 → Destin/DavisDestin/mnist/config_32 b/Destin/DavisDestin/config_32 → Destin/DavisDestin/mnist/config_32
diff --git a/Destin/DavisDestin/destin_mnist_16.sh → Destin/DavisDestin/mnist/destin_mnist_16.sh b/Destin/DavisDestin/destin_mnist_16.sh → Destin/DavisDestin/mnist/destin_mnist_16.sh
diff --git a/Destin/DavisDestin/destin_mnist_32.sh → Destin/DavisDestin/mnist/destin_mnist_32.sh b/Destin/DavisDestin/destin_mnist_32.sh → Destin/DavisDestin/mnist/destin_mnist_32.sh
diff --git a/Destin/DavisDestin/generate_mnist_set_16.m → ...DavisDestin/mnist/generate_mnist_set_16.m b/Destin/DavisDestin/generate_mnist_set_16.m → ...DavisDestin/mnist/generate_mnist_set_16.m
diff --git a/Destin/DavisDestin/generate_mnist_set_32.m → ...DavisDestin/mnist/generate_mnist_set_32.m b/Destin/DavisDestin/generate_mnist_set_32.m → ...DavisDestin/mnist/generate_mnist_set_32.m
diff --git a/Destin/DavisDestin/mnist/generate_mnist_sets.sh b/Destin/DavisDestin/mnist/generate_mnist_sets.sh
@@ -0,0 +1,36 @@
+#bin/bash
+
+url="http://yann.lecun.com/exdb/mnist"
+files="train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte"
+dir="data"
+
+cd "$(dirname "$0")"
+
+if [ ! -d $dir ]
+then
+    mkdir -p $dir
+fi
+
+for f in $files; do
+
+    if [ -e $dir/$f ]
+    then
+        break
+    fi
+
+    wget -v -P $dir $url/$f.gz
+    gunzip $dir/$f.gz
+
+done
+
+if [ ! -e $dir/train.bin ]
+then
+    echo "Generating 16x16 data ..."
+    octave -q generate_mnist_set_16.m
+fi
+
+if [ ! -e $dir/destin_train_32.bin ]
+then
+    echo "Generating 32x32 data ..."
+    octave -q generate_mnist_set_16.m
+fi
diff --git a/Destin/DavisDestin/mnist/loadMNISTImages.m b/Destin/DavisDestin/mnist/loadMNISTImages.m
@@ -0,0 +1,26 @@
+function images = loadMNISTImages(filename)
+%loadMNISTImages returns a 28x28x[number of MNIST images] matrix containing
+%the raw MNIST images
+
+fp = fopen(filename, 'rb');
+assert(fp ~= -1, ['Could not open ', filename, '']);
+
+magic = fread(fp, 1, 'int32', 0, 'ieee-be');
+assert(magic == 2051, ['Bad magic number in ', filename, '']);
+
+numImages = fread(fp, 1, 'int32', 0, 'ieee-be');
+numRows = fread(fp, 1, 'int32', 0, 'ieee-be');
+numCols = fread(fp, 1, 'int32', 0, 'ieee-be');
+
+images = fread(fp, inf, 'unsigned char');
+images = reshape(images, numCols, numRows, numImages);
+images = permute(images,[2 1 3]);
+
+fclose(fp);
+
+% Reshape to #pixels x #examples
+images = reshape(images, size(images, 1) * size(images, 2), size(images, 3));
+% Convert to double and rescale to [0,1]
+images = double(images) / 255;
+
+end
diff --git a/Destin/DavisDestin/mnist/loadMNISTLabels.m b/Destin/DavisDestin/mnist/loadMNISTLabels.m
@@ -0,0 +1,19 @@
+function labels = loadMNISTLabels(filename)
+%loadMNISTLabels returns a [number of MNIST images]x1 matrix containing
+%the labels for the MNIST images
+
+fp = fopen(filename, 'rb');
+assert(fp ~= -1, ['Could not open ', filename, '']);
+
+magic = fread(fp, 1, 'int32', 0, 'ieee-be');
+assert(magic == 2049, ['Bad magic number in ', filename, '']);
+
+numLabels = fread(fp, 1, 'int32', 0, 'ieee-be');
+
+labels = fread(fp, inf, 'unsigned char');
+
+assert(size(labels,1) == numLabels, 'Mismatch in label count');
+
+fclose(fp);
+
+end