Skip to content

Commit

Permalink
improved scaling input modifier
Browse files Browse the repository at this point in the history
  • Loading branch information
@ivan-vasilev
ivan-vasilev committed May 15, 2014
1 parent ea22196 commit 4fdb640
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 65 deletions.
44 changes: 40 additions & 4 deletions nn-core/src/main/java/com/github/neuralnetworks/input/ScalingInputFunction.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.github.neuralnetworks.input;

import com.github.neuralnetworks.calculation.neuronfunctions.TensorFunction;
import com.github.neuralnetworks.training.TrainingInputProvider;
import com.github.neuralnetworks.util.Tensor;

/**
Expand All @@ -10,13 +11,41 @@ public class ScalingInputFunction implements TensorFunction {

private static final long serialVersionUID = 1L;

private float scale;
private Float scale;
private float[] inputScales;

/**
* All inputs are scaled according to the scale value
*/
public ScalingInputFunction(float scale) {
super();
this.scale = scale;
}

/**
* All inputs are scaled according to the scale value
*/
public ScalingInputFunction(TrainingInputProvider input) {
super();

input.reset();

for (int i = 0; i < input.getInputSize(); i++) {
input.beforeSample();
float[] in = input.getNextInput();
if (inputScales == null) {
inputScales = new float[in.length];
}
input.afterSample();

for (int j = 0; j < in.length; j++) {
inputScales[j] = Math.abs(in[j]) > inputScales[j] ? Math.abs(in[j]) : inputScales[j];
}
}

input.reset();
}

public float getScale() {
return scale;
}
Expand All @@ -28,8 +57,15 @@ public void setScale(float scale) {
@Override
public void value(Tensor inputOutput) {
float[] elements = inputOutput.getElements();
for (int i = 0; i < elements.length; i++) {
elements[i] /= scale;
}
if (inputScales != null) {
int s = inputOutput.getSize() / inputScales.length;
for (int i = 0; i < inputOutput.getSize(); i++) {
elements[inputOutput.getStartOffset() + i] /= inputScales[i / s];
}
} else {
for (int i = inputOutput.getStartOffset(); i < inputOutput.getStartOffset() + inputOutput.getSize(); i++) {
elements[i] /= scale;
}
};
}
}
6 changes: 5 additions & 1 deletion nn-core/src/main/java/com/github/neuralnetworks/input/SimpleInputProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,13 @@ public class SimpleInputProvider extends TrainingInputProviderImpl {
private float[][] input;
private float[][] target;

public SimpleInputProvider(float[][] input, float[][] target) {
public SimpleInputProvider(float[][] input) {
super();
this.input = input;
}

public SimpleInputProvider(float[][] input, float[][] target) {
super();
this.input = input;
this.target = target;
}
Expand Down
15 changes: 15 additions & 0 deletions nn-core/src/test/java/com/github/neuralnetworks/test/GeneralTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import com.github.neuralnetworks.architecture.types.NNFactory;
import com.github.neuralnetworks.calculation.memory.ValuesProvider;
import com.github.neuralnetworks.calculation.neuronfunctions.SoftmaxFunction;
import com.github.neuralnetworks.input.ScalingInputFunction;
import com.github.neuralnetworks.input.SimpleInputProvider;
import com.github.neuralnetworks.training.random.MersenneTwisterRandomInitializer;
import com.github.neuralnetworks.training.random.NNRandomInitializer;
import com.github.neuralnetworks.util.Environment;
Expand Down Expand Up @@ -353,4 +355,17 @@ public void testMatrix() {
assertEquals(10, m2.get(1, 0), 0);
assertEquals(11, m2.get(1, 1), 0);
}

@Test
public void testScaling() {
float[][] input = new float[][] { {1, 3}, { -2, 1.5f } };
ScalingInputFunction si = new ScalingInputFunction(new SimpleInputProvider(input));
Matrix m = TensorFactory.matrix(input);
si.value(m);

assertEquals(0.5f, m.get(0, 0), 0);
assertEquals(-1f, m.get(0, 1), 0);
assertEquals(1f, m.get(1, 0), 0);
assertEquals(0.5f, m.get(1, 1), 0);
}
}
17 changes: 1 addition & 16 deletions nn-samples/src/main/java/com/github/neuralnetworks/samples/iris/IrisInputProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,27 +17,12 @@ public class IrisInputProvider extends TrainingInputProviderImpl {
private Random random;
private int currentIndex;

public IrisInputProvider(InputConverter targetConverter, boolean useRandom, boolean scale) {
public IrisInputProvider(InputConverter targetConverter, boolean useRandom) {
super();
this.targetConverter = targetConverter;
this.random = useRandom ? new Random() : null;
this.target = new float[3];
reset();

if (scale) {
for (int i = 0; i < irisData[0].length; i++) {
float max = irisData[0][i];
for (int j = 0; j < irisData.length; j++) {
if (irisData[j][i] > max) {
max = irisData[j][i];
}
}

for (int j = 0; j < irisData.length; j++) {
irisData[j][i] /= max;
}
}
}
}

@Override
Expand Down
103 changes: 59 additions & 44 deletions nn-samples/src/test/java/com/github/neuralnetworks/samples/test/IrisTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,13 @@
import com.github.neuralnetworks.architecture.types.RBM;
import com.github.neuralnetworks.architecture.types.StackedAutoencoder;
import com.github.neuralnetworks.calculation.OutputError;
import com.github.neuralnetworks.calculation.neuronfunctions.AparapiSoftmax;
import com.github.neuralnetworks.input.MultipleNeuronsOutputError;
import com.github.neuralnetworks.input.ScalingInputFunction;
import com.github.neuralnetworks.samples.iris.IrisInputProvider;
import com.github.neuralnetworks.samples.iris.IrisTargetMultiNeuronOutputConverter;
import com.github.neuralnetworks.training.DNNLayerTrainer;
import com.github.neuralnetworks.training.OneStepTrainer;
import com.github.neuralnetworks.training.TrainerFactory;
import com.github.neuralnetworks.training.TrainingInputProvider;
import com.github.neuralnetworks.training.backpropagation.BackPropagationAutoencoder;
import com.github.neuralnetworks.training.backpropagation.BackPropagationTrainer;
import com.github.neuralnetworks.training.events.EarlyStoppingListener;
Expand All @@ -49,8 +48,10 @@ public void testMLPSigmoidBP() {
NeuralNetworkImpl mlp = NNFactory.mlpSigmoid(new int[] { 4, 2, 3 }, true);

// training and testing data providers
IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
trainInputProvider.addInputModifier(new ScalingInputFunction(trainInputProvider));
IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
testInputProvider.addInputModifier(new ScalingInputFunction(testInputProvider));
OutputError outputError = new MultipleNeuronsOutputError();

// trainer
Expand All @@ -71,37 +72,39 @@ public void testMLPSigmoidBP() {
assertEquals(0, bpt.getOutputError().getTotalNetworkError(), 0.1);
}

@Test
public void testMaxout() {
// execution mode
Environment.getInstance().setExecutionMode(EXECUTION_MODE.SEQ);
Environment.getInstance().setUseWeightsSharedMemory(true);

// create the network
NeuralNetworkImpl mlp = NNFactory.maxout(new int[] { 4, 2, 3 }, false, new AparapiSoftmax());

// training and testing data providers
IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
OutputError outputError = new MultipleNeuronsOutputError();

// trainer
BackPropagationTrainer<?> bpt = TrainerFactory.backPropagation(mlp, trainInputProvider, testInputProvider, outputError, new NNRandomInitializer(new MersenneTwisterRandomInitializer(-0.001f, 0.001f), 0.5f), 0.01f, 0.7f, 0f, 0f, 0f, 1, 1, 1000);

// log data
bpt.addEventListener(new LogTrainingListener(Thread.currentThread().getStackTrace()[1].getMethodName()));

// early stopping
//bpt.addEventListener(new EarlyStoppingListener(testInputProvider, 100, 0.015f));

// train
bpt.train();

// test
bpt.test();

assertEquals(0, bpt.getOutputError().getTotalNetworkError(), 0.1);
}
// @Test
// public void testMaxout() {
// // execution mode
// Environment.getInstance().setExecutionMode(EXECUTION_MODE.SEQ);
// Environment.getInstance().setUseWeightsSharedMemory(true);
//
// // create the network
// NeuralNetworkImpl mlp = NNFactory.maxout(new int[] { 4, 2, 3 }, false, new AparapiSoftmax());
//
// // training and testing data providers
// IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
// trainInputProvider.addInputModifier(new ScalingInputFunction(trainInputProvider));
// IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
// testInputProvider.addInputModifier(new ScalingInputFunction(testInputProvider));
// OutputError outputError = new MultipleNeuronsOutputError();
//
// // trainer
// BackPropagationTrainer<?> bpt = TrainerFactory.backPropagation(mlp, trainInputProvider, testInputProvider, outputError, new NNRandomInitializer(new MersenneTwisterRandomInitializer(-0.001f, 0.001f), 0.5f), 0.01f, 0.7f, 0f, 0f, 0f, 1, 1, 1000);
//
// // log data
// bpt.addEventListener(new LogTrainingListener(Thread.currentThread().getStackTrace()[1].getMethodName()));
//
// // early stopping
// //bpt.addEventListener(new EarlyStoppingListener(testInputProvider, 100, 0.015f));
//
// // train
// bpt.train();
//
// // test
// bpt.test();
//
// assertEquals(0, bpt.getOutputError().getTotalNetworkError(), 0.1);
// }

/**
* Contrastive Divergence testing
Expand All @@ -117,8 +120,10 @@ public void testRBMCDSigmoidBP() {
RBM rbm = NNFactory.rbm(4, 3, true);

// training and testing input providers
TrainingInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
TrainingInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
trainInputProvider.addInputModifier(new ScalingInputFunction(trainInputProvider));
IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
testInputProvider.addInputModifier(new ScalingInputFunction(testInputProvider));
MultipleNeuronsOutputError error = new MultipleNeuronsOutputError();

// trainers
Expand Down Expand Up @@ -152,9 +157,12 @@ public void testDBN() {

dbn.setLayerCalculator(NNFactory.lcSigmoid(dbn, null));

TrainingInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
TrainingInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);

IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
trainInputProvider.addInputModifier(new ScalingInputFunction(trainInputProvider));

IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
testInputProvider.addInputModifier(new ScalingInputFunction(testInputProvider));

// rbm trainers for each layer
AparapiCDTrainer firstTrainer = TrainerFactory.cdSigmoidBinaryTrainer(dbn.getFirstNeuralNetwork(), null, null, null, new NNRandomInitializer(new MersenneTwisterRandomInitializer(-0.01f, 0.01f)), 0.01f, 0.5f, 0f, 0f, 1, 150, 1000, true);
AparapiCDTrainer lastTrainer = TrainerFactory.cdSigmoidBinaryTrainer(dbn.getLastNeuralNetwork(), null, null, null, new NNRandomInitializer(new MersenneTwisterRandomInitializer(-0.01f, 0.01f)), 0.01f, 0.5f, 0f, 0f, 1, 150, 1000, true);
Expand Down Expand Up @@ -197,8 +205,12 @@ public void testAE() {
Autoencoder ae = NNFactory.autoencoderSigmoid(4, 3, true);

// training, testing and error
TrainingInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
TrainingInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
trainInputProvider.addInputModifier(new ScalingInputFunction(trainInputProvider));

IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
testInputProvider.addInputModifier(new ScalingInputFunction(testInputProvider));

MultipleNeuronsOutputError error = new MultipleNeuronsOutputError();

// backpropagation autoencoder training
Expand Down Expand Up @@ -228,8 +240,11 @@ public void testSAE() {
StackedAutoencoder sae = NNFactory.saeSigmoid(new int[] { 4, 4, 3 }, true);

// data and error providers
TrainingInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
TrainingInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false, true);
IrisInputProvider trainInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
trainInputProvider.addInputModifier(new ScalingInputFunction(trainInputProvider));

IrisInputProvider testInputProvider = new IrisInputProvider(new IrisTargetMultiNeuronOutputConverter(), false);
testInputProvider.addInputModifier(new ScalingInputFunction(testInputProvider));

// stacked networks
Autoencoder firstNN = sae.getFirstNeuralNetwork();
Expand Down

0 comments on commit 4fdb640

Please sign in to comment.