Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@ivan-vasilev
ivan-vasilev committed Apr 16, 2014
1 parent ce64cc2 commit 4fe1fbf
Show file tree
Hide file tree
Showing 19 changed files with 118 additions and 105 deletions.
8 changes: 4 additions & 4 deletions nn-core/src/main/java/com/github/neuralnetworks/calculation/RBMLayerCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,15 @@ public void gibbsSampling(RBM rbm, /*Matrix posPhaseVisible, Matrix posPhaseHidd
for (int i = 1; i <= samplingCount; i++) {
connections.clear();
connections.add(rbm.getMainConnections());
if (rbm.getHiddenBiasConnections() != null) {
connections.add(rbm.getHiddenBiasConnections());
if (rbm.getVisibleBiasConnections() != null) {
connections.add(rbm.getVisibleBiasConnections());
}
negPhaseHiddenToVisibleCC.calculate(connections, negPhaseVP, rbm.getVisibleLayer());

connections.clear();
connections.add(rbm.getMainConnections());
if (rbm.getVisibleBiasConnections() != null) {
connections.add(rbm.getVisibleBiasConnections());
if (rbm.getHiddenBiasConnections() != null) {
connections.add(rbm.getHiddenBiasConnections());
}
negPhaseVisibleToHiddenCC.calculate(connections, negPhaseVP, rbm.getHiddenLayer());
}
Expand Down
4 changes: 3 additions & 1 deletion nn-core/src/main/java/com/github/neuralnetworks/calculation/memory/ValuesProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,10 @@ public <T extends Tensor> T get(Object key, int... dimensions) {
if (dimensions == null || dimensions.length == 0) {
if (values.get(key) != null && values.get(key).size() == 1) {
return (T) values.get(key).iterator().next();
} else {
} else if (values.get(key) != null && values.get(key).size() > 1) {
throw new IllegalArgumentException("No dimensions provided");
} else {
return null;
}
}

Expand Down
21 changes: 17 additions & 4 deletions ...ore/src/main/java/com/github/neuralnetworks/calculation/neuronfunctions/AparapiNoise.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,42 @@
import com.github.neuralnetworks.training.random.XORShiftKernel;
import com.github.neuralnetworks.util.Tensor;

/**
* Random noise
*/
public class AparapiNoise extends XORShiftKernel implements TensorFunction {

private static final long serialVersionUID = 1L;

private final float corruptionLevel;
private final int startIndex;
private float[] inputOutput;

public AparapiNoise(int maximumRange, float corruptionLevel) {
public AparapiNoise(Tensor inputOutput, int maximumRange, float corruptionLevel) {
super(maximumRange);
this.inputOutput = inputOutput.getElements();
this.startIndex = inputOutput.getStartIndex();
this.corruptionLevel = corruptionLevel;
}

@Override
public void value(Tensor inputOutput) {
this.inputOutput = inputOutput.getElements();
execute(this.inputOutput.length);
if (inputOutput.getElements() != this.inputOutput) {
if (startIndex != inputOutput.getStartIndex()) {
throw new IllegalArgumentException("Different tensors");
}

this.inputOutput = inputOutput.getElements();
}

execute(inputOutput.getSize());
}

@Override
public void run() {
int id = getGlobalId();
if (random01() < corruptionLevel) {
inputOutput[id] = 0;
inputOutput[startIndex + id] = 0;
}
}
}
15 changes: 15 additions & 0 deletions nn-core/src/main/java/com/github/neuralnetworks/training/DNNLayerTrainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,21 @@ public Map<NeuralNetwork, OneStepTrainer<?>> getTrainers() {
return properties.getParameter(Constants.LAYER_TRAINERS);
}

@Override
public Integer getTrainingBatchSize() {
return getTrainers().values().iterator().next().getTrainingBatchSize();
}

@Override
public Integer getTestBatchSize() {
return getTrainers().values().iterator().next().getTestBatchSize();
}

@Override
public Integer getEpochs() {
return getTrainers().values().iterator().next().getEpochs();
}

/**
* Triggered when a nested neural network has finished training
*/
Expand Down
4 changes: 2 additions & 2 deletions nn-core/src/main/java/com/github/neuralnetworks/training/Trainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ public void test() {

OutputError oe = getOutputError();
if (oe != null) {
getOutputError().reset();
results.add(oe, results.get(n.getInputLayer()).getDimensions());
oe.reset();
results.add(oe, results.get(n.getOutputLayer()).getDimensions());
}

TrainingInputData input = new TrainingInputDataImpl(results.get(n.getInputLayer()), results.get(oe));
Expand Down
12 changes: 8 additions & 4 deletions nn-core/src/main/java/com/github/neuralnetworks/training/TrainerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,13 @@ private static BackPropagationLayerCalculatorImpl bplc(NeuralNetworkImpl nn, Pro
}

public static BackPropagationAutoencoder backPropagationAutoencoder(NeuralNetworkImpl nn, TrainingInputProvider trainingSet, TrainingInputProvider testingSet, OutputError error, NNRandomInitializer rand, float learningRate, float momentum, float l1weightDecay, float l2weightDecay, float inputCorruptionRate, int trainingBatchSize, int testBatchSize, int epochs) {
BackPropagationAutoencoder t = new BackPropagationAutoencoder(backpropProperties(nn, trainingSet, testingSet, error, rand, learningRate, momentum, l1weightDecay, l2weightDecay, trainingBatchSize, testBatchSize, epochs));
Properties p = backpropProperties(nn, trainingSet, testingSet, error, rand, learningRate, momentum, l1weightDecay, l2weightDecay, trainingBatchSize, testBatchSize, epochs);
p.setParameter(Constants.CORRUPTION_LEVEL, inputCorruptionRate);

BackPropagationAutoencoder t = new BackPropagationAutoencoder(p);

BackPropagationLayerCalculatorImpl bplc = bplc(nn, p);

BackPropagationLayerCalculatorImpl bplc = bplc(nn, t.getProperties());
t.getProperties().setParameter(Constants.BACKPROPAGATION, bplc);

return t;
Expand Down Expand Up @@ -225,7 +229,7 @@ public static AparapiCDTrainer cdSigmoidTrainer(RBM rbm, TrainingInputProvider t
return new AparapiCDTrainer(rbmProperties(rbm, lc, trainingSet, testingSet, error, rand, learningRate, momentum, l1weightDecay, l2weightDecay, gibbsSampling, trainingBatchSize, epochs, isPersistentCD));
}

protected static Properties rbmProperties(RBM rbm, RBMLayerCalculator lc, TrainingInputProvider trainingSet, TrainingInputProvider testingSet, OutputError error, NNRandomInitializer rand, float learningRate, float momentum, float l1weightDecay, float l2weightDecay, int gibbsSampling, int trainingBatchSize, int epochs, boolean resetRBM) {
protected static Properties rbmProperties(RBM rbm, RBMLayerCalculator lc, TrainingInputProvider trainingSet, TrainingInputProvider testingSet, OutputError error, NNRandomInitializer rand, float learningRate, float momentum, float l1weightDecay, float l2weightDecay, int gibbsSampling, int trainingBatchSize, int epochs, boolean isPersistentCD) {
Properties p = new Properties();
p.setParameter(Constants.NEURAL_NETWORK, rbm);
p.setParameter(Constants.TRAINING_INPUT_PROVIDER, trainingSet);
Expand All @@ -237,7 +241,7 @@ protected static Properties rbmProperties(RBM rbm, RBMLayerCalculator lc, Traini
p.setParameter(Constants.GIBBS_SAMPLING_COUNT, gibbsSampling);
p.setParameter(Constants.OUTPUT_ERROR, error);
p.setParameter(Constants.RANDOM_INITIALIZER, rand);
p.setParameter(Constants.RESET_RBM, resetRBM);
p.setParameter(Constants.PERSISTENT_CD, isPersistentCD);
p.setParameter(Constants.LAYER_CALCULATOR, lc);
p.setParameter(Constants.TRAINING_BATCH_SIZE, trainingBatchSize);
p.setParameter(Constants.EPOCHS, epochs);
Expand Down
4 changes: 2 additions & 2 deletions nn-core/src/main/java/com/github/neuralnetworks/training/TrainingInputProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public default void populateNext(TrainingInputData ti) {
if (ti.getInput() != null) {
int[] inputDims = ti.getInput().getDimensions();
int[][] limits = new int[2][inputDims.length];
IntStream.range(0, inputDims.length - 1).forEach(i -> limits[1][i] = inputDims[i]);
IntStream.range(0, inputDims.length - 1).forEach(i -> limits[1][i] = inputDims[i] - 1);
IntStream.range(0, inputDims[inputDims.length - 1]).forEach(i -> {
limits[0][inputDims.length - 1] = limits[1][inputDims.length - 1] = i;
TensorIterator it = ti.getInput().iterator(limits);
Expand All @@ -44,7 +44,7 @@ public default void populateNext(TrainingInputData ti) {
if (ti.getTarget() != null) {
int[] targetDims = ti.getTarget().getDimensions();
int[][] limits = new int[2][targetDims.length];
IntStream.range(0, targetDims.length - 1).forEach(i -> limits[1][i] = targetDims[i]);
IntStream.range(0, targetDims.length - 1).forEach(i -> limits[1][i] = targetDims[i] - 1);
IntStream.range(0, targetDims[targetDims.length - 1]).forEach(i -> {
limits[0][targetDims.length - 1] = limits[1][targetDims.length - 1] = i;
TensorIterator it = ti.getTarget().iterator(limits);
Expand Down
65 changes: 38 additions & 27 deletions ...n/java/com/github/neuralnetworks/training/backpropagation/BackPropagationAutoencoder.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package com.github.neuralnetworks.training.backpropagation;

import java.util.Arrays;

import com.github.neuralnetworks.architecture.types.Autoencoder;
import com.github.neuralnetworks.training.TrainingInputData;
import com.github.neuralnetworks.calculation.neuronfunctions.AparapiNoise;
import com.github.neuralnetworks.training.TrainingInputProvider;
import com.github.neuralnetworks.training.TrainingInputProviderImpl;
import com.github.neuralnetworks.util.Constants;
import com.github.neuralnetworks.util.Properties;
import com.github.neuralnetworks.util.Tensor;
import com.github.neuralnetworks.util.TensorFactory;
Expand All @@ -16,49 +17,59 @@ public class BackPropagationAutoencoder extends BackPropagationTrainer<Autoencod

private static final long serialVersionUID = 1L;

private AutoencoderTrainingInputData autoencoderTrainingInputData;

public BackPropagationAutoencoder(Properties properties) {
super(properties);
setTrainingInputProvider(new AutoencoderTrainingInputrovider(getTrainingInputProvider(), getCorruptionRate()));
}

@Override
protected TrainingInputData getInput() {
if (autoencoderTrainingInputData == null) {
autoencoderTrainingInputData = new AutoencoderTrainingInputData();
autoencoderTrainingInputData.setInputOutput(activations.get(getNeuralNetwork().getInputLayer()));
}
public Float getCorruptionRate() {
return getProperties().getParameter(Constants.CORRUPTION_LEVEL);
}

return autoencoderTrainingInputData;
public void setCorruptionRate(Float corruptionRate) {
getProperties().setParameter(Constants.CORRUPTION_LEVEL, corruptionRate);
}

/**
* Input and target are the same
*/
private static class AutoencoderTrainingInputData implements TrainingInputData {
private static class AutoencoderTrainingInputrovider extends TrainingInputProviderImpl {

private static final long serialVersionUID = 1L;

private Tensor input;
private Tensor target;
private TrainingInputProvider base;
private Float corruptionRate;
private AparapiNoise noise;
private Tensor noiseTensor;

public AutoencoderTrainingInputrovider(TrainingInputProvider base, Float corruptionRate) {
super();
this.base = base;
this.corruptionRate = corruptionRate;
}

@Override
public Tensor getInput() {
return input;
public int getInputSize() {
return base.getInputSize();
}

@Override
public Tensor getTarget() {
return target;
public float[] getNextInput() {
return base.getNextInput();
}

public void setInputOutput(Tensor inputOutput) {
this.input = inputOutput;
if (target == null || !Arrays.equals(target.getDimensions(), input.getDimensions())) {
target = TensorFactory.tensor(input.getDimensions());
@Override
public float[] getNextTarget() {
float[] result = base.getNextInput();
if (corruptionRate != null && corruptionRate > 0) {
if (noise == null) {
noiseTensor = TensorFactory.tensor(base.getNextInput().length);
noise = new AparapiNoise(noiseTensor, base.getNextInput().length, corruptionRate);
}

System.arraycopy(result, 0, noiseTensor.getElements(), 0, result.length);
noise.value(noiseTensor);
result = noiseTensor.getElements();
}

System.arraycopy(input.getElements(), 0, target.getElements(), 0, target.getElements().length);
return result;
}
}
}
35 changes: 3 additions & 32 deletions .../main/java/com/github/neuralnetworks/training/backpropagation/BackPropagationTrainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public class BackPropagationTrainer<N extends NeuralNetwork> extends OneStepTrai
public BackPropagationTrainer(Properties properties) {
super(properties);
activations = TensorFactory.tensorProvider(getNeuralNetwork(), getTrainingBatchSize(), Environment.getInstance().getUseSharedMemory());
activations.add(getOutputError(), activations.get(getNeuralNetwork().getOutputLayer()).getDimensions());
activations.add(getProperties().getParameter(Constants.OUTPUT_ERROR_DERIVATIVE), activations.get(getNeuralNetwork().getOutputLayer()).getDimensions());
backpropagation = TensorFactory.tensorProvider(getNeuralNetwork(), getTrainingBatchSize(), Environment.getInstance().getUseSharedMemory());
}

Expand All @@ -51,7 +51,7 @@ protected void learnInput(int batch) {

// backward
OutputErrorDerivative d = getProperties().getParameter(Constants.OUTPUT_ERROR_DERIVATIVE);
d.getOutputErrorDerivative(activations.get(nn.getOutputLayer()), activations.get(getOutputError()), backpropagation.get(nn.getOutputLayer()));
d.getOutputErrorDerivative(activations.get(nn.getOutputLayer()), activations.get(d), backpropagation.get(nn.getOutputLayer()));
calculatedLayers.clear();
calculatedLayers.add(nn.getOutputLayer());
BackPropagationLayerCalculator blc = getBPLayerCalculator();
Expand All @@ -61,41 +61,12 @@ protected void learnInput(int batch) {
@Override
protected TrainingInputData getInput() {
if (input == null) {
input = new TrainingInputDataImpl(activations.get(getNeuralNetwork().getOutputLayer()), activations.get(getOutputError()));
input = new TrainingInputDataImpl(activations.get(getNeuralNetwork().getInputLayer()), activations.get(getProperties().getParameter(Constants.OUTPUT_ERROR_DERIVATIVE)));
}

return input;
}

// /* (non-Javadoc)
// * @see com.github.neuralnetworks.training.OneStepTrainer#learnInput(com.github.neuralnetworks.training.TrainingInputData)
// * The training example is propagated forward through the network (via the LayerCalculator lc) and the results are stored.
// * After that the error is backpropagated (via BackPropagationLayerCalculator blc).
// */
// @Override
// protected void learnInput(TrainingInputData data, int batch) {
// propagateForward(data.getInput());
// propagateBackward(data.getTarget());
// }
//
// public void propagateForward(Tensor input) {
// NeuralNetwork nn = getNeuralNetwork();
// Set<Layer> calculatedLayers = new UniqueList<Layer>();
// calculatedLayers.add(nn.getInputLayer());
// nn.getLayerCalculator().calculate(nn, nn.getOutputLayer(), calculatedLayers, activations);
// }
//
// public void propagateBackward(Tensor target) {
// NeuralNetwork nn = getNeuralNetwork();
//
// OutputErrorDerivative d = getProperties().getParameter(Constants.OUTPUT_ERROR_DERIVATIVE);
// d.getOutputErrorDerivative(activations.get(nn.getOutputLayer()), target, backpropagation.get(nn.getOutputLayer()));
// Set<Layer> calculatedLayers = new UniqueList<Layer>();
// calculatedLayers.add(nn.getOutputLayer());
// BackPropagationLayerCalculator blc = getBPLayerCalculator();
// blc.backpropagate(nn, calculatedLayers, activations, backpropagation);
// }

public BackPropagationLayerCalculator getBPLayerCalculator() {
return getProperties().getParameter(Constants.BACKPROPAGATION);
}
Expand Down
6 changes: 3 additions & 3 deletions nn-core/src/main/java/com/github/neuralnetworks/training/rbm/AparapiCDTrainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public AparapiCDTrainer(Properties properties) {
* before each update the kernel update parameters are refreshed
*/
@Override
protected void updateWeights(/*Matrix posPhaseVisible, Matrix posPhaseHidden, Matrix negPhaseVisible, Matrix negPhaseHidden*/) {
protected void updateWeights() {
RBM rbm = getNeuralNetwork();

RBMLayerCalculator lc = getLayerCalculator();
Expand Down Expand Up @@ -71,15 +71,15 @@ protected void updateWeights(/*Matrix posPhaseVisible, Matrix posPhaseHidden, Ma
protected float getLearningRate() {
return properties.getParameter(Constants.LEARNING_RATE);
}

protected float getMomentum() {
return (float) (properties.getParameter(Constants.MOMENTUM) != null ? properties.getParameter(Constants.MOMENTUM) : 0f);
}

protected float getl1weightDecay() {
return (float) (properties.getParameter(Constants.L1_WEIGHT_DECAY) != null ? properties.getParameter(Constants.L1_WEIGHT_DECAY) : 0f);
}

protected float getl2weightDecay() {
return (float) (properties.getParameter(Constants.L2_WEIGHT_DECAY) != null ? properties.getParameter(Constants.L2_WEIGHT_DECAY) : 0f);
}
Expand Down
Loading

0 comments on commit 4fe1fbf

Please sign in to comment.