diff --git a/code/charms/charms/faerie/wisp2.py b/code/charms/charms/faerie/wisp2.py index 488c6e4..4c1d1f6 100644 --- a/code/charms/charms/faerie/wisp2.py +++ b/code/charms/charms/faerie/wisp2.py @@ -1,10 +1,20 @@ import numpy as np +#from mnist import load as mnist_load -# NOTE - Based on http://iamtrask.github.io/2015/07/27/python-network-part2/ +# NOTE - Based on: +# http://iamtrask.github.io/2015/07/12/basic-python-network/ +# http://iamtrask.github.io/2015/07/27/python-network-part2/ +# http://iamtrask.github.io/2015/07/28/dropout/ # Puts input and output into correct form for network -def wrap_io(x, y): - return np.array(x), np.array([y]).T +def wrap_x(x): + return np.array(x) + +def wrap_y(y): + return np.array([y]).T + +def wrap_xy(x, y): + return wrap_x(x), wrap_y(y) def activate(x, mode="sigmoid", deriv=False): valid_modes = ["sigmoid", "relu"] @@ -35,29 +45,33 @@ def generate_weights(*layer_sizes, adjust): return weights -def generate_layers(x, weights): - layers = [x] - layer_count = len(weights) - for l in range(layer_count): - prev_layer = layers[l] - weight = weights[i] - layer = activate(np.dot(prev_layer, weight)) - layers.append(layer) +def apply_dropout(layer, prev_layer, percent): + layer_size = len(layer[0]) + prev_layer_size = len(prev_layer[0]) + ones = [np.ones((layer_size, layer_size))] + keep = 1 - percent + cloak = np.random.binomial(ones, keep)[0] + drop = cloak * (1.0 / keep) + return layer * drop - return layers - -def forward_feed(x, weights): +def forward_feed(x, weights, modes, dropout): layers = [x] layer_count = len(weights) for l in range(layer_count): prev_layer = layers[l] weight = weights[l] - layer = activate(np.dot(prev_layer, weight)) + mode = modes[l] + layer = activate(np.dot(prev_layer, weight), mode=mode) + + percent = dropout[l] + if percent > 0: + apply_dropout(layer, prev_layer, percent) + layers.append(layer) return layers -def backpropagate(i, y, weights, layers): +def backpropagate(i, y, weights, layers, modes): output_layer = layers[-1] target_error = y - output_layer @@ -69,13 +83,14 @@ def backpropagate(i, y, weights, layers): output_layer_delta = target_error * activate(output_layer, deriv=True) deltas = [output_layer_delta] layer_count = len(layers) - for ll in range(2, layer_count): - w = ll - 1 - layer = layers[-ll] - weight = weights[-w] + for l in range(2, layer_count): + ll = l - 1 + layer = layers[-l] + weight = weights[-ll] + mode = modes[-ll] prev_delta = deltas[-1] layer_error = prev_delta.dot(weight.T) - layer_delta = layer_error * activate(layer, deriv=True) + layer_delta = layer_error * activate(layer, mode=mode, deriv=True) deltas.append(layer_delta) return deltas @@ -89,22 +104,66 @@ def update_weights(deltas, weights, layers, learning_rate): return weights -def train(x, y, layer_sizes, learning_rate, iterations, adjust=noop, seed=1): + + +def init_parameters(weights, modes, dropout=None): + if modes is None or dropout is None: + set_modes = False + if modes is None: + modes = list() + set_modes = True + + set_dropout = False + if dropout is None: + dropout = list() + set_dropout = True + + for w in range(len(weights)): + if set_modes: + modes.append("sigmoid") + + if set_dropout: + dropout.append(0) + + return modes, dropout + +def train(x, y, layer_sizes, iterations, learning_rate, dropout=None, + modes=None, adjust=noop, seed=1): np.random.seed(seed) - x, y = wrap_io(x, y) weights = generate_weights(*layer_sizes, adjust=adjust) + modes, dropout = init_parameters(weights, modes, dropout) + for i in range(iterations): - layers = forward_feed(x, weights) - deltas = backpropagate(i, y, weights, layers) + layers = forward_feed(x, weights, modes, dropout) + deltas = backpropagate(i, y, weights, layers, modes) weights = update_weights(deltas, weights, layers, learning_rate) return weights +def run(test, weights, modes=None): + modes, dropout = init_parameters(weights, modes) + layers = forward_feed(test, weights, modes, dropout) + return layers[-1] + if __name__ == "__main__": x = [[0,0,1],[0,1,1],[1,0,1],[1,1,1]] y = [0,1,1,0] + x, y = wrap_xy(x, y) layer_sizes = [3,4,1] - learning_rate = 0.5 iterations = 60000 + learning_rate = 0.5 + dropout = [0.2,0,0] adjust = mean_zero - weights = train(x, y, layer_sizes, learning_rate, iterations, adjust) + args = x, y, layer_sizes, iterations, learning_rate, dropout + + #x_train, y_train, x_test, y_test = mnist_load() + #x, y = x_train, wrap_y(y_train) + #layer_sizes = [784,32,10] + #iterations = 1000 + #learning_rate = 1e-4 + #dropout = [0.2,0,0] + #adjust = mean_zero + #modes = ["relu", "sigmoid"] + #args = x, y, layer_sizes, iterations, learning_rate, dropout, modes + + weights = train(*args, adjust=adjust)