Added function to run network after training it

code/charms/charms/faerie/wisp2.py

@@ -1,10 +1,20 @@
 import numpy as np
+#from mnist import load as mnist_load
 
-# NOTE - Based on http://iamtrask.github.io/2015/07/27/python-network-part2/ 
+# NOTE - Based on:
+# http://iamtrask.github.io/2015/07/12/basic-python-network/
+# http://iamtrask.github.io/2015/07/27/python-network-part2/ 
+# http://iamtrask.github.io/2015/07/28/dropout/
 
 # Puts input and output into correct form for network
-def wrap_io(x, y):
-    return np.array(x), np.array([y]).T
+def wrap_x(x):
+    return np.array(x)
+
+def wrap_y(y):
+    return np.array([y]).T
+
+def wrap_xy(x, y):
+    return wrap_x(x), wrap_y(y)
 
 def activate(x, mode="sigmoid", deriv=False):
     valid_modes = ["sigmoid", "relu"]
@@ -35,29 +45,33 @@ def generate_weights(*layer_sizes, adjust):
 
     return weights
 
-def generate_layers(x, weights):
-    layers = [x]
-    layer_count = len(weights)
-    for l in range(layer_count):
-        prev_layer = layers[l]
-        weight = weights[i]
-        layer = activate(np.dot(prev_layer, weight))
-        layers.append(layer)
+def apply_dropout(layer, prev_layer, percent):
+    layer_size = len(layer[0])
+    prev_layer_size = len(prev_layer[0])
+    ones = [np.ones((layer_size, layer_size))]
+    keep = 1 - percent
+    cloak = np.random.binomial(ones, keep)[0]
+    drop = cloak * (1.0 / keep)
+    return layer * drop
 
-    return layers
-
-def forward_feed(x, weights):
+def forward_feed(x, weights, modes, dropout):
     layers = [x]
     layer_count = len(weights)
     for l in range(layer_count):
         prev_layer = layers[l]
         weight = weights[l]
-        layer = activate(np.dot(prev_layer, weight))
+        mode = modes[l]
+        layer = activate(np.dot(prev_layer, weight), mode=mode)
+
+        percent = dropout[l]
+        if percent > 0:
+            apply_dropout(layer, prev_layer, percent)
+
         layers.append(layer)
 
     return layers
 
-def backpropagate(i, y, weights, layers):
+def backpropagate(i, y, weights, layers, modes):
     output_layer = layers[-1]
     target_error = y - output_layer
 
@@ -69,13 +83,14 @@ def backpropagate(i, y, weights, layers):
     output_layer_delta = target_error * activate(output_layer, deriv=True)
     deltas = [output_layer_delta]
     layer_count = len(layers)
-    for ll in range(2, layer_count):
-        w = ll - 1
-        layer = layers[-ll]
-        weight = weights[-w]
+    for l in range(2, layer_count):
+        ll = l - 1
+        layer = layers[-l]
+        weight = weights[-ll]
+        mode = modes[-ll]
         prev_delta = deltas[-1]
         layer_error = prev_delta.dot(weight.T)
-        layer_delta = layer_error * activate(layer, deriv=True)
+        layer_delta = layer_error * activate(layer, mode=mode, deriv=True)
         deltas.append(layer_delta)
 
     return deltas
@@ -89,22 +104,66 @@ def update_weights(deltas, weights, layers, learning_rate):
 
     return weights
 
-def train(x, y, layer_sizes, learning_rate, iterations, adjust=noop, seed=1):
+
+
+def init_parameters(weights, modes, dropout=None):
+    if modes is None or dropout is None:
+        set_modes = False
+        if modes is None:
+            modes = list()
+            set_modes = True
+
+        set_dropout = False
+        if dropout is None:
+            dropout = list()
+            set_dropout = True
+
+        for w in range(len(weights)):
+            if set_modes:
+                modes.append("sigmoid")
+
+            if set_dropout:
+                dropout.append(0)
+
+    return modes, dropout
+
+def train(x, y, layer_sizes, iterations, learning_rate, dropout=None,
+    modes=None, adjust=noop, seed=1):
     np.random.seed(seed)
-    x, y = wrap_io(x, y)
     weights = generate_weights(*layer_sizes, adjust=adjust)
+    modes, dropout = init_parameters(weights, modes, dropout)
+
     for i in range(iterations):
-        layers = forward_feed(x, weights)
-        deltas = backpropagate(i, y, weights, layers)
+        layers = forward_feed(x, weights, modes, dropout)
+        deltas = backpropagate(i, y, weights, layers, modes)
         weights = update_weights(deltas, weights, layers, learning_rate)
 
     return weights
 
+def run(test, weights, modes=None):
+    modes, dropout = init_parameters(weights, modes)
+    layers = forward_feed(test, weights, modes, dropout)
+    return layers[-1]
+
 if __name__ == "__main__":
     x = [[0,0,1],[0,1,1],[1,0,1],[1,1,1]]
     y = [0,1,1,0]
+    x, y = wrap_xy(x, y)
     layer_sizes = [3,4,1]
-    learning_rate = 0.5
     iterations = 60000
+    learning_rate = 0.5
+    dropout = [0.2,0,0]
     adjust = mean_zero
-    weights = train(x, y, layer_sizes, learning_rate, iterations, adjust)
+    args = x, y, layer_sizes, iterations, learning_rate, dropout
+
+    #x_train, y_train, x_test, y_test = mnist_load()
+    #x, y = x_train, wrap_y(y_train)
+    #layer_sizes = [784,32,10]
+    #iterations = 1000
+    #learning_rate = 1e-4
+    #dropout = [0.2,0,0]
+    #adjust = mean_zero
+    #modes = ["relu", "sigmoid"]
+    #args = x, y, layer_sizes, iterations, learning_rate, dropout, modes
+
+    weights = train(*args, adjust=adjust)