170 lines
4.7 KiB
Python
170 lines
4.7 KiB
Python
import numpy as np
|
|
#from mnist import load as mnist_load
|
|
|
|
# NOTE - Based on:
|
|
# http://iamtrask.github.io/2015/07/12/basic-python-network/
|
|
# http://iamtrask.github.io/2015/07/27/python-network-part2/
|
|
# http://iamtrask.github.io/2015/07/28/dropout/
|
|
|
|
# Puts input and output into correct form for network
|
|
def wrap_x(x):
|
|
return np.array(x)
|
|
|
|
def wrap_y(y):
|
|
return np.array([y]).T
|
|
|
|
def wrap_xy(x, y):
|
|
return wrap_x(x), wrap_y(y)
|
|
|
|
def activate(x, mode="sigmoid", deriv=False):
|
|
valid_modes = ["sigmoid", "relu"]
|
|
assert mode in valid_modes, f"Mode '{mode}' is invalid"
|
|
|
|
if mode == "sigmoid":
|
|
return x * (1 - x) if deriv else 1 / (1 + np.exp(-x))
|
|
|
|
elif mode == "relu":
|
|
return (1 if x > 0 else 0) if deriv else np.maximum(0, x)
|
|
|
|
def noop(n):
|
|
return n
|
|
|
|
def mean_zero(n):
|
|
return 2 * n - 1
|
|
|
|
def generate_weights(*layer_sizes, adjust):
|
|
layer_count = len(layer_sizes)
|
|
assert layer_count >= 3, f"Not enough layers given: {layer_count}"
|
|
|
|
weights = list()
|
|
for l in range(1, layer_count):
|
|
layer_size = layer_sizes[l]
|
|
prev_layer_size = layer_sizes[l-1]
|
|
weight = adjust(np.random.random((prev_layer_size, layer_size)))
|
|
weights.append(weight)
|
|
|
|
return weights
|
|
|
|
def apply_dropout(layer, prev_layer, percent):
|
|
layer_size = len(layer[0])
|
|
prev_layer_size = len(prev_layer[0])
|
|
ones = [np.ones((layer_size, layer_size))]
|
|
keep = 1 - percent
|
|
cloak = np.random.binomial(ones, keep)[0]
|
|
drop = cloak * (1.0 / keep)
|
|
return layer * drop
|
|
|
|
def forward_feed(x, weights, modes, dropout):
|
|
layers = [x]
|
|
layer_count = len(weights)
|
|
for l in range(layer_count):
|
|
prev_layer = layers[l]
|
|
weight = weights[l]
|
|
mode = modes[l]
|
|
layer = activate(np.dot(prev_layer, weight), mode=mode)
|
|
|
|
percent = dropout[l]
|
|
if percent > 0:
|
|
apply_dropout(layer, prev_layer, percent)
|
|
|
|
layers.append(layer)
|
|
|
|
return layers
|
|
|
|
def backpropagate(i, y, weights, layers, modes):
|
|
output_layer = layers[-1]
|
|
target_error = y - output_layer
|
|
|
|
if i % 100 == 0:
|
|
epoch = i // 100
|
|
display_error = str(np.mean(np.abs(target_error)))
|
|
print(f"Epoch {epoch}: {display_error}")
|
|
|
|
output_layer_delta = target_error * activate(output_layer, deriv=True)
|
|
deltas = [output_layer_delta]
|
|
layer_count = len(layers)
|
|
for l in range(2, layer_count):
|
|
ll = l - 1
|
|
layer = layers[-l]
|
|
weight = weights[-ll]
|
|
mode = modes[-ll]
|
|
prev_delta = deltas[-1]
|
|
layer_error = prev_delta.dot(weight.T)
|
|
layer_delta = layer_error * activate(layer, mode=mode, deriv=True)
|
|
deltas.append(layer_delta)
|
|
|
|
return deltas
|
|
|
|
def update_weights(deltas, weights, layers, learning_rate):
|
|
for d, delta in enumerate(reversed(deltas)):
|
|
layer = layers[d]
|
|
offset = learning_rate * layer.T.dot(delta)
|
|
prev_weight = weights[d]
|
|
weights[d] = prev_weight + offset
|
|
|
|
return weights
|
|
|
|
|
|
|
|
def init_parameters(weights, modes, dropout=None):
|
|
if modes is None or dropout is None:
|
|
set_modes = False
|
|
if modes is None:
|
|
modes = list()
|
|
set_modes = True
|
|
|
|
set_dropout = False
|
|
if dropout is None:
|
|
dropout = list()
|
|
set_dropout = True
|
|
|
|
for w in range(len(weights)):
|
|
if set_modes:
|
|
modes.append("sigmoid")
|
|
|
|
if set_dropout:
|
|
dropout.append(0)
|
|
|
|
return modes, dropout
|
|
|
|
def train(x, y, layer_sizes, iterations, learning_rate, dropout=None,
|
|
modes=None, adjust=noop, seed=1):
|
|
np.random.seed(seed)
|
|
weights = generate_weights(*layer_sizes, adjust=adjust)
|
|
modes, dropout = init_parameters(weights, modes, dropout)
|
|
|
|
for i in range(iterations):
|
|
layers = forward_feed(x, weights, modes, dropout)
|
|
deltas = backpropagate(i, y, weights, layers, modes)
|
|
weights = update_weights(deltas, weights, layers, learning_rate)
|
|
|
|
return weights
|
|
|
|
def run(test, weights, modes=None):
|
|
modes, dropout = init_parameters(weights, modes)
|
|
layers = forward_feed(test, weights, modes, dropout)
|
|
return layers[-1]
|
|
|
|
if __name__ == "__main__":
|
|
x = [[0,0,1],[0,1,1],[1,0,1],[1,1,1]]
|
|
y = [0,1,1,0]
|
|
x, y = wrap_xy(x, y)
|
|
layer_sizes = [3,4,1]
|
|
iterations = 60000
|
|
learning_rate = 0.5
|
|
dropout = [0.2,0,0]
|
|
adjust = mean_zero
|
|
args = x, y, layer_sizes, iterations, learning_rate, dropout
|
|
|
|
#x_train, y_train, x_test, y_test = mnist_load()
|
|
#x, y = x_train, wrap_y(y_train)
|
|
#layer_sizes = [784,32,10]
|
|
#iterations = 1000
|
|
#learning_rate = 1e-4
|
|
#dropout = [0.2,0,0]
|
|
#adjust = mean_zero
|
|
#modes = ["relu", "sigmoid"]
|
|
#args = x, y, layer_sizes, iterations, learning_rate, dropout, modes
|
|
|
|
weights = train(*args, adjust=adjust)
|