%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt
import time
from IPython import display
from mlp import MLP

Auxiliar functions

To visualize and store the results

def update_plot(X,Y,grid,prediction,plt):
    """
    Updates the image of the actual plot with the original target
    and the prediction of the model
    """
    plt.clf()
    plt.scatter(X,Y,color='red')
    plt.plot(grid,prediction)
    display.clear_output(wait=True)
    display.display(plt.gcf())

def export_plot(plt, filename, format='pdf', date=True):
    """
    Exports the figure into the project folder
    suported formats : eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff.
    """
    if date:
        formated_filename = "{0}_{1}.{2}".format(time.strftime("%Y%m%d_%H%M%S"), filename, format)
    else:
        formated_filename = "{0}.{1}".format(filename, format)
    plt.savefig(formated_filename, format=format)

Pattern to approximate

N = 20 # datapoints

X = np.random.uniform(-1,1,N)*np.pi
grid = np.linspace(-1,1,150)*(np.pi + 0.5)

PATTERN = 'sin' # 'linear', 'abs', 'squared', 'sin', 'cos', 'sin_cos'
BIAS = 1
NOISE = 0.5

if PATTERN == 'sin':
    original = np.sin(grid)
    pattern = np.sin(X)
elif PATTERN == 'linear':
    original = grid
    pattern = X
elif PATTERN == 'abs':
    original = np.abs(grid)
    pattern = np.abs(X)
elif PATTERN == 'squared':
    original = grid**2
    pattern = X**2
elif PATTERN == 'sin':
    original = np.sin(grid)
    pattern = np.sin(X)
elif PATTERN == 'cos':
    original = np.cos(grid)
    pattern = np.cos(X)
elif PATTERN == 'sin_cos':
    original = np.sin(np.cos(grid)*np.pi)
    pattern = np.sin(np.cos(X)*np.pi)
else:
    original = grid
    pattern = X

original += BIAS
pattern += BIAS
if NOISE > 0:
    Y = pattern + np.random.normal(size=N, scale=NOISE)
else:
    Y = pattern

plt.plot(grid, original, color='red', label='original')
plt.scatter(X, Y, color='red', label='samples')
plt.ylabel('target')
plt.xlabel('x')
plt.legend()
export_plot(plt,filename="{0}_generalization_target_pattern".format(PATTERN), date=False)

Especification of the MLP

it shows the initial weights

n_input = 1
N_HIDDEN = np.multiply(range(2,6),2)
n_output = 1
activation = 'tanh' # 'tanh', 'linear'

# Learning rate parameters
lr_policy = 'fixed' # 'step', 'rand', 'fixed'
lr = 0.1
gamma = 0.1
low = 0.0001
high = 0.1
stepsize = 100*N

EPOCHS = 300
REPETITIONS = 5

train_error = np.zeros((len(N_HIDDEN), REPETITIONS, EPOCHS))
test_error = np.zeros((len(N_HIDDEN), REPETITIONS, EPOCHS))

for i, n_hidden in enumerate(N_HIDDEN):
    for r in range(REPETITIONS):
        mlp = MLP(n_input, n_hidden, n_output, activation=activation, learning_rate=lr, 
                  lr_policy=lr_policy, stepsize=stepsize, gamma=gamma, low=low, high=high)
        plt.clf()
        for epoch in range(EPOCHS):
            prediction = mlp.test(grid)
            plt.plot(grid,original, 'r-.', color='red', label='original')
            update_plot(X,Y,grid,prediction,plt)
            train_error[i, r, epoch] = mlp.mean_error(X,Y)
            
            test_error[i, r, epoch] = mlp.mean_error(grid,original)

            mlp.train(X,Y)
    

Training error

colors = plt.cm.rainbow(np.linspace(0, 1, len(N_HIDDEN)))

for i, error in enumerate(train_error.mean(axis=1)):
    plt.plot(error[1:], color=colors[i], label="{0}".format(N_HIDDEN[i]))
    
plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.2),
          fancybox=True, shadow=True, ncol=5)

plt.ylabel('mean_squared_error')
plt.xlabel('epoch')
export_plot(plt,filename="{0}_generalization_train_error".format(PATTERN), date=False)

Test error

for i, error in enumerate(test_error.mean(axis=1)):
    plt.plot(error[1:], color=colors[i], label="{0}".format(N_HIDDEN[i]))
    
plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.2),
          fancybox=True, shadow=True, ncol=5)
plt.ylabel('mean_squared_error')
plt.xlabel('epoch')
export_plot(plt,filename="{0}_generalization_test_error".format(PATTERN), date=False)

for i, (train, test) in enumerate(zip(train_error.mean(axis=1), test_error.mean(axis=1))):
    plt.plot(train[1:], '-.', color=colors[i], linewidth=1, label="{0}_train".format(N_HIDDEN[i]))
    plt.plot(test[1:], color=colors[i], linewidth=2, label="{0}_test".format(N_HIDDEN[i]))
    
plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.3),
          fancybox=True, shadow=True, ncol=4)
plt.ylabel('mean_squared_error')
plt.xlabel('epoch')
export_plot(plt,filename="{0}_generalization_test_error".format(PATTERN), date=False)