MLP-Non-Linear-Function-Approximation Using Circle-Dataset

Import Necessary Package

from DL_utilities import *
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.datasets import make_moons, make_circles 
%matplotlib inline

Data Preparation

def generate_data(samples, shape_type='circles', noise=0.05):
    if shape_type == 'moons':
        X, Y = make_moons(n_samples=samples, noise=noise)
    elif shape_type == 'circles':
        X, Y = make_circles(n_samples=samples, noise=noise)
    data = pd.DataFrame(dict(x=X[:,0], y=X[:,1], label=Y))
    return data

def plot_generated_data(data):
    plt.style.use('seaborn-whitegrid')
    ax = data.plot.scatter(x='x', y='y', figsize=(13,10), color=data['label'], 
                 cmap=matplotlib.colors.ListedColormap(['skyblue', 'salmon']), grid=True);
    return ax

def plot_generated_data(data):
    plt.style.use('seaborn-whitegrid')
    ax = data.plot.scatter(x='x', y='y', figsize=(13,10), color=data['label'], 
                 cmap=matplotlib.colors.ListedColormap(['skyblue', 'salmon']), grid=True);
    return ax

data = generate_data(samples=10000, shape_type='circles', noise=0.04)
plot_generated_data(data);

output:

Notice that this data is very tough to classify perfectly, as many of the data points are intertwined( i.e blue and red points are too close to each other)

print(data)

output:

X_train = data[['x', 'y']].values
Y_train = data['label'].T.values

print("X_train shape \n", X_train.shape)
print("Y_train shape \n", Y_train.shape)

output:

X_train shape (10000, 2) Y_train shape (10000,)

Y_train = Y_train.reshape(Y_train.shape[0], 1)
X_train = X_train.T
Y_train = Y_train.T

print("X_train new shape \n", X_train.shape)
print("Y_train new shape \n", Y_train.shape)

output:

X_train new shape (2, 10000) Y_train new shape (1, 10000)

Neural Network Model and Training

np.random.seed(48)

learning_rate = 0.3
number_of_epoch=2000
m_batch = X_train.shape[1] #entire samples as 1 batch

# Neural Network Model and Initialize parameters
# 2 --> 10 --> 10 --> 1
layers_dims = [X_train.shape[0], 30, 30, 30, 1]
params = initialize_parameters(layers_dims)
hn_activation = "relu"

# intialize cost list
costs = []

# iterate over number_of_epoch
for epoch in range(number_of_epoch):
    # iterate over L-layers to get the final output and the cache
    AL, caches = L_model_forward(X_train, params, hn_activation)
    # iterate over L-layers backward to get gradients
    grads = L_model_backward(AL, Y_train, caches, hn_activation)
    params = update_parameters(params, grads, learning_rate)

    cost = BinaryCrossEntropy(AL, Y_train)
     
    if (epoch % 50) == 0 or epoch == number_of_epoch - 1:
        print(f"Cost at epoch #{epoch + 1} : {cost:.3f}")
        costs.append(cost) 

output:

Cost at epoch #1 : 0.690 Cost at epoch #51 : 0.639 Cost at epoch #101 : 0.438 Cost at epoch #151 : 0.422 Cost at epoch #201 : 0.342 Cost at epoch #251 : 0.089 Cost at epoch #301 : 0.044 Cost at epoch #351 : 0.032 Cost at epoch #401 : 0.027 Cost at epoch #451 : 0.024 Cost at epoch #501 : 0.023 Cost at epoch #551 : 0.021 Cost at epoch #601 : 0.020

...

...

Performance Evaluation

plot_learning_curve(costs, learning_rate, number_of_epoch, save=True)

output:

def predict(X, Y, params, threshold, hn_activation):
    probs, caches = L_model_forward(X, params, hn_activation)
    prediction = (probs >= threshold) * 1.0
    accuracy = np.mean(prediction == Y) * 100
    return probs, prediction, accuracy

classifcation_thresh = 0.5

probs, prediction, accuracy = predict(X_train, Y_train, params, classifcation_thresh, hn_activation)

print("The prediction of the first 5 examples: \n{}".format(prediction[:,:5]))
print("The  prbability of the first 5 examples:\n {}".format(np.round(probs[:,:5], decimals=3)) )
print("\nThe accuracy of the model is: {}%".format(accuracy))

output:

The prediction of the first 5 examples: [[1. 0. 0. 1. 0.]] The prbability of the first 5 examples: [[1. 0.061 0. 1. 0. ]] The accuracy of the model is: 99.42%

def predict_dec_prob(params, X, activation_fn, probabilities=False):
    probs, caches = L_model_forward(X, params, activation_fn)
    prediction = (probs >= 0.5)
    if probabilities:
        return probs
    else:
        return prediction

def plot_decision_boundary_range(model, X, Y, shaded=True):    

    plt.style.use("bmh")
    xmin, xmax = X[:,0].min()-0.2, X[:,0].max()+0.2
    ymin, ymax = X[:,1].min()-0.2, X[:,1].max()+0.2

    x_span = np.linspace(xmin, xmax, 1000)
    y_span = np.linspace(ymin, ymax, 1000)
    xx, yy = np.meshgrid(x_span, y_span)

    # Predict the function value for the whole grid
    prediction_data = np.c_[xx.ravel(), yy.ravel()].T

    z = model(prediction_data)
    z = z.reshape(xx.shape)
    
    plt.style.use('seaborn-whitegrid')
    fig, ax = plt.subplots(figsize=(12,12))
    
    if shaded:
        # highlight boundary    
        ax.scatter(X[:,0], X[:,1], c=Y, cmap="Paired", lw=0)
        ax.contourf(xx, yy, z, cmap="Paired", alpha=0.3)
    else:
        # draw a blue colored decision boundary    
        ax.scatter(X[:,0], X[:,1], c=Y, 
                   cmap=matplotlib.colors.ListedColormap(['skyblue', 'salmon']), lw=0)
        ax.contour(xx, yy, z, cmap='Blues')  

    return fig, ax

plot_decision_boundary_range(
    lambda x: predict_dec_prob(params,x,hn_activation,probabilities=False), 
    X_train.T, Y_train, shaded=False)

output:

plot_decision_boundary_range(
    lambda x: predict_dec_prob(params,x,hn_activation,probabilities=True), 
    X_train.T, Y_train, shaded=False)

output:

plot_decision_boundary_range(
    lambda x: predict_dec_prob(params,x,hn_activation,probabilities=True), 
    X_train.T, Y_train, shaded=True)

output: