AIFunctions.py

import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm


def BinaryCrossEntropy(y_true, y_pred):
    y_pred = np.clip(y_pred, 1e-7, 1 - 1e-7)
    term_0 = (1 - y_true) * np.log(1 - y_pred + 1e-7)
    term_1 = y_true * np.log(y_pred + 1e-7)
    return -np.mean(term_0 + term_1, axis=0)


def sigmoid(z):
    return 1 / (1 + np.exp(-z))


def sigmoid_prime(z):
    return sigmoid(z) * (1 - sigmoid(z))


class NeuralNetwork:
    def __init__(self, layer_sizes):
        self.num_layers = len(layer_sizes)
        self.layer_sizes = layer_sizes
        self.biases = [np.random.randn(layer_size, 1) for layer_size in
                       layer_sizes[1:]]  # Bias is for all the layers except the input layer.
        self.weights = [np.random.randn(y, x) for x, y in zip(layer_sizes[:-1], layer_sizes[1:])]

    def forward(self, a):
        for w, b in tqdm(zip(self.weights, self.biases), desc="Forward Pass", dynamic_ncols=True):
            a = sigmoid(np.dot(a, w) + b)
        return a

    def backpropagation(self, x, y):
        nabla_b = [np.zeros(b.shape) for b in self.biases]
        nabla_w = [np.zeros(w.shape) for w in self.biases]
        # feedforward
        activation = x
        activations = [x]  # list to store all the activations, layer by layer
        zs = []  # list to store all the z vectors, layer by layer
        for b, w in zip(self.biases, self.weights):
            z = np.dot(w, activation) + b
            zs.append(z)
            activation = sigmoid(z)
            activations.append(activation)
        # backward pass
        delta = self.cost_derivative(activations[-1], y) * \
                sigmoid_prime(zs[-1])
        nabla_b[-1] = delta
        nabla_w[-1] = np.dot(delta, activations[-2].transpose())cd cd cd
        for l in range(2, self.num_layers):
            z = zs[-l]
            sp = sigmoid_prime(z)
            delta = np.dot(self.weights[-l + 1].transpose(), delta) * sp
            nabla_b[-l] = delta
            nabla_w[-l] = np.dot(delta, activations[-l - 1].transpose())
        return nabla_b, nabla_w

    def SGD(self, mini_batch, learning_rate):
        nabla_b = [np.zeros(b.shape) for b in self.biases]
        nabla_w = [np.zeros(w.shape) for w in self.weights]
        for x, y in mini_batch:
            delta_nabla_b, delta_nabla_w = self.backpropagation(x, y)
            nabla_b = [nb + dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
            nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
        self.weights = [
            w - (learning_rate / len(mini_batch)) * nw for w, nw in zip(self.weights, nabla_w)
        ]
        self.biases = [
            b - (learning_rate / len(mini_batch)) + nb for b, nb in zip(self.biases, nabla_b)
        ]

    def fit(self, train_data, val_data=None, epochs=10, mini_batch_size=None, learning_rate=0.1):
        n_train = train_data.shape[0]
        # Train the model
        for epoch in tqdm(range(epochs), desc="Training..."):
            np.random.shuffle(train_data)
            if not mini_batch_size:
                mini_batch_size = 0.1 * n_train
            mini_batches = [
                train_data[k:k + mini_batch_size] for k in range(0, n_train, mini_batch_size)
            ]
            for mini_batch in mini_batches:
                self.SGD(mini_batch, learning_rate)
            if (epoch % 5 == 0) and val_data:
                print(f"Epoch: {epoch}, Accuracy: {self.evaluate(val_data)}")

    def predict(self):
        pass

    def cost_derivative(self, output_activations, y):
        """Return the vector of partial derivatives \partial C_x /
        \partial a for the output activations."""
        return (output_activations - y)

    def evaluate(self, data):
        results = [
            np.argmax(self.forward(data) for (x, y) in data)
        ]
        return sum(int(x == y) for x, y in results)


if __name__ == "__main__":
    network = NeuralNetwork([2, 3, 1])
    print(network.biases)
    print(network.weights)