simple Neural Net example

@inq is giving a tutorial on building a neural network from ground up.

As a supplement I have found this link helpful as a simple starter.

https://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/

I wrote a FreeBASIC version at the time (2018) and this is a GPTChat Python translation:

import math


def sigmoid(x):
    return 1 / (1 + math.exp(-x))


# ============================================================
# INITIAL VALUES
# ============================================================

# Inputs
i1 = 0.05
i2 = 0.10

# Weights
w1 = 0.15
w2 = 0.20
w3 = 0.25
w4 = 0.30
w5 = 0.40
w6 = 0.45
w7 = 0.50
w8 = 0.55

# Biases
b1 = 0.35
b2 = 0.60

# Targets
target_o1 = 0.01
target_o2 = 0.99

# Learning rate
learning_rate = 0.5


# ============================================================
# FORWARD PASS
# ============================================================

# Hidden neuron h1
net_h1 = w1 * i1 + w2 * i2 + b1
out_h1 = sigmoid(net_h1)

# Hidden neuron h2
net_h2 = w3 * i1 + w4 * i2 + b1
out_h2 = sigmoid(net_h2)

# Output neuron o1
net_o1 = w5 * out_h1 + w6 * out_h2 + b2
out_o1 = sigmoid(net_o1)

# Output neuron o2
net_o2 = w7 * out_h1 + w8 * out_h2 + b2
out_o2 = sigmoid(net_o2)


# ============================================================
# ERROR
# ============================================================

E_o1 = 0.5 * (target_o1 - out_o1) ** 2
E_o2 = 0.5 * (target_o2 - out_o2) ** 2

E_total = E_o1 + E_o2


# ============================================================
# BACKWARD PASS
#
# First calculate all new weights using the ORIGINAL weights.
# The actual update happens after all gradients have been
# calculated.
# ============================================================


# ============================================================
# OUTPUT LAYER
# ============================================================

# ---- w5 ----

dE_do1 = -(target_o1 - out_o1)
do1_dneto1 = out_o1 * (1 - out_o1)
dneto1_dw5 = out_h1

gradient_w5 = dE_do1 * do1_dneto1 * dneto1_dw5

new_w5 = w5 - learning_rate * gradient_w5


# ---- w6 ----

dE_do1 = -(target_o1 - out_o1)
do1_dneto1 = out_o1 * (1 - out_o1)
dneto1_dw6 = out_h2

gradient_w6 = dE_do1 * do1_dneto1 * dneto1_dw6

new_w6 = w6 - learning_rate * gradient_w6


# ---- w7 ----

dE_do2 = -(target_o2 - out_o2)
do2_dneto2 = out_o2 * (1 - out_o2)
dneto2_dw7 = out_h1

gradient_w7 = dE_do2 * do2_dneto2 * dneto2_dw7

new_w7 = w7 - learning_rate * gradient_w7


# ---- w8 ----

dE_do2 = -(target_o2 - out_o2)
do2_dneto2 = out_o2 * (1 - out_o2)
dneto2_dw8 = out_h2

gradient_w8 = dE_do2 * do2_dneto2 * dneto2_dw8

new_w8 = w8 - learning_rate * gradient_w8


# ============================================================
# HIDDEN LAYER
# ============================================================

# ------------------------------------------------------------
# w1
# ------------------------------------------------------------

# Error flowing from output neuron o1
dE_do1 = -(target_o1 - out_o1)
do1_dneto1 = out_o1 * (1 - out_o1)
dneto1_douth1 = w5

dEo1_douth1 = dE_do1 * do1_dneto1 * dneto1_douth1


# Error flowing from output neuron o2
dE_do2 = -(target_o2 - out_o2)
do2_dneto2 = out_o2 * (1 - out_o2)
dneto2_douth1 = w7

dEo2_douth1 = dE_do2 * do2_dneto2 * dneto2_douth1


# Total error with respect to h1 output
dE_douth1 = dEo1_douth1 + dEo2_douth1

# h1 sigmoid derivative
douth1_dneth1 = out_h1 * (1 - out_h1)

# h1 net input with respect to w1
dneth1_dw1 = i1

gradient_w1 = (
    dE_douth1
    * douth1_dneth1
    * dneth1_dw1
)

new_w1 = w1 - learning_rate * gradient_w1


# ------------------------------------------------------------
# w2
# ------------------------------------------------------------

dneth1_dw2 = i2

gradient_w2 = (
    dE_douth1
    * douth1_dneth1
    * dneth1_dw2
)

new_w2 = w2 - learning_rate * gradient_w2


# ------------------------------------------------------------
# w3
# ------------------------------------------------------------

# Error flowing through h2
dE_do1 = -(target_o1 - out_o1)
do1_dneto1 = out_o1 * (1 - out_o1)
dneto1_douth2 = w6

dEo1_douth2 = dE_do1 * do1_dneto1 * dneto1_douth2


dE_do2 = -(target_o2 - out_o2)
do2_dneto2 = out_o2 * (1 - out_o2)
dneto2_douth2 = w8

dEo2_douth2 = dE_do2 * do2_dneto2 * dneto2_douth2


# Total error with respect to h2 output
dE_douth2 = dEo1_douth2 + dEo2_douth2

# h2 sigmoid derivative
douth2_dneth2 = out_h2 * (1 - out_h2)

# h2 net input with respect to w3
dneth2_dw3 = i1

gradient_w3 = (
    dE_douth2
    * douth2_dneth2
    * dneth2_dw3
)

new_w3 = w3 - learning_rate * gradient_w3


# ------------------------------------------------------------
# w4
# ------------------------------------------------------------

dneth2_dw4 = i2

gradient_w4 = (
    dE_douth2
    * douth2_dneth2
    * dneth2_dw4
)

new_w4 = w4 - learning_rate * gradient_w4


# ============================================================
# DISPLAY RESULTS
# ============================================================

print("Forward pass:")
print(f"net_h1  = {net_h1:.9f}")
print(f"out_h1  = {out_h1:.9f}")
print(f"net_h2  = {net_h2:.9f}")
print(f"out_h2  = {out_h2:.9f}")
print(f"net_o1  = {net_o1:.9f}")
print(f"out_o1  = {out_o1:.9f}")
print(f"net_o2  = {net_o2:.9f}")
print(f"out_o2  = {out_o2:.9f}")
print()

print("Error:")
print(f"E_o1    = {E_o1:.9f}")
print(f"E_o2    = {E_o2:.9f}")
print(f"E_total = {E_total:.9f}")
print()

print("New weights:")
print(f"new w1 = {new_w1:.9f}")
print(f"new w2 = {new_w2:.9f}")
print(f"new w3 = {new_w3:.9f}")
print(f"new w4 = {new_w4:.9f}")
print(f"new w5 = {new_w5:.9f}")
print(f"new w6 = {new_w6:.9f}")
print(f"new w7 = {new_w7:.9f}")
print(f"new w8 = {new_w8:.9f}")

A utube someone recommended to me at the time:

https://www.youtube.com/watch?v=BR9h47Jtqyw

Evolution is learning system. Slightly modify the DNA on a set of animals and see which one survives long enough to reproduce. The survivors repeat the process thus complexity increases and the sticks.