重みと損失関数の更新履歴を追跡してみた。(初心者が本とChatGPTを使ってAIを作って研究してみた）

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.datasets import mnist
from tensorflow.keras.utils import to_categorical

(x_train, t_train), (x_test, t_test) = mnist.load_data()

x_train = x_train.reshape((x_train.shape[0], -1)).astype('float32') / 255
x_test = x_test.reshape((x_test.shape[0], -1)).astype('float32') / 255
t_train = to_categorical(t_train, 10)
t_test = to_categorical(t_test, 10)
 
input_size = 784
output_size = 10
 
hidden_size = 50
 
weight_init_std = 0.01
 
W1 = weight_init_std * np.random.randn(input_size, hidden_size)
b1 = np.zeros(hidden_size)
W2 = weight_init_std * np.random.randn(hidden_size, output_size)
b2 = np.zeros(output_size)
 
def sigmoid(x):
    return 1 / (1 + np.exp(-x))
 
def softmax(x):
    x = x - np.max(x, axis=-1, keepdims=True)
    return np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True)
 
def sigmoid_grad(x):
        return (1.0 - sigmoid(x)) * sigmoid(x)
    
learning_rate = 0.1

x = x_train
t = t_train
 
W1_good_list = []
W1_notgood_list = []
W2_good_list = []
W2_notgood_list = []
MSE_list = []

for _ in range(10000):
    a1 = np.dot(x, W1) + b1
    z1 = sigmoid(a1)
    a2 = np.dot(z1, W2) + b2
    y = softmax(a2)

    dy = (y - t) / x.shape[0]
    MSE = (1/x.shape[0]) * np.sum((t - y)**2)
    MSE_list.append(MSE)

    grad_W2 = np.dot(z1.T, dy)
    grad_b2 = np.sum(dy, axis=0)
    W2 = W2 - learning_rate * grad_W2
    b2 = b2 - learning_rate * grad_b2
    
    dz1 = np.dot(dy, W2.T)
    da1 = sigmoid_grad(a1) * dz1
    grad_W1 = np.dot(x.T, da1)
    grad_b1 = np.sum(da1, axis=0)
    W1 = W1 - learning_rate * grad_W1
    b1 = b1 - learning_rate * grad_b1
    
    W1_good = W1[235,0]
    W1_notgood = W1[235,25]
    W2_good = W2[0,5]
    W2_notgood = W2[0,0]
    W1_good_list.append(W1_good)
    W1_notgood_list.append(W1_notgood)
    W2_good_list.append(W2_good)
    W2_notgood_list.append(W2_notgood)
    print(_)

x = x_train
a1 = np.dot(x, W1) + b1
z1 = sigmoid(a1)
a2 = np.dot(z1, W2) + b2
y = softmax(a2)

test_data = np.argmax(t, axis = 1)
pred_data = np.argmax(y, axis = 1)
error = np.sum(test_data == pred_data)
print(str(error) + " / " + str(x_train.shape[0]))

df = pd.DataFrame(list(zip(list(range(10000)), W1_good_list, W1_notgood_list, W2_good_list, W2_notgood_list)), columns=["iter", "W1_good", "W1_notgood", "W2_good", "W2_notgood"])
df.to_csv("dW.csv", index=None)

df2 = pd.DataFrame(list(zip(list(range(10000)), MSE_list)), columns=["iter", "MSE"])
df2.to_csv("loss.csv", index=None)