一步一步用numpy实现神经网络各种层
1. 首先准备一下数据
if __name__ == "__main__":
data = np.array([[2, 1, 0],
[2, 2, 0],
[5, 4, 1],
[4, 5, 1],
[2, 3, 0],
[3, 2, 0],
[6, 5, 1],
[4, 1, 0],
[6, 3, 1],
[7, 4, 1]])
x = data[:, :-1]
y = data[:, -1]
for epoch in range(1000):
...
2. 实现Softmax+CrossEntropy层
单独求softmax层有点麻烦, 将softmax+entropy一起求导更方便。
假设对于输入向量
(
x
1
,
x
2
,
x
3
)
(x_1, x_2, x_3)
(x1,x2,x3), 则对应的Loss为:
L
=
−
∑
i
=
1
C
y
i
ln
p
i
=
−
(
y
1
ln
p
1
+
y
2
ln
p
2
+
y
3
ln
p
3
)
\begin{align*} L&=-\sum_{i=1}^Cy_i \ln p^i \ &=-(y_1\ln p_1+y_2\ln p_2+y_3\ln p_3) \end{align*}
L=−i=1∑Cyilnpi=−(y1lnp1+y2lnp2+y3lnp3)
其中
y
i
y_i
yi为ground truth, 为one-hot vector.
p
i
p_i
pi为输出概率。
p
1
=
e
x
1
e
x
1
+
e
x
2
+
e
x
3
p
2
=
e
x
2
e
x
1
+
e
x
2
+
e
x
3
p
3
=
e
x
3
e
x
1
+
e
x
2
+
e
x
3
p_1=\frac{e^{x_1}}{e^{x_1}+e^{x_2}+e^{x_3}}\ p_2=\frac{e^{x_2}}{e^{x_1}+e^{x_2}+e^{x_3}}\ p_3=\frac{e^{x_3}}{e^{x_1}+e^{x_2}+e^{x_3}}\
p1=ex1+ex2+ex3ex1p2=ex1+ex2+ex3ex2p3=ex1+ex2+ex3ex3
则偏导为
∂
L
∂
x
1
=
−
y
1
1
p
1
∗
∂
p
1
∂
x
1
−
y
2
1
p
2
∗
∂
p
2
∂
x
1
−
y
3
1
p
3
∗
∂
p
3
∂
x
1
=
−
y
1
1
p
1
∗
e
x
1
∗
(
e
x
1
+
e
x
2
+
e
x
3
)
−
e
x
1
∗
e
x
1
(
e
x
1
+
e
x
2
+
e
x
3
)
2
−
y
2
1
p
2
∗
−
e
x
2
∗
e
x
1
(
e
x
1
+
e
x
2
+
e
x
3
)
2
−
y
3
1
p
3
∗
−
e
x
3
∗
e
x
1
(
e
x
1
+
e
x
2
+
e
x
3
)
2
=
−
y
1
1
p
1
(
p
1
∗
p
2
+
p
1
∗
p
3
)
−
y
2
1
p
2
(
−
p
1
∗
p
2
)
−
y
3
1
p
3
(
−
p
1
∗
p
3
)
=
−
y
1
(
p
2
+
p
3
)
+
y
2
∗
p
2
+
y
3
∗
p
3
=
−
y
1
(
1
−
p
1
)
+
y
2
∗
p
1
+
y
3
∗
p
1
=
y
1
(
p
1
−
1
)
+
y
2
∗
p
1
+
y
3
∗
p
1
\begin{align*} \frac{\partial L}{\partial x_1} &= -y_1\frac{1}{p_1}*\frac{\partial p_1}{\partial x_1} – y_2\frac{1}{p_2}*\frac{\partial p_2}{\partial x_1} – y_3\frac{1}{p_3}*\frac{\partial p_3}{\partial x_1} \ &= -y_1\frac{1}{p_1} * \frac{e^{x_1} * (e^{x_1}+e^{x_2}+e^{x_3})-e^{x_1}*e^{x_1}}{(e^{x_1}+e^{x_2}+e^{x_3})^2} \ &\quad\quad-y_2\frac{1}{p_2}*\frac{-e^{x_2}*e^{x_1}}{{(e^{x_1}+e^{x_2}+e^{x_3})^2}}\ &\quad\quad-y_3\frac{1}{p_3}*\frac{-e^{x_3}*e^{x_1}}{{(e^{x_1}+e^{x_2}+e^{x_3})^2}}\ &=-y_1\frac{1}{p_1}(p_1*p_2+p_1*p_3)\ &\quad\quad -y_2\frac{1}{p_2}(-p_1*p_2)\ &\quad\quad -y_3\frac{1}{p_3}(-p_1*p_3)\ &=-y1(p_2+p_3)+y_2*p_2+y_3*p_3\ &=-y_1(1-p_1)+y_2*p_1+y_3*p_1\ &=y_1(p_1-1)+y_2*p_1+y_3*p_1 \end{align*}
∂x1∂L=−y1p11∗∂x1∂p1−y2p21∗∂x1∂p2−y3p31∗∂x1∂p3=−y1p11∗(ex1+ex2+ex3)2ex1∗(ex1+ex2+ex3)−ex1∗ex1−y2p21∗(ex1+ex2+ex3)2−ex2∗ex1−y3p31∗(ex1+ex2+ex3)2−ex3∗ex1=−y1p11(p1∗p2+p1∗p3)−y2p21(−p1∗p2)−y3p31(−p1∗p3)=−y1(p2+p3)+y2∗p2+y3∗p3=−y1(1−p1)+y2∗p1+y3∗p1=y1(p1−1)+y2∗p1+y3∗p1
同理:
∂
L
∂
x
2
=
y
1
∗
p
2
+
y
2
(
p
2
−
1
)
+
y
3
∗
p
2
∂
L
∂
x
3
=
y
1
∗
p
3
+
y
2
p
3
+
y
3
∗
(
p
3
−
1
)
\frac{\partial L}{\partial x_2}=y_1*p_2+y_2(p_2-1)+y_3*p_2\ \frac{\partial L}{\partial x_3}=y_1*p_3+y_2p_3+y_3*(p_3-1)
∂x2∂L=y1∗p2+y2(p2−1)+y3∗p2∂x3∂L=y1∗p3+y2p3+y3∗(p3−1)
当
y
1
=
1
y_1=1
y1=1时, 对应的导数为
(
p
1
−
1
,
p
2
,
p
3
)
(p1-1, p_2, p_3)
(p1−1,p2,p3). 当
y
2
=
1
y_2=1
y2=1时,对应的导数为:
(
p
1
,
p
2
−
1
,
p
3
)
(p_1, p2-1, p3)
(p1,p2−1,p3).
例如求得概率为
(
0.2
,
0.3
,
0.5
)
(0.2, 0.3, 0.5)
(0.2,0.3,0.5), label为
(
0
,
0
,
1
)
(0, 0, 1)
(0,0,1), 则导数为
(
0.2
,
0.3
,
−
0.5
)
(0.2, 0.3, -0.5)
(0.2,0.3,−0.5)
python代码为:
注意求softmax时需要np.exp(x-np.max(x, axis=1, keepdims=True))防止指数运算溢出。
class Softmax:
def __init__(self, n_classes):
self.n_classes = n_classes
def forward(self, x, y):
prob = np.exp(x-np.max(x, axis=1, keepdims=True))
prob /= np.sum(prob, axis=1, keepdims=True)
# 选出y==1位置的概率
loss = -np.sum(np.log(prob[np.arange(len(y), y])) / len(y)
self.grad = prob.copy()
self.grad[np.arange(len(y), y] -= 1
"""
因为后面求导数都是直接np.sum而不是np.mean, 因此这里mean一次就可以了
"""
self.grad /= len(y)
return prob, loss
def backward(self):
return self.grad
3. 单独的CrossEntropy
python代码为:
class Entropy:
def __init__(self, n_classes):
self.n_classes = n_classes
self.grad = None
def forward(self, x, y):
# x: (b, c), y: (b)
b = y.shape[0]
one_hot_y = np.zeros((b, self.n_classes))
one_hot_y[range(len(y)), y] = 1
self.grad = one_hot_y * -1 / x
return np.mean(-one_hot_y * np.log(x), axis=0)
def backward(self):
return self.grad
2. 单独的Softmax层
from einops import repeat, rearrange, einsum
class Softmax:
def __init__(self):
def forward(self, x):
# x: (b, c)
x_exp = np.exp(x)
self.output = x_xep / np.sum(x_exp, axis=1, keep_dims=True)
return self.output
def backward(self, prev_grad):
b, c = self.output.shape
o = repeat(self.output, 'b c -> b c r', r=c)
I = repeat(np.eye(x.shape[1]), 'c1 c2 -> b c1 c2', b=b)
self.grad = o * (I - rearrange(o, 'b c1 c2 -> b c2 c1'))
return einsum(self.grad, grad[..., None], 'b c c, b c m -> b c m')[..., 0]
3. Linear层
注意更新
w
w
w时用的
d
w
d_w
dw, 但是往上一层传递的是
d
x
d_x
dx。因为上一层需要
d
L
/
d
o
u
t
dL/d_{out}
dL/dout, 而本层的输入
x
x
x即是上一次层的输出
d
L
/
d
o
u
t
=
d
L
/
d
x
dL/d_{out} = dL/dx
dL/dout=dL/dx
class Linear:
def __init__(self, in_channels, out_channels, lr):
self.lr = lr
self.w = np.random.rand(in_channels, out_channels)
self.b = np.random.rand(out_channels)
def forward(self, x):
self.x = x
return x@self.w + self.b
def backward(self, grad):
dx = einsum(prev_grad, rearrange(self.w, 'w1 w2 -> w2 w1'), 'c1 b, b c2 -> c1 c2')
dw = einsum(rearrange(self.x, 'b c -> c b'), prev_grad, 'c1 b, b c2 -> c1 c2')
db = np.sum(prev_grad, axis=0)
self.w -= self.lr * dw
self.b -= self.lr * db
"""
注意这里往上一层传递的是dx, 因为上一层需要dL/d_out, 而本层的输入x即是上一次层的输出
dL/d_out = dL/dx
"""
return dx
5. 完整训练代码
from einops import *
import numpy as np
class Softmax:
def __init__(self, train=True):
self.grad = None
self.train = train
def forward(self, x, y):
prob = np.exp(x-np.max(x, axis=1, keepdims=True))
prob /= np.sum(prob, axis=1, keepdims=True)
if self.train:
loss = -np.sum(np.log(prob[range(len(y)), y]))/len(y)
self.grad = prob.copy()
self.grad[range(len(y)), y] -= 1
self.grad /= len(y)
return prob, loss
else:
return prob
def backward(self):
return self.grad
class Linear:
def __init__(self, in_channels, out_channels, lr):
self.w = np.random.rand(in_channels, out_channels)
self.b = np.random.rand(out_channels)
self.lr = lr
def forward(self, x):
self.x = x
output = einsum(x, self.w, 'b c1, c1 c2 -> b c2') + self.b
return output
def backward(self, prev_grad):
cur_grad = einsum(rearrange(self.x, 'b c -> c b'), prev_grad, 'c1 b, b c2 -> c1 c2')
self.w -= self.lr * cur_grad
self.b -= self.lr * np.sum(prev_grad, axis=0)
return cur_grad
class Network:
def __init__(self, in_channels, out_channels, n_classes, lr):
self.lr = lr
self.linear = Linear(in_channels, out_channels, lr)
self.softmax = Softmax()
def forward(self, x, y=None):
out = self.linear.forward(x)
out = self.softmax.forward(out, y)
return out
def backward(self):
grad = self.softmax.backward()
grad = self.linear.backward(grad)
return grad
if __name__ == "__main__":
data = np.array([[2, 1, 0],
[2, 2, 0],
[5, 4, 1],
[4, 5, 1],
[2, 3, 0],
[3, 2, 0],
[6, 5, 1],
[4, 1, 0],
[6, 3, 1],
[7, 4, 1]])
# x = np.concatenate([np.array([[1]] * data.shape[0]), data[:, :2]], axis=1)
x = data[:, :-1]
y = data[:, -1:].flatten()
net = Network(2, 2, 2, 0.1)
# loss_fn = CrossEntropy(n_classes=2)
for epoch in range(500):
prob, loss = net.forward(x, y)
# loss = loss_fn.forward(out, y)
# grad_ = loss_fn.backward()
grad = net.backward()
print(loss)
net.softmax.train = False
print(net.forward(np.array([[0, 0], [0, 4], [8, 6], [10, 10]])), y)
