错误怎样穿越一层层神经元

参数 → 中间状态 → 预测结果 → 误差

z.grad = 6

z.grad = 2

z.grad += 6
z.grad += 2

z.grad == 8

输入 → 隐藏层1 → 隐藏层2 → 输出

import math


class Value:
    def __init__(
        self,
        data,
        children=(),
        op="",
        label="",
    ):
        self.data = float(data)
        self.grad = 0.0

        # 当前节点依赖的上游节点
        self._prev = tuple(children)

        self._op = op
        self.label = label

        # 当前节点的局部反向传播函数
        self._backward = lambda: None

    def __repr__(self):
        return (
            f"Value(data={self.data:.6f}, "
            f"grad={self.grad:.6f})"
        )

    def __add__(self, other):
        other = (
            other
            if isinstance(other, Value)
            else Value(other)
        )

        out = Value(
            self.data + other.data,
            (self, other),
            op="+",
        )

        def backward():
            # 加法对两个输入的局部导数均为1。
            self.grad += out.grad
            other.grad += out.grad

        out._backward = backward
        return out

    def __radd__(self, other):
        return self + other

    def __mul__(self, other):
        other = (
            other
            if isinstance(other, Value)
            else Value(other)
        )

        out = Value(
            self.data * other.data,
            (self, other),
            op="*",
        )

        def backward():
            self.grad += other.data * out.grad
            other.grad += self.data * out.grad

        out._backward = backward
        return out

    def __rmul__(self, other):
        return self * other

    def __pow__(self, exponent):
        if not isinstance(exponent, (int, float)):
            raise TypeError(
                "本教学版本仅支持常数指数"
            )

        out = Value(
            self.data ** exponent,
            (self,),
            op="pow",
        )

        def backward():
            self.grad += (
                exponent
                * self.data ** (exponent - 1)
                * out.grad
            )

        out._backward = backward
        return out

    def __neg__(self):
        return self * -1

    def __sub__(self, other):
        return self + (-other if isinstance(other, Value)
                       else -other)

    def __rsub__(self, other):
        return other + (-self)

    def tanh(self):
        result = math.tanh(self.data)

        out = Value(
            result,
            (self,),
            op="tanh",
        )

        def backward():
            self.grad += (
                (1 - result * result)
                * out.grad
            )

        out._backward = backward
        return out

    def sigmoid(self):
        # 数值稳定的Sigmoid
        if self.data >= 0:
            result = 1 / (1 + math.exp(-self.data))
        else:
            e = math.exp(self.data)
            result = e / (1 + e)

        out = Value(
            result,
            (self,),
            op="sigmoid",
        )

        def backward():
            self.grad += (
                result * (1 - result)
                * out.grad
            )

        out._backward = backward
        return out

w = Value(1)
x = Value(2)
b = Value(1)

z = w*x + b

w,x,b → z → a,c → y → L

def backward(self):
    """
    对当前标量输出执行反向模式自动微分。

    使用拓扑排序：
        先建立前向依赖顺序
        再按照逆序传播梯度
    """

    topo = []
    visited = set()

    def build(node):
        if node in visited:
            return

        visited.add(node)

        for parent in node._prev:
            build(parent)

        topo.append(node)

    build(self)

    # 本次调用重新计算完整梯度
    for node in topo:
        node.grad = 0.0

    # dL/dL = 1
    self.grad = 1.0

    for node in reversed(topo):
        node._backward()


Value.backward = backward

+=

=

x = Value(2, label="x")
w = Value(1, label="w")
b = Value(1, label="b")

target = Value(14)

z = w*x + b

a = z**2
c = 2*z

prediction = a + c

loss = 0.5 * (prediction - target)**2

print("前向结果:")
print("z =", z.data)
print("a =", a.data)
print("c =", c.data)
print("预测 =", prediction.data)
print("损失 =", loss.data)

loss.backward()

print()
print("反向梯度:")
print("dL/dz =", z.grad)
print("dL/dw =", w.grad)
print("dL/db =", b.grad)
print("dL/dx =", x.grad)

assert abs(loss.data - 0.5) < 1e-12
assert abs(z.grad - 8.0) < 1e-12
assert abs(w.grad - 16.0) < 1e-12
assert abs(b.grad - 8.0) < 1e-12
assert abs(x.grad - 8.0) < 1e-12

print("自动微分测试通过")

前向结果:
z = 3.0
a = 9.0
c = 6.0
预测 = 15.0
损失 = 0.5

反向梯度:
dL/dz = 8.0
dL/dw = 16.0
dL/db = 8.0
dL/dx = 8.0

自动微分测试通过

x = Value(3)

y = x*x + x

y.backward()

print("y =", y.data)
print("dy/dx =", x.grad)

assert y.data == 12
assert x.grad == 7

print("重复使用变量的梯度累加正确")

x*x

x = Value(2)
w = Value(1)
b = Value(1)

target = Value(14)

learning_rate = 0.002

for step in range(40):

    # 每轮重新建立计算图
    z = w*x + b
    prediction = z*z + 2*z
    loss = 0.5 * (prediction - target)**2

    # 计算全部参数梯度
    loss.backward()

    if step % 5 == 0:
        print(
            f"step={step:2d}",
            f"loss={loss.data:.8f}",
            f"w={w.data:.6f}",
            f"b={b.data:.6f}",
        )

    # 在图的本轮梯度全部计算完成后，
    # 再修改参数。
    w.data -= learning_rate * w.grad
    b.data -= learning_rate * b.grad

print("训练结束")

dL/dw = 16

def ordinary_loss(w, b, x=2.0, target=14.0):
    z = w*x + b
    prediction = z*z + 2*z

    return 0.5 * (prediction-target)**2


def numerical_gradient_w(w, b, h=1e-6):
    return (
        ordinary_loss(w+h, b)
        - ordinary_loss(w-h, b)
    ) / (2*h)


def numerical_gradient_b(w, b, h=1e-6):
    return (
        ordinary_loss(w, b+h)
        - ordinary_loss(w, b-h)
    ) / (2*h)


print("w数值梯度:", numerical_gradient_w(1, 1))
print("b数值梯度:", numerical_gradient_b(1, 1))

assert abs(
    numerical_gradient_w(1, 1) - 16
) < 1e-5

assert abs(
    numerical_gradient_b(1, 1) - 8
) < 1e-5

print("有限差分梯度检查通过")

prediction = model(x)
loss = criterion(prediction, target)

loss.backward()

optimizer.step()

loss.backward()

loss.backward()

.grad

loss.backward()

+=

optimizer.zero_grad()