Pytorch中Tensor Variable Parameter以及求导运算

在较新的版本中，Variable被弃用，将功能合并给Tensor，所以不用考虑

• 求导
• • 重要理论
  • autogard
  • backward
  • no_gard
  • optimizer更新参数
• Tensor vs Parameter
• • 重要理论
  • 代码演示
  • 1 默认requires_grad
  • 模型绑定
  • • 查看模型参数
  • 绑定和优化

1. 从头开始创建的Tensor（例如x = torch.tensor(1.)）称为leaf Tensor（叶张量），依赖其他Tensor计算而来的（例如y = 2 * x）称为non-leaf Tensor（非叶张量）
2. 若一Tensor的requires_grad=True，则依赖它的所有Tensor的requires_grad=True；若一Tensor的requires_grad=False，则依赖它的所有Tensor的requires_grad=False。
3. 若一Tensor的requires_grad=True，则该Tensor可以.backward()向前传播梯度；若一Tensor的requires_grad=False，则该Tensor.backward()报错。
4. backward()只更新leaf的梯度，因为一般只优化leaf的值。若想得到non-leaf的梯度，可以使用autogard()函数。
5. 使用with torch.no_gard()可以使得某non-leaf Tensor的requires_grad=False（即使其依赖的Tensor的requires_grad=True)，这表示他不需要传播梯度，从而节省显存，但这对leaf Tensor无效。

# autogard可以得到non-leaf的梯度
import torch
from torch import autograd

x = torch.tensor(1.)
a = torch.tensor(1., requires_grad=True)
b = torch.tensor(2., requires_grad=True)
c = torch.tensor(3., requires_grad=True)

# requires_grad=True
y = a**2 * x + b * x + c

# requires_grad=True
z = y**2 + 2*y

print('before:', a.grad, b.grad, c.grad, y.grad)
grads = autograd.grad(z, [a, b, c, y])
print('after :', grads[0], grads[1], grads[2], grads[3])
print('after :', a.grad, b.grad, c.grad, y.grad)

# before: None None None None
# after : tensor(28.) tensor(14.) tensor(14.) tensor(14.)
# after : None None None None

# backward()只更新leaf的梯度，因为一般只优化leaf的值
import torch

x = torch.tensor(1.)
a = torch.tensor(1., requires_grad=True)
b = torch.tensor(2., requires_grad=True)
c = torch.tensor(3., requires_grad=True)

# requires_grad=True
y = a**2 * x + b * x + c

# requires_grad=True
z = y**2 + 2*y

print('before:', a.grad, b.grad, c.grad, y.grad)
z.backward()
print('after :', a.grad, b.grad, c.grad, y.grad)

# before: None None None None
# after : tensor(28.) tensor(14.) tensor(14.) None

常用的场景为测试过程，因为只有训练的过程需要梯度来更新参数，测试过程的不需要更新参数，所以将整个测试过程放到no_grad()下

with torch.no_grad():
        for data in test_dataloader:
            imgs, targets = data
            outputs = my_model(imgs)
            loss = loss_fn(outputs, targets)
            total_test_loss = total_test_loss + loss.item()
            accuracy = (outputs.argmax(1) == targets).sum()
            total_accuracy = total_accuracy + accuracy

# 使用with torch.no_gard()可以使得某non-leaf Tensor的requires_grad=False（即使其依赖的Tensor的requires_grad=True)，这表示他不需要传播梯度，从而节省显存，但这对leaf Tensor无效
import torch

x = torch.tensor(1.)
a = torch.tensor(1., requires_grad=True)
b = torch.tensor(2., requires_grad=True)
c = torch.tensor(3., requires_grad=True)

y = a**2 * x + b * x + c

z = y**2 + 2*y

# True True True
print(c.requires_grad, y.requires_grad, z.requires_grad)


# y不需要梯度
with torch.no_grad():
    y = a**2 * x + b * x + c

# z依赖于y，也不需要梯度
z = y**2 + 2*y

# True False False
print(c.requires_grad, y.requires_grad, z.requires_grad)

# 此处调用z.backward()报错，z的requires_grad=False


# no_grad()作用于leaf是无效的
with torch.no_grad():
    a = torch.tensor(1., requires_grad=True)
    b = torch.tensor(2., requires_grad=True)
    c = torch.tensor(3., requires_grad=True)

y = a**2 * x + b * x + c

z = y**2 + 2*y

print(c.requires_grad, y.requires_grad, z.requires_grad)
# True True True

# non-leaf不能被更新

import torch
from torch import optim

x = torch.tensor(1.)
a = torch.tensor(1., requires_grad=True)
b = torch.tensor(2., requires_grad=True)
c = torch.tensor(3., requires_grad=True)

y = a**2 * x + b * x + c

z = y**2 + 2*y

# 报错：can't optimize a non-leaf Tensor
# optimizer = optim.SGD([a, b, c, y], lr=0.01)

optimizer = optim.SGD([a, b, c], lr=0.01)
print('before:', a, b, c)
optimizer.zero_grad()
z.backward()
optimizer.step()
print('after :', a, b, c)

# before: tensor(1., requires_grad=True) tensor(2., requires_grad=True) tensor(3., requires_grad=True)
# after : tensor(0.7200, requires_grad=True) tensor(1.8600, requires_grad=True) tensor(2.8600, requires_grad=True)

1. Tensor默认requires_grad=False，Parameter默认requires_grad=True
2. Parameter和nn.Module搭配，可以绑定到模型中，搭配optimizer = optim.SGD(net.parameters(), lr=0.01)进行梯度下降

import torch
from torch import optim

x = torch.tensor(1.)
a = torch.tensor(1., requires_grad=True)
b = torch.tensor(2., requires_grad=True)
c = torch.tensor(3., requires_grad=True)

y = a**2 * x + b * x + c

z = y**2 + 2*y

optimizer = optim.SGD([a, b, c], lr=0.01)
print('before:', a, b, c)
optimizer.zero_grad()
z.backward()
optimizer.step()
print('after :', a, b, c)

# before: tensor(1., requires_grad=True) tensor(2., requires_grad=True) tensor(3., requires_grad=True)
# after : tensor(0.7200, requires_grad=True) tensor(1.8600, requires_grad=True) tensor(2.8600, requires_grad=True)

import torch
from torch import optim
from torch.nn.parameter import Parameter

x = torch.tensor(1.)

a = Parameter(torch.tensor(1.0))
b = Parameter(torch.tensor(1.0))
c = Parameter(torch.tensor(3.0))

y = a**2 * x + b * x + c

z = y**2 + 2*y

optimizer = optim.SGD([a, b, c], lr=0.01)
print('before:', a, b, c)
optimizer.zero_grad()
z.backward()
optimizer.step()
print('after :', a, b, c)

# before: Parameter containing:
# tensor(1., requires_grad=True) Parameter containing:
# tensor(1., requires_grad=True) Parameter containing:
# tensor(3., requires_grad=True)
# after : Parameter containing:
# tensor(0.7600, requires_grad=True) Parameter containing:
# tensor(0.8800, requires_grad=True) Parameter containing:
# tensor(2.8800, requires_grad=True)

import torch
from torch import optim
from torch.nn.parameter import Parameter
from torch import nn
from torch.nn import functional as F


class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()

        # xw+b
        self.fc1 = nn.Linear(28 * 28, 256)
        self.fc2 = nn.Linear(256, 64)
        self.fc3 = nn.Linear(64, 10)

    def forward(self, x):
        # x: [b, 1, 28, 28]
        # h1 = relu(xw1+b1)
        x = F.relu(self.fc1(x))
        # h2 = relu(h1w2+b2)
        x = F.relu(self.fc2(x))
        # h3 = h2w3+b3
        x = self.fc3(x)

        return x


net = Net()
print(net)
for name, parameter in net.named_parameters():
    print(name, ':', parameter.size())

# Net(
#   (fc1): Linear(in_features=784, out_features=256, bias=True)
#   (fc2): Linear(in_features=256, out_features=64, bias=True)
#   (fc3): Linear(in_features=64, out_features=10, bias=True)
# )
# fc1.weight : torch.Size([256, 784])
# fc1.bias : torch.Size([256])
# fc2.weight : torch.Size([64, 256])
# fc2.bias : torch.Size([64])
# fc3.weight : torch.Size([10, 64])
# fc3.bias : torch.Size([10])
#
# Process finished with exit code 0

import torch
from torch.nn.parameter import Parameter
from torch import nn
from torch.nn import functional as F
from torch import optim


class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()

        self.testTensor = torch.tensor([1.0, 1.0])
        self.testParameter = Parameter(torch.tensor([2.0, 2.0]))

    def forward(self, x):
        x = F.relu(self.testTensor * x)
        x = F.relu(self.testParameter * x)
        return x


net = Net()
for name, parameter in net.named_parameters():
    print(name, ':', parameter.size())
    # testParameter: torch.Size([2])
    # 即testTensor并没有绑定到模型中

input = torch.tensor([5.0, 5.0])
out = net(input)
loss = F.mse_loss(out, torch.tensor([3.0, 3.0]))

# 仅对绑定的参数进行更新
optimizer = optim.SGD(net.parameters(), lr=0.01)
for i in net.parameters():
    print('before:', i)
optimizer.zero_grad()
loss.backward()
optimizer.step()
for i in net.parameters():
    print('after:', i)