NNDL实验 优化算法3D轨迹 复现cs231经典动画

这个动画很有名气，学习深度学习的朋友大多都见过。

CS231n Convolutional Neural Networks for Visual Recognition

复现动画，向经典致敬~ 

【注】本例中3D动画被优化函数使用的是：- x[0] * x[0] / 2 + x[1] * x[1] / 1，与原图并不完全吻合。

2D动画使用的被优化函数更为复杂，有兴趣的同学可以尝试。

第一版：轨迹版

第二版：小球版

“轨迹版”不如原图生动，观察后，发现缺少移动的小球~

失去了小球，就失去了灵魂~

赶紧把灵魂找回来~

 第三版：综合版

这样就有点“高仿”的意思了~  收工~

俯视效果：

放在一起对比下：

代码改编自《神经网络与深度学习：案例与实践》

NNDL实验 优化算法3D轨迹 pytorch版_HBU_David的博客-CSDN博客

第一版 源代码：

  1. import torch
  2. import numpy as np
  3. import copy
  4. from matplotlib import pyplot as plt
  5. from matplotlib import animation
  6. from itertools import zip_longest
  7. from matplotlib import cm
  8.  
  9.  
  10. class Op(object):
  11.     def __init__(self):
  12.         pass
  13.  
  14.     def __call__(self, inputs):
  15.         return self.forward(inputs)
  16.  
  17.     # 输入：张量inputs
  18.     # 输出：张量outputs
  19.     def forward(self, inputs):
  20.         # return outputs
  21.         raise NotImplementedError
  22.  
  23.     # 输入：最终输出对outputs的梯度outputs_grads
  24.     # 输出：最终输出对inputs的梯度inputs_grads
  25.     def backward(self, outputs_grads):
  26.         # return inputs_grads
  27.         raise NotImplementedError
  28.  
  29.  
  30. class Optimizer(object):  # 优化器基类
  31.     def __init__(self, init_lr, model):
  32.         """
  33.         优化器类初始化
  34.         """
  35.         # 初始化学习率，用于参数更新的计算
  36.         self.init_lr = init_lr
  37.         # 指定优化器需要优化的模型
  38.         self.model = model
  39.  
  40.     def step(self):
  41.         """
  42.         定义每次迭代如何更新参数
  43.         """
  44.         pass
  45.  
  46.  
  47. class SimpleBatchGD(Optimizer):
  48.     def __init__(self, init_lr, model):
  49.         super(SimpleBatchGD, self).__init__(init_lr=init_lr, model=model)
  50.  
  51.     def step(self):
  52.         # 参数更新
  53.         if isinstance(self.model.params, dict):
  54.             for key in self.model.params.keys():
  55.                 self.model.params[key] = self.model.params[key] - self.init_lr * self.model.grads[key]
  56.  
  57.  
  58. class Adagrad(Optimizer):
  59.     def __init__(self, init_lr, model, epsilon):
  60.         """
  61.         Adagrad 优化器初始化
  62.         输入：
  63.             - init_lr： 初始学习率 - model：模型，model.params存储模型参数值  - epsilon：保持数值稳定性而设置的非常小的常数
  64.         """
  65.         super(Adagrad, self).__init__(init_lr=init_lr, model=model)
  66.         self.G = {}
  67.         for key in self.model.params.keys():
  68.             self.G[key] = 0
  69.         self.epsilon = epsilon
  70.  
  71.     def adagrad(self, x, gradient_x, G, init_lr):
  72.         """
  73.         adagrad算法更新参数，G为参数梯度平方的累计值。
  74.         """
  75.         G += gradient_x ** 2
  76.         x -= init_lr / torch.sqrt(G + self.epsilon) * gradient_x
  77.         return x, G
  78.  
  79.     def step(self):
  80.         """
  81.         参数更新
  82.         """
  83.         for key in self.model.params.keys():
  84.             self.model.params[key], self.G[key] = self.adagrad(self.model.params[key],
  85.                                                                self.model.grads[key],
  86.                                                                self.G[key],
  87.                                                                self.init_lr)
  88.  
  89.  
  90. class RMSprop(Optimizer):
  91.     def __init__(self, init_lr, model, beta, epsilon):
  92.         """
  93.         RMSprop优化器初始化
  94.         输入：
  95.             - init_lr：初始学习率
  96.             - model：模型，model.params存储模型参数值
  97.             - beta：衰减率
  98.             - epsilon：保持数值稳定性而设置的常数
  99.         """
  100.         super(RMSprop, self).__init__(init_lr=init_lr, model=model)
  101.         self.G = {}
  102.         for key in self.model.params.keys():
  103.             self.G[key] = 0
  104.         self.beta = beta
  105.         self.epsilon = epsilon
  106.  
  107.     def rmsprop(self, x, gradient_x, G, init_lr):
  108.         """
  109.         rmsprop算法更新参数，G为迭代梯度平方的加权移动平均
  110.         """
  111.         G = self.beta * G + (1 - self.beta) * gradient_x ** 2
  112.         x -= init_lr / torch.sqrt(G + self.epsilon) * gradient_x
  113.         return x, G
  114.  
  115.     def step(self):
  116.         """参数更新"""
  117.         for key in self.model.params.keys():
  118.             self.model.params[key], self.G[key] = self.rmsprop(self.model.params[key],
  119.                                                                self.model.grads[key],
  120.                                                                self.G[key],
  121.                                                                self.init_lr)
  122.  
  123.  
  124. class Momentum(Optimizer):
  125.     def __init__(self, init_lr, model, rho):
  126.         """
  127.         Momentum优化器初始化
  128.         输入：
  129.             - init_lr：初始学习率
  130.             - model：模型，model.params存储模型参数值
  131.             - rho：动量因子
  132.         """
  133.         super(Momentum, self).__init__(init_lr=init_lr, model=model)
  134.         self.delta_x = {}
  135.         for key in self.model.params.keys():
  136.             self.delta_x[key] = 0
  137.         self.rho = rho
  138.  
  139.     def momentum(self, x, gradient_x, delta_x, init_lr):
  140.         """
  141.         momentum算法更新参数，delta_x为梯度的加权移动平均
  142.         """
  143.         delta_x = self.rho * delta_x - init_lr * gradient_x
  144.         x += delta_x
  145.         return x, delta_x
  146.  
  147.     def step(self):
  148.         """参数更新"""
  149.         for key in self.model.params.keys():
  150.             self.model.params[key], self.delta_x[key] = self.momentum(self.model.params[key],
  151.                                                                       self.model.grads[key],
  152.                                                                       self.delta_x[key],
  153.                                                                       self.init_lr)
  154.  
  155.  
  156. class Adam(Optimizer):
  157.     def __init__(self, init_lr, model, beta1, beta2, epsilon):
  158.         """
  159.         Adam优化器初始化
  160.         输入：
  161.             - init_lr：初始学习率
  162.             - model：模型，model.params存储模型参数值
  163.             - beta1, beta2：移动平均的衰减率
  164.             - epsilon：保持数值稳定性而设置的常数
  165.         """
  166.         super(Adam, self).__init__(init_lr=init_lr, model=model)
  167.         self.beta1 = beta1
  168.         self.beta2 = beta2
  169.         self.epsilon = epsilon
  170.         self.M, self.G = {}, {}
  171.         for key in self.model.params.keys():
  172.             self.M[key] = 0
  173.             self.G[key] = 0
  174.         self.t = 1
  175.  
  176.     def adam(self, x, gradient_x, G, M, t, init_lr):
  177.         """
  178.         adam算法更新参数
  179.         输入：
  180.             - x：参数
  181.             - G：梯度平方的加权移动平均
  182.             - M：梯度的加权移动平均
  183.             - t：迭代次数
  184.             - init_lr：初始学习率
  185.         """
  186.         M = self.beta1 * M + (1 - self.beta1) * gradient_x
  187.         G = self.beta2 * G + (1 - self.beta2) * gradient_x ** 2
  188.         M_hat = M / (1 - self.beta1 ** t)
  189.         G_hat = G / (1 - self.beta2 ** t)
  190.         t += 1
  191.         x -= init_lr / torch.sqrt(G_hat + self.epsilon) * M_hat
  192.         return x, G, M, t
  193.  
  194.     def step(self):
  195.         """参数更新"""
  196.         for key in self.model.params.keys():
  197.             self.model.params[key], self.G[key], self.M[key], self.t = self.adam(self.model.params[key],
  198.                                                                                  self.model.grads[key],
  199.                                                                                  self.G[key],
  200.                                                                                  self.M[key],
  201.                                                                                  self.t,
  202.                                                                                  self.init_lr)
  203.  
  204.  
  205. class OptimizedFunction3D(Op):
  206.     def __init__(self):
  207.         super(OptimizedFunction3D, self).__init__()
  208.         self.params = {'x': 0}
  209.         self.grads = {'x': 0}
  210.  
  211.     def forward(self, x):
  212.         self.params['x'] = x
  213.         return - x[0] * x[0] / 2 + x[1] * x[1] / 1  # x[0] ** 2 + x[1] ** 2 + x[1] ** 3 + x[0] * x[1]
  214.  
  215.     def backward(self):
  216.         x = self.params['x']
  217.         gradient1 = - 2 * x[0] / 2
  218.         gradient2 = 2 * x[1] / 1
  219.         grad1 = torch.Tensor([gradient1])
  220.         grad2 = torch.Tensor([gradient2])
  221.         self.grads['x'] = torch.cat([grad1, grad2])
  222.  
  223.  
  224. class Visualization3D(animation.FuncAnimation):
  225.     """    绘制动态图像，可视化参数更新轨迹    """
  226.  
  227.     def __init__(self, *xy_values, z_values, labels=[], colors=[], fig, ax, interval=100, blit=True, **kwargs):
  228.         """
  229.         初始化3d可视化类
  230.         输入：
  231.             xy_values：三维中x,y维度的值
  232.             z_values：三维中z维度的值
  233.             labels：每个参数更新轨迹的标签
  234.             colors：每个轨迹的颜色
  235.             interval：帧之间的延迟（以毫秒为单位）
  236.             blit：是否优化绘图
  237.         """
  238.         self.fig = fig
  239.         self.ax = ax
  240.         self.xy_values = xy_values
  241.         self.z_values = z_values
  242.  
  243.         frames = max(xy_value.shape[0] for xy_value in xy_values)
  244.         # , marker = 'o'
  245.         self.lines = [ax.plot([], [], [], label=label, color=color, lw=2)[0]
  246.                       for _, label, color in zip_longest(xy_values, labels, colors)]
  247.         print(self.lines)
  248.         super(Visualization3D, self).__init__(fig, self.animate, init_func=self.init_animation, frames=frames,
  249.                                               interval=interval, blit=blit, **kwargs)
  250.  
  251.     def init_animation(self):
  252.         # 数值初始化
  253.         for line in self.lines:
  254.             line.set_data([], [])
  255.             # line.set_3d_properties(np.asarray([]))  # 源程序中有这一行，加上会报错。 Edit by David 2022.12.4
  256.         return self.lines
  257.  
  258.     def animate(self, i):
  259.         # 将x,y,z三个数据传入，绘制三维图像
  260.         for line, xy_value, z_value in zip(self.lines, self.xy_values, self.z_values):
  261.             line.set_data(xy_value[:i, 0], xy_value[:i, 1])
  262.             line.set_3d_properties(z_value[:i])
  263.         return self.lines
  264.  
  265.  
  266. def train_f(model, optimizer, x_init, epoch):
  267.     x = x_init
  268.     all_x = []
  269.     losses = []
  270.     for i in range(epoch):
  271.         all_x.append(copy.deepcopy(x.numpy()))  # 浅拷贝 改为 深拷贝, 否则List的原值会被改变。 Edit by David 2022.12.4.
  272.         loss = model(x)
  273.         losses.append(loss)
  274.         model.backward()
  275.         optimizer.step()
  276.         x = model.params['x']
  277.     return torch.Tensor(np.array(all_x)), losses
  278.  
  279.  
  280. # 构建5个模型，分别配备不同的优化器
  281. model1 = OptimizedFunction3D()
  282. opt_gd = SimpleBatchGD(init_lr=0.05, model=model1)
  283.  
  284. model2 = OptimizedFunction3D()
  285. opt_adagrad = Adagrad(init_lr=0.05, model=model2, epsilon=1e-7)
  286.  
  287. model3 = OptimizedFunction3D()
  288. opt_rmsprop = RMSprop(init_lr=0.05, model=model3, beta=0.9, epsilon=1e-7)
  289.  
  290. model4 = OptimizedFunction3D()
  291. opt_momentum = Momentum(init_lr=0.05, model=model4, rho=0.9)
  292.  
  293. model5 = OptimizedFunction3D()
  294. opt_adam = Adam(init_lr=0.05, model=model5, beta1=0.9, beta2=0.99, epsilon=1e-7)
  295.  
  296. models = [model5, model2, model3, model4, model1]
  297. opts = [opt_adam, opt_adagrad, opt_rmsprop, opt_momentum, opt_gd]
  298.  
  299. x_all_opts = []
  300. z_all_opts = []
  301.  
  302. # 使用不同优化器训练
  303.  
  304. for model, opt in zip(models, opts):
  305.     x_init = torch.FloatTensor([0.00001, 0.5])
  306.     x_one_opt, z_one_opt = train_f(model, opt, x_init, 100)  # epoch
  307.     # 保存参数值
  308.     x_all_opts.append(x_one_opt.numpy())
  309.     z_all_opts.append(np.squeeze(z_one_opt))
  310.  
  311. # 使用numpy.meshgrid生成x1,x2矩阵，矩阵的每一行为[-3, 3]，以0.1为间隔的数值
  312. x1 = np.arange(-1, 2, 0.01)
  313. x2 = np.arange(-1, 1, 0.05)
  314. x1, x2 = np.meshgrid(x1, x2)
  315. init_x = torch.Tensor(np.array([x1, x2]))
  316.  
  317. model = OptimizedFunction3D()
  318.  
  319. # 绘制 f_3d函数 的 三维图像
  320. fig = plt.figure()
  321. ax = plt.axes(projection='3d')
  322. X = init_x[0].numpy()
  323. Y = init_x[1].numpy()
  324. Z = model(init_x).numpy()  # 改为 model(init_x).numpy() David 2022.12.4
  325. surf = ax.plot_surface(X, Y, Z, edgecolor='grey', cmap=cm.coolwarm)
  326. # fig.colorbar(surf, shrink=0.5, aspect=1)
  327. ax.set_zlim(-3, 2)
  328. ax.set_xlabel('x1')
  329. ax.set_ylabel('x2')
  330. ax.set_zlabel('f(x1,x2)')
  331.  
  332. labels = ['Adam', 'AdaGrad', 'RMSprop', 'Momentum', 'SGD']
  333. colors = ['#8B0000', '#0000FF', '#000000', '#008B00', '#FF0000']
  334.  
  335. animator = Visualization3D(*x_all_opts, z_values=z_all_opts, labels=labels, colors=colors, fig=fig, ax=ax)
  336. ax.legend(loc='upper right')
  337.  
  338. plt.show()
  339. # animator.save('animation.gif') # 效果不好，估计被挡住了…… 有待进一步提高 Edit by David 2022.12.4

第二版 源代码：

  1. import torch
  2. import numpy as np
  3. import copy
  4. from matplotlib import pyplot as plt
  5. from matplotlib import animation
  6. from itertools import zip_longest
  7. from matplotlib import cm
  8.  
  9.  
  10. class Op(object):
  11.     def __init__(self):
  12.         pass
  13.  
  14.     def __call__(self, inputs):
  15.         return self.forward(inputs)
  16.  
  17.     # 输入：张量inputs
  18.     # 输出：张量outputs
  19.     def forward(self, inputs):
  20.         # return outputs
  21.         raise NotImplementedError
  22.  
  23.     # 输入：最终输出对outputs的梯度outputs_grads
  24.     # 输出：最终输出对inputs的梯度inputs_grads
  25.     def backward(self, outputs_grads):
  26.         # return inputs_grads
  27.         raise NotImplementedError
  28.  
  29.  
  30. class Optimizer(object):  # 优化器基类
  31.     def __init__(self, init_lr, model):
  32.         """
  33.         优化器类初始化
  34.         """
  35.         # 初始化学习率，用于参数更新的计算
  36.         self.init_lr = init_lr
  37.         # 指定优化器需要优化的模型
  38.         self.model = model
  39.  
  40.     def step(self):
  41.         """
  42.         定义每次迭代如何更新参数
  43.         """
  44.         pass
  45.  
  46.  
  47. class SimpleBatchGD(Optimizer):
  48.     def __init__(self, init_lr, model):
  49.         super(SimpleBatchGD, self).__init__(init_lr=init_lr, model=model)
  50.  
  51.     def step(self):
  52.         # 参数更新
  53.         if isinstance(self.model.params, dict):
  54.             for key in self.model.params.keys():
  55.                 self.model.params[key] = self.model.params[key] - self.init_lr * self.model.grads[key]
  56.  
  57.  
  58. class Adagrad(Optimizer):
  59.     def __init__(self, init_lr, model, epsilon):
  60.         """
  61.         Adagrad 优化器初始化
  62.         输入：
  63.             - init_lr： 初始学习率 - model：模型，model.params存储模型参数值  - epsilon：保持数值稳定性而设置的非常小的常数
  64.         """
  65.         super(Adagrad, self).__init__(init_lr=init_lr, model=model)
  66.         self.G = {}
  67.         for key in self.model.params.keys():
  68.             self.G[key] = 0
  69.         self.epsilon = epsilon
  70.  
  71.     def adagrad(self, x, gradient_x, G, init_lr):
  72.         """
  73.         adagrad算法更新参数，G为参数梯度平方的累计值。
  74.         """
  75.         G += gradient_x ** 2
  76.         x -= init_lr / torch.sqrt(G + self.epsilon) * gradient_x
  77.         return x, G
  78.  
  79.     def step(self):
  80.         """
  81.         参数更新
  82.         """
  83.         for key in self.model.params.keys():
  84.             self.model.params[key], self.G[key] = self.adagrad(self.model.params[key],
  85.                                                                self.model.grads[key],
  86.                                                                self.G[key],
  87.                                                                self.init_lr)
  88.  
  89.  
  90. class RMSprop(Optimizer):
  91.     def __init__(self, init_lr, model, beta, epsilon):
  92.         """
  93.         RMSprop优化器初始化
  94.         输入：
  95.             - init_lr：初始学习率
  96.             - model：模型，model.params存储模型参数值
  97.             - beta：衰减率
  98.             - epsilon：保持数值稳定性而设置的常数
  99.         """
  100.         super(RMSprop, self).__init__(init_lr=init_lr, model=model)
  101.         self.G = {}
  102.         for key in self.model.params.keys():
  103.             self.G[key] = 0
  104.         self.beta = beta
  105.         self.epsilon = epsilon
  106.  
  107.     def rmsprop(self, x, gradient_x, G, init_lr):
  108.         """
  109.         rmsprop算法更新参数，G为迭代梯度平方的加权移动平均
  110.         """
  111.         G = self.beta * G + (1 - self.beta) * gradient_x ** 2
  112.         x -= init_lr / torch.sqrt(G + self.epsilon) * gradient_x
  113.         return x, G
  114.  
  115.     def step(self):
  116.         """参数更新"""
  117.         for key in self.model.params.keys():
  118.             self.model.params[key], self.G[key] = self.rmsprop(self.model.params[key],
  119.                                                                self.model.grads[key],
  120.                                                                self.G[key],
  121.                                                                self.init_lr)
  122.  
  123.  
  124. class Momentum(Optimizer):
  125.     def __init__(self, init_lr, model, rho):
  126.         """
  127.         Momentum优化器初始化
  128.         输入：
  129.             - init_lr：初始学习率
  130.             - model：模型，model.params存储模型参数值
  131.             - rho：动量因子
  132.         """
  133.         super(Momentum, self).__init__(init_lr=init_lr, model=model)
  134.         self.delta_x = {}
  135.         for key in self.model.params.keys():
  136.             self.delta_x[key] = 0
  137.         self.rho = rho
  138.  
  139.     def momentum(self, x, gradient_x, delta_x, init_lr):
  140.         """
  141.         momentum算法更新参数，delta_x为梯度的加权移动平均
  142.         """
  143.         delta_x = self.rho * delta_x - init_lr * gradient_x
  144.         x += delta_x
  145.         return x, delta_x
  146.  
  147.     def step(self):
  148.         """参数更新"""
  149.         for key in self.model.params.keys():
  150.             self.model.params[key], self.delta_x[key] = self.momentum(self.model.params[key],
  151.                                                                       self.model.grads[key],
  152.                                                                       self.delta_x[key],
  153.                                                                       self.init_lr)
  154.  
  155.  
  156. class Adam(Optimizer):
  157.     def __init__(self, init_lr, model, beta1, beta2, epsilon):
  158.         """
  159.         Adam优化器初始化
  160.         输入：
  161.             - init_lr：初始学习率
  162.             - model：模型，model.params存储模型参数值
  163.             - beta1, beta2：移动平均的衰减率
  164.             - epsilon：保持数值稳定性而设置的常数
  165.         """
  166.         super(Adam, self).__init__(init_lr=init_lr, model=model)
  167.         self.beta1 = beta1
  168.         self.beta2 = beta2
  169.         self.epsilon = epsilon
  170.         self.M, self.G = {}, {}
  171.         for key in self.model.params.keys():
  172.             self.M[key] = 0
  173.             self.G[key] = 0
  174.         self.t = 1
  175.  
  176.     def adam(self, x, gradient_x, G, M, t, init_lr):
  177.         """
  178.         adam算法更新参数
  179.         输入：
  180.             - x：参数
  181.             - G：梯度平方的加权移动平均
  182.             - M：梯度的加权移动平均
  183.             - t：迭代次数
  184.             - init_lr：初始学习率
  185.         """
  186.         M = self.beta1 * M + (1 - self.beta1) * gradient_x
  187.         G = self.beta2 * G + (1 - self.beta2) * gradient_x ** 2
  188.         M_hat = M / (1 - self.beta1 ** t)
  189.         G_hat = G / (1 - self.beta2 ** t)
  190.         t += 1
  191.         x -= init_lr / torch.sqrt(G_hat + self.epsilon) * M_hat
  192.         return x, G, M, t
  193.  
  194.     def step(self):
  195.         """参数更新"""
  196.         for key in self.model.params.keys():
  197.             self.model.params[key], self.G[key], self.M[key], self.t = self.adam(self.model.params[key],
  198.                                                                                  self.model.grads[key],
  199.                                                                                  self.G[key],
  200.                                                                                  self.M[key],
  201.                                                                                  self.t,
  202.                                                                                  self.init_lr)
  203.  
  204.  
  205. class OptimizedFunction3D(Op):
  206.     def __init__(self):
  207.         super(OptimizedFunction3D, self).__init__()
  208.         self.params = {'x': 0}
  209.         self.grads = {'x': 0}
  210.  
  211.     def forward(self, x):
  212.         self.params['x'] = x
  213.         return - x[0] * x[0] / 2 + x[1] * x[1] / 1  # x[0] ** 2 + x[1] ** 2 + x[1] ** 3 + x[0] * x[1]
  214.  
  215.     def backward(self):
  216.         x = self.params['x']
  217.         gradient1 = - 2 * x[0] / 2
  218.         gradient2 = 2 * x[1] / 1
  219.         grad1 = torch.Tensor([gradient1])
  220.         grad2 = torch.Tensor([gradient2])
  221.         self.grads['x'] = torch.cat([grad1, grad2])
  222.  
  223.  
  224. class Visualization3D(animation.FuncAnimation):
  225.     """    绘制动态图像，可视化参数更新轨迹    """
  226.  
  227.     def __init__(self, *xy_values, z_values, labels=[], colors=[], fig, ax, interval=100, blit=True, **kwargs):
  228.         """
  229.         初始化3d可视化类
  230.         输入：
  231.             xy_values：三维中x,y维度的值
  232.             z_values：三维中z维度的值
  233.             labels：每个参数更新轨迹的标签
  234.             colors：每个轨迹的颜色
  235.             interval：帧之间的延迟（以毫秒为单位）
  236.             blit：是否优化绘图
  237.         """
  238.         self.fig = fig
  239.         self.ax = ax
  240.         self.xy_values = xy_values
  241.         self.z_values = z_values
  242.  
  243.         frames = max(xy_value.shape[0] for xy_value in xy_values)
  244.  
  245.         self.lines = [ax.plot([], [], [], label=label, color=color, lw=2)[0]
  246.                       for _, label, color in zip_longest(xy_values, labels, colors)]
  247.         self.points = [ax.plot([], [], [],  color=color, lw=2, marker='o')[0]
  248.                        for _,color in zip_longest(xy_values, colors)]
  249.         # print(self.lines)
  250.         super(Visualization3D, self).__init__(fig, self.animate, init_func=self.init_animation, frames=frames,
  251.                                               interval=interval, blit=blit, **kwargs)
  252.  
  253.     def init_animation(self):
  254.         # 数值初始化
  255.         for line in self.lines:
  256.             line.set_data_3d([], [], [])
  257.         for point in self.points:
  258.             point.set_data_3d([], [], [])
  259.         return self.points
  260.  
  261.     def animate(self, i):
  262.         # 将x,y,z三个数据传入，绘制三维图像
  263.         for line, xy_value, z_value in zip(self.lines, self.xy_values, self.z_values):
  264.             line.set_data_3d(xy_value[:i, 0], xy_value[:i, 1], z_value[:i])
  265.         for point, xy_value, z_value in zip(self.points, self.xy_values, self.z_values):
  266.             point.set_data_3d(xy_value[i, 0], xy_value[i, 1], z_value[i])
  267.         return self.points
  268.  
  269.  
  270. def train_f(model, optimizer, x_init, epoch):
  271.     x = x_init
  272.     all_x = []
  273.     losses = []
  274.     for i in range(epoch):
  275.         all_x.append(copy.deepcopy(x.numpy()))  # 浅拷贝 改为 深拷贝, 否则List的原值会被改变。 Edit by David 2022.12.4.
  276.         loss = model(x)
  277.         losses.append(loss)
  278.         model.backward()
  279.         optimizer.step()
  280.         x = model.params['x']
  281.     return torch.Tensor(np.array(all_x)), losses
  282.  
  283.  
  284. # 构建5个模型，分别配备不同的优化器
  285. model1 = OptimizedFunction3D()
  286. opt_gd = SimpleBatchGD(init_lr=0.05, model=model1)
  287.  
  288. model2 = OptimizedFunction3D()
  289. opt_adagrad = Adagrad(init_lr=0.05, model=model2, epsilon=1e-7)
  290.  
  291. model3 = OptimizedFunction3D()
  292. opt_rmsprop = RMSprop(init_lr=0.05, model=model3, beta=0.9, epsilon=1e-7)
  293.  
  294. model4 = OptimizedFunction3D()
  295. opt_momentum = Momentum(init_lr=0.05, model=model4, rho=0.9)
  296.  
  297. model5 = OptimizedFunction3D()
  298. opt_adam = Adam(init_lr=0.05, model=model5, beta1=0.9, beta2=0.99, epsilon=1e-7)
  299.  
  300. models = [model5, model2, model3, model4, model1]
  301. opts = [opt_adam, opt_adagrad, opt_rmsprop, opt_momentum, opt_gd]
  302.  
  303. x_all_opts = []
  304. z_all_opts = []
  305.  
  306. # 使用不同优化器训练
  307.  
  308. for model, opt in zip(models, opts):
  309.     x_init = torch.FloatTensor([0.00001, 0.5])
  310.     x_one_opt, z_one_opt = train_f(model, opt, x_init, 100)  # epoch
  311.     # 保存参数值
  312.     x_all_opts.append(x_one_opt.numpy())
  313.     z_all_opts.append(np.squeeze(z_one_opt))
  314.  
  315. # 使用numpy.meshgrid生成x1,x2矩阵，矩阵的每一行为[-3, 3]，以0.1为间隔的数值
  316. x1 = np.arange(-1, 2, 0.01)
  317. x2 = np.arange(-1, 1, 0.05)
  318. x1, x2 = np.meshgrid(x1, x2)
  319. init_x = torch.Tensor(np.array([x1, x2]))
  320.  
  321. model = OptimizedFunction3D()
  322.  
  323. # 绘制 f_3d函数 的 三维图像
  324. fig = plt.figure()
  325. ax = plt.axes(projection='3d')
  326. X = init_x[0].numpy()
  327. Y = init_x[1].numpy()
  328. Z = model(init_x).numpy()  # 改为 model(init_x).numpy() David 2022.12.4
  329. surf = ax.plot_surface(X, Y, Z, edgecolor='grey', cmap=cm.coolwarm)
  330. # fig.colorbar(surf, shrink=0.5, aspect=1)
  331. ax.set_zlim(-3, 2)
  332. ax.set_xlabel('x1')
  333. ax.set_ylabel('x2')
  334. ax.set_zlabel('f(x1,x2)')
  335.  
  336. labels = ['Adam', 'AdaGrad', 'RMSprop', 'Momentum', 'SGD']
  337. colors = ['#8B0000', '#0000FF', '#000000', '#008B00', '#FF0000']
  338.  
  339. animator = Visualization3D(*x_all_opts, z_values=z_all_opts, labels=labels, colors=colors, fig=fig, ax=ax)
  340. ax.legend(loc='upper right')
  341.  
  342. plt.show()
  343. # animator.save('animation.gif') # 效果不好，估计被挡住了…… 有待进一步提高 Edit by David 2022.12.4

第三版 源代码：

  1. import torch
  2. import numpy as np
  3. import copy
  4. from matplotlib import pyplot as plt
  5. from matplotlib import animation
  6. from itertools import zip_longest
  7. from matplotlib import cm
  8.  
  9.  
  10. class Op(object):
  11.     def __init__(self):
  12.         pass
  13.  
  14.     def __call__(self, inputs):
  15.         return self.forward(inputs)
  16.  
  17.     # 输入：张量inputs
  18.     # 输出：张量outputs
  19.     def forward(self, inputs):
  20.         # return outputs
  21.         raise NotImplementedError
  22.  
  23.     # 输入：最终输出对outputs的梯度outputs_grads
  24.     # 输出：最终输出对inputs的梯度inputs_grads
  25.     def backward(self, outputs_grads):
  26.         # return inputs_grads
  27.         raise NotImplementedError
  28.  
  29.  
  30. class Optimizer(object):  # 优化器基类
  31.     def __init__(self, init_lr, model):
  32.         """
  33.         优化器类初始化
  34.         """
  35.         # 初始化学习率，用于参数更新的计算
  36.         self.init_lr = init_lr
  37.         # 指定优化器需要优化的模型
  38.         self.model = model
  39.  
  40.     def step(self):
  41.         """
  42.         定义每次迭代如何更新参数
  43.         """
  44.         pass
  45.  
  46.  
  47. class SimpleBatchGD(Optimizer):
  48.     def __init__(self, init_lr, model):
  49.         super(SimpleBatchGD, self).__init__(init_lr=init_lr, model=model)
  50.  
  51.     def step(self):
  52.         # 参数更新
  53.         if isinstance(self.model.params, dict):
  54.             for key in self.model.params.keys():
  55.                 self.model.params[key] = self.model.params[key] - self.init_lr * self.model.grads[key]
  56.  
  57.  
  58. class Adagrad(Optimizer):
  59.     def __init__(self, init_lr, model, epsilon):
  60.         """
  61.         Adagrad 优化器初始化
  62.         输入：
  63.             - init_lr： 初始学习率 - model：模型，model.params存储模型参数值  - epsilon：保持数值稳定性而设置的非常小的常数
  64.         """
  65.         super(Adagrad, self).__init__(init_lr=init_lr, model=model)
  66.         self.G = {}
  67.         for key in self.model.params.keys():
  68.             self.G[key] = 0
  69.         self.epsilon = epsilon
  70.  
  71.     def adagrad(self, x, gradient_x, G, init_lr):
  72.         """
  73.         adagrad算法更新参数，G为参数梯度平方的累计值。
  74.         """
  75.         G += gradient_x ** 2
  76.         x -= init_lr / torch.sqrt(G + self.epsilon) * gradient_x
  77.         return x, G
  78.  
  79.     def step(self):
  80.         """
  81.         参数更新
  82.         """
  83.         for key in self.model.params.keys():
  84.             self.model.params[key], self.G[key] = self.adagrad(self.model.params[key],
  85.                                                                self.model.grads[key],
  86.                                                                self.G[key],
  87.                                                                self.init_lr)
  88.  
  89.  
  90. class RMSprop(Optimizer):
  91.     def __init__(self, init_lr, model, beta, epsilon):
  92.         """
  93.         RMSprop优化器初始化
  94.         输入：
  95.             - init_lr：初始学习率
  96.             - model：模型，model.params存储模型参数值
  97.             - beta：衰减率
  98.             - epsilon：保持数值稳定性而设置的常数
  99.         """
  100.         super(RMSprop, self).__init__(init_lr=init_lr, model=model)
  101.         self.G = {}
  102.         for key in self.model.params.keys():
  103.             self.G[key] = 0
  104.         self.beta = beta
  105.         self.epsilon = epsilon
  106.  
  107.     def rmsprop(self, x, gradient_x, G, init_lr):
  108.         """
  109.         rmsprop算法更新参数，G为迭代梯度平方的加权移动平均
  110.         """
  111.         G = self.beta * G + (1 - self.beta) * gradient_x ** 2
  112.         x -= init_lr / torch.sqrt(G + self.epsilon) * gradient_x
  113.         return x, G
  114.  
  115.     def step(self):
  116.         """参数更新"""
  117.         for key in self.model.params.keys():
  118.             self.model.params[key], self.G[key] = self.rmsprop(self.model.params[key],
  119.                                                                self.model.grads[key],
  120.                                                                self.G[key],
  121.                                                                self.init_lr)
  122.  
  123.  
  124. class Momentum(Optimizer):
  125.     def __init__(self, init_lr, model, rho):
  126.         """
  127.         Momentum优化器初始化
  128.         输入：
  129.             - init_lr：初始学习率
  130.             - model：模型，model.params存储模型参数值
  131.             - rho：动量因子
  132.         """
  133.         super(Momentum, self).__init__(init_lr=init_lr, model=model)
  134.         self.delta_x = {}
  135.         for key in self.model.params.keys():
  136.             self.delta_x[key] = 0
  137.         self.rho = rho
  138.  
  139.     def momentum(self, x, gradient_x, delta_x, init_lr):
  140.         """
  141.         momentum算法更新参数，delta_x为梯度的加权移动平均
  142.         """
  143.         delta_x = self.rho * delta_x - init_lr * gradient_x
  144.         x += delta_x
  145.         return x, delta_x
  146.  
  147.     def step(self):
  148.         """参数更新"""
  149.         for key in self.model.params.keys():
  150.             self.model.params[key], self.delta_x[key] = self.momentum(self.model.params[key],
  151.                                                                       self.model.grads[key],
  152.                                                                       self.delta_x[key],
  153.                                                                       self.init_lr)
  154.  
  155.  
  156. class Adam(Optimizer):
  157.     def __init__(self, init_lr, model, beta1, beta2, epsilon):
  158.         """
  159.         Adam优化器初始化
  160.         输入：
  161.             - init_lr：初始学习率
  162.             - model：模型，model.params存储模型参数值
  163.             - beta1, beta2：移动平均的衰减率
  164.             - epsilon：保持数值稳定性而设置的常数
  165.         """
  166.         super(Adam, self).__init__(init_lr=init_lr, model=model)
  167.         self.beta1 = beta1
  168.         self.beta2 = beta2
  169.         self.epsilon = epsilon
  170.         self.M, self.G = {}, {}
  171.         for key in self.model.params.keys():
  172.             self.M[key] = 0
  173.             self.G[key] = 0
  174.         self.t = 1
  175.  
  176.     def adam(self, x, gradient_x, G, M, t, init_lr):
  177.         """
  178.         adam算法更新参数
  179.         输入：
  180.             - x：参数
  181.             - G：梯度平方的加权移动平均
  182.             - M：梯度的加权移动平均
  183.             - t：迭代次数
  184.             - init_lr：初始学习率
  185.         """
  186.         M = self.beta1 * M + (1 - self.beta1) * gradient_x
  187.         G = self.beta2 * G + (1 - self.beta2) * gradient_x ** 2
  188.         M_hat = M / (1 - self.beta1 ** t)
  189.         G_hat = G / (1 - self.beta2 ** t)
  190.         t += 1
  191.         x -= init_lr / torch.sqrt(G_hat + self.epsilon) * M_hat
  192.         return x, G, M, t
  193.  
  194.     def step(self):
  195.         """参数更新"""
  196.         for key in self.model.params.keys():
  197.             self.model.params[key], self.G[key], self.M[key], self.t = self.adam(self.model.params[key],
  198.                                                                                  self.model.grads[key],
  199.                                                                                  self.G[key],
  200.                                                                                  self.M[key],
  201.                                                                                  self.t,
  202.                                                                                  self.init_lr)
  203.  
  204.  
  205. class OptimizedFunction3D(Op):
  206.     def __init__(self):
  207.         super(OptimizedFunction3D, self).__init__()
  208.         self.params = {'x': 0}
  209.         self.grads = {'x': 0}
  210.  
  211.     def forward(self, x):
  212.         self.params['x'] = x
  213.         return - x[0] * x[0] / 2 + x[1] * x[1] / 1  # x[0] ** 2 + x[1] ** 2 + x[1] ** 3 + x[0] * x[1]
  214.  
  215.     def backward(self):
  216.         x = self.params['x']
  217.         gradient1 = - 2 * x[0] / 2
  218.         gradient2 = 2 * x[1] / 1
  219.         grad1 = torch.Tensor([gradient1])
  220.         grad2 = torch.Tensor([gradient2])
  221.         self.grads['x'] = torch.cat([grad1, grad2])
  222.  
  223.  
  224. class Visualization3D(animation.FuncAnimation):
  225.     """    绘制动态图像，可视化参数更新轨迹    """
  226.  
  227.     def __init__(self, *xy_values, z_values, labels=[], colors=[], fig, ax, interval=100, blit=True, **kwargs):
  228.         """
  229.         初始化3d可视化类
  230.         输入：
  231.             xy_values：三维中x,y维度的值
  232.             z_values：三维中z维度的值
  233.             labels：每个参数更新轨迹的标签
  234.             colors：每个轨迹的颜色
  235.             interval：帧之间的延迟（以毫秒为单位）
  236.             blit：是否优化绘图
  237.         """
  238.         self.fig = fig
  239.         self.ax = ax
  240.         self.xy_values = xy_values
  241.         self.z_values = z_values
  242.  
  243.         frames = max(xy_value.shape[0] for xy_value in xy_values)
  244.  
  245.         self.lines = [ax.plot([], [], [], label=label, color=color, lw=2)[0]
  246.                       for _, label, color in zip_longest(xy_values, labels, colors)]
  247.         self.points = [ax.plot([], [], [],  color=color, markeredgewidth =1, markeredgecolor='black', marker='o')[0]
  248.                        for _,color in zip_longest(xy_values, colors)]
  249.         # print(self.lines)
  250.         super(Visualization3D, self).__init__(fig, self.animate, init_func=self.init_animation, frames=frames,
  251.                                               interval=interval, blit=blit, **kwargs)
  252.  
  253.     def init_animation(self):
  254.         # 数值初始化
  255.         for line in self.lines:
  256.             line.set_data_3d([], [], [])
  257.         for point in self.points:
  258.             point.set_data_3d([], [], [])
  259.         return self.points + self.lines
  260.  
  261.     def animate(self, i):
  262.         # 将x,y,z三个数据传入，绘制三维图像
  263.         for line, xy_value, z_value in zip(self.lines, self.xy_values, self.z_values):
  264.             line.set_data_3d(xy_value[:i, 0], xy_value[:i, 1], z_value[:i])
  265.         for point, xy_value, z_value in zip(self.points, self.xy_values, self.z_values):
  266.             point.set_data_3d(xy_value[i, 0], xy_value[i, 1], z_value[i])
  267.         return self.points + self.lines
  268.  
  269.  
  270. def train_f(model, optimizer, x_init, epoch):
  271.     x = x_init
  272.     all_x = []
  273.     losses = []
  274.     for i in range(epoch):
  275.         all_x.append(copy.deepcopy(x.numpy()))  # 浅拷贝 改为 深拷贝, 否则List的原值会被改变。 Edit by David 2022.12.4.
  276.         loss = model(x)
  277.         losses.append(loss)
  278.         model.backward()
  279.         optimizer.step()
  280.         x = model.params['x']
  281.     return torch.Tensor(np.array(all_x)), losses
  282.  
  283.  
  284. # 构建5个模型，分别配备不同的优化器
  285. model1 = OptimizedFunction3D()
  286. opt_gd = SimpleBatchGD(init_lr=0.05, model=model1)
  287.  
  288. model2 = OptimizedFunction3D()
  289. opt_adagrad = Adagrad(init_lr=0.05, model=model2, epsilon=1e-7)
  290.  
  291. model3 = OptimizedFunction3D()
  292. opt_rmsprop = RMSprop(init_lr=0.05, model=model3, beta=0.9, epsilon=1e-7)
  293.  
  294. model4 = OptimizedFunction3D()
  295. opt_momentum = Momentum(init_lr=0.05, model=model4, rho=0.9)
  296.  
  297. model5 = OptimizedFunction3D()
  298. opt_adam = Adam(init_lr=0.05, model=model5, beta1=0.9, beta2=0.99, epsilon=1e-7)
  299.  
  300. models = [model5, model2, model3, model4, model1]
  301. opts = [opt_adam, opt_adagrad, opt_rmsprop, opt_momentum, opt_gd]
  302.  
  303. x_all_opts = []
  304. z_all_opts = []
  305.  
  306. # 使用不同优化器训练
  307.  
  308. for model, opt in zip(models, opts):
  309.     x_init = torch.FloatTensor([0.00001, 0.5])
  310.     x_one_opt, z_one_opt = train_f(model, opt, x_init, 100)  # epoch
  311.     # 保存参数值
  312.     x_all_opts.append(x_one_opt.numpy())
  313.     z_all_opts.append(np.squeeze(z_one_opt))
  314.  
  315. # 使用numpy.meshgrid生成x1,x2矩阵，矩阵的每一行为[-3, 3]，以0.1为间隔的数值
  316. x1 = np.arange(-1, 2, 0.01)
  317. x2 = np.arange(-1, 1, 0.05)
  318. x1, x2 = np.meshgrid(x1, x2)
  319. init_x = torch.Tensor(np.array([x1, x2]))
  320.  
  321. model = OptimizedFunction3D()
  322.  
  323. # 绘制 f_3d函数 的 三维图像
  324. fig = plt.figure()
  325. ax = plt.axes(projection='3d')
  326. X = init_x[0].numpy()
  327. Y = init_x[1].numpy()
  328. Z = model(init_x).numpy()  # 改为 model(init_x).numpy() David 2022.12.4
  329. surf = ax.plot_surface(X, Y, Z, edgecolor='grey', cmap=cm.coolwarm)
  330. # fig.colorbar(surf, shrink=0.5, aspect=1)
  331. ax.set_zlim(-3, 2)
  332. ax.set_xlabel('x1')
  333. ax.set_ylabel('x2')
  334. ax.set_zlabel('f(x1,x2)')
  335.  
  336. labels = ['Adam', 'AdaGrad', 'RMSprop', 'Momentum', 'SGD']
  337. colors = ['#8B0000', '#0000FF', '#000000', '#008B00', '#FF0000']
  338.  
  339. animator = Visualization3D(*x_all_opts, z_values=z_all_opts, labels=labels, colors=colors, fig=fig, ax=ax)
  340. ax.legend(loc='upper right')
  341.  
  342. plt.show()
  343. # animator.save('teaser' + '.gif', writer='imagemagick',fps=10) # 效果不好，估计被挡住了…… 有待进一步提高 Edit by David 2022.12.4
  344. # save不好用，不费劲了，安装个软件做gif https://pc.qq.com/detail/13/detail_23913.html

注：本文转载自blog.csdn.net的HBU_David的文章"https://blog.csdn.net/qq_38975453/article/details/128192767"。版权归原作者所有，此博客不拥有其著作权，亦不承担相应法律责任。如有侵权，请联系我们删除。
复制链接
相关推荐
发表评论

评论记录：

未查询到任何数据！