NNDL 作业11：优化算法比较

 

目录

1. 编程实现图6-1，并观察特征

2. 观察梯度方向

3. 编写代码实现算法，并可视化轨迹

4. 分析上图，说明原理（选做）

5. 总结SGD、Momentum、AdaGrad、Adam的优缺点（选做）

6. Adam这么好，SGD是不是就用不到了？（选做）

7. 增加RMSprop、Nesterov算法。（选做）

8. 基于MNIST数据集的更新方法的比较（选做）

参考：深度学习入门：基于Python的理论与实现 (ituring.com.cn)

1. 编程实现图6-1，并观察特征

 

 

 参考代码：

  1. import numpy as np
  2. from matplotlib import pyplot as plt
  3. from mpl_toolkits.mplot3d import Axes3D
  4.  
  5.  
  6. # https://blog.csdn.net/weixin_39228381/article/details/108511882
  7.  
  8. def func(x, y):
  9.     return x * x / 20 + y * y
  10.  
  11.  
  12. def paint_loss_func():
  13.     x = np.linspace(-50, 50, 100)  # x的绘制范围是-50到50，从改区间均匀取100个数
  14.     y = np.linspace(-50, 50, 100)  # y的绘制范围是-50到50，从改区间均匀取100个数
  15.  
  16.     X, Y = np.meshgrid(x, y)
  17.     Z = func(X, Y)
  18.  
  19.     fig = plt.figure()  # figsize=(10, 10))
  20.     ax = Axes3D(fig)
  21.     plt.xlabel('x')
  22.     plt.ylabel('y')
  23.  
  24.     ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap='rainbow')
  25.     plt.show()
  26.  
  27.  
  28. paint_loss_func()

 2. 观察梯度方向

3. 编写代码实现算法，并可视化轨迹

SGD、Momentum、Adagrad、Adam

参考代码：

  1. # coding: utf-8
  2. import numpy as np
  3. import matplotlib.pyplot as plt
  4. from collections import OrderedDict
  5.  
  6.  
  7. class SGD:
  8.     """随机梯度下降法（Stochastic Gradient Descent）"""
  9.  
  10.     def __init__(self, lr=0.01):
  11.         self.lr = lr
  12.  
  13.     def update(self, params, grads):
  14.         for key in params.keys():
  15.             params[key] -= self.lr * grads[key]
  16.  
  17.  
  18. class Momentum:
  19.     """Momentum SGD"""
  20.  
  21.     def __init__(self, lr=0.01, momentum=0.9):
  22.         self.lr = lr
  23.         self.momentum = momentum
  24.         self.v = None
  25.  
  26.     def update(self, params, grads):
  27.         if self.v is None:
  28.             self.v = {}
  29.             for key, val in params.items():
  30.                 self.v[key] = np.zeros_like(val)
  31.  
  32.         for key in params.keys():
  33.             self.v[key] = self.momentum * self.v[key] - self.lr * grads[key]
  34.             params[key] += self.v[key]
  35.  
  36.  
  37. class Nesterov:
  38.     """Nesterov's Accelerated Gradient (http://arxiv.org/abs/1212.0901)"""
  39.  
  40.     def __init__(self, lr=0.01, momentum=0.9):
  41.         self.lr = lr
  42.         self.momentum = momentum
  43.         self.v = None
  44.  
  45.     def update(self, params, grads):
  46.         if self.v is None:
  47.             self.v = {}
  48.             for key, val in params.items():
  49.                 self.v[key] = np.zeros_like(val)
  50.  
  51.         for key in params.keys():
  52.             self.v[key] *= self.momentum
  53.             self.v[key] -= self.lr * grads[key]
  54.             params[key] += self.momentum * self.momentum * self.v[key]
  55.             params[key] -= (1 + self.momentum) * self.lr * grads[key]
  56.  
  57.  
  58. class AdaGrad:
  59.     """AdaGrad"""
  60.  
  61.     def __init__(self, lr=0.01):
  62.         self.lr = lr
  63.         self.h = None
  64.  
  65.     def update(self, params, grads):
  66.         if self.h is None:
  67.             self.h = {}
  68.             for key, val in params.items():
  69.                 self.h[key] = np.zeros_like(val)
  70.  
  71.         for key in params.keys():
  72.             self.h[key] += grads[key] * grads[key]
  73.             params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)
  74.  
  75.  
  76. class RMSprop:
  77.     """RMSprop"""
  78.  
  79.     def __init__(self, lr=0.01, decay_rate=0.99):
  80.         self.lr = lr
  81.         self.decay_rate = decay_rate
  82.         self.h = None
  83.  
  84.     def update(self, params, grads):
  85.         if self.h is None:
  86.             self.h = {}
  87.             for key, val in params.items():
  88.                 self.h[key] = np.zeros_like(val)
  89.  
  90.         for key in params.keys():
  91.             self.h[key] *= self.decay_rate
  92.             self.h[key] += (1 - self.decay_rate) * grads[key] * grads[key]
  93.             params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)
  94.  
  95.  
  96. class Adam:
  97.     """Adam (http://arxiv.org/abs/1412.6980v8)"""
  98.  
  99.     def __init__(self, lr=0.001, beta1=0.9, beta2=0.999):
  100.         self.lr = lr
  101.         self.beta1 = beta1
  102.         self.beta2 = beta2
  103.         self.iter = 0
  104.         self.m = None
  105.         self.v = None
  106.  
  107.     def update(self, params, grads):
  108.         if self.m is None:
  109.             self.m, self.v = {}, {}
  110.             for key, val in params.items():
  111.                 self.m[key] = np.zeros_like(val)
  112.                 self.v[key] = np.zeros_like(val)
  113.  
  114.         self.iter += 1
  115.         lr_t = self.lr * np.sqrt(1.0 - self.beta2 ** self.iter) / (1.0 - self.beta1 ** self.iter)
  116.  
  117.         for key in params.keys():
  118.             self.m[key] += (1 - self.beta1) * (grads[key] - self.m[key])
  119.             self.v[key] += (1 - self.beta2) * (grads[key] ** 2 - self.v[key])
  120.  
  121.             params[key] -= lr_t * self.m[key] / (np.sqrt(self.v[key]) + 1e-7)
  122.  
  123.  
  124. def f(x, y):
  125.     return x ** 2 / 20.0 + y ** 2
  126.  
  127.  
  128. def df(x, y):
  129.     return x / 10.0, 2.0 * y
  130.  
  131.  
  132. init_pos = (-7.0, 2.0)
  133. params = {}
  134. params['x'], params['y'] = init_pos[0], init_pos[1]
  135. grads = {}
  136. grads['x'], grads['y'] = 0, 0
  137.  
  138. optimizers = OrderedDict()
  139. optimizers["SGD"] = SGD(lr=0.95)
  140. optimizers["Momentum"] = Momentum(lr=0.1)
  141. optimizers["AdaGrad"] = AdaGrad(lr=1.5)
  142. optimizers["Adam"] = Adam(lr=0.3)
  143.  
  144. idx = 1
  145.  
  146. for key in optimizers:
  147.     optimizer = optimizers[key]
  148.     x_history = []
  149.     y_history = []
  150.     params['x'], params['y'] = init_pos[0], init_pos[1]
  151.  
  152.     for i in range(30):
  153.         x_history.append(params['x'])
  154.         y_history.append(params['y'])
  155.  
  156.         grads['x'], grads['y'] = df(params['x'], params['y'])
  157.         optimizer.update(params, grads)
  158.  
  159.     x = np.arange(-10, 10, 0.01)
  160.     y = np.arange(-5, 5, 0.01)
  161.  
  162.     X, Y = np.meshgrid(x, y)
  163.     Z = f(X, Y)
  164.     # for simple contour line
  165.     mask = Z > 7
  166.     Z[mask] = 0
  167.  
  168.     # plot
  169.     plt.subplot(2, 2, idx)
  170.     idx += 1
  171.     plt.plot(x_history, y_history, 'o-', color="red")
  172.     plt.contour(X, Y, Z)  # 绘制等高线
  173.     plt.ylim(-10, 10)
  174.     plt.xlim(-10, 10)
  175.     plt.plot(0, 0, '+')
  176.     plt.title(key)
  177.     plt.xlabel("x")
  178.     plt.ylabel("y")
  179.  
  180. plt.subplots_adjust(wspace=0, hspace=0)  # 调整子图间距
  181. plt.show()

4. 分析上图，说明原理（选做）

  1. 为什么SGD会走“之字形”？其它算法为什么会比较平滑？
  2. Momentum、AdaGrad对SGD的改进体现在哪里？速度？方向？在图上有哪些体现？
  3. 仅从轨迹来看，Adam似乎不如AdaGrad效果好，是这样么？
  4. 四种方法分别用了多长时间？是否符合预期？
  5. 调整学习率、动量等超参数，轨迹有哪些变化？

5. 总结SGD、Momentum、AdaGrad、Adam的优缺点（选做）

6. Adam这么好，SGD是不是就用不到了？（选做）

7. 增加RMSprop、Nesterov算法。（选做）

对比Momentum与Nesterov、AdaGrad与RMSprop。

8. 基于MNIST数据集的更新方法的比较（选做）

在原图基础上，增加RMSprop、Nesterov算法。

编程实现，并谈谈自己的看法。

 优化算法代码可参考前面的内容。

 MNIST数据集的更新方法的比较：

  1. # coding: utf-8
  2. import os
  3. import sys
  4. sys.path.append(os.pardir)  # 为了导入父目录的文件而进行的设定
  5. import matplotlib.pyplot as plt
  6. from dataset.mnist import load_mnist
  7. from common.util import smooth_curve
  8. from common.multi_layer_net import MultiLayerNet
  9. from common.optimizer import *
  10.  
  11.  
  12. # 0:读入MNIST数据==========
  13. (x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)
  14.  
  15. train_size = x_train.shape[0]
  16. batch_size = 128
  17. max_iterations = 2000
  18.  
  19.  
  20. # 1:进行实验的设置==========
  21. optimizers = {}
  22. optimizers['SGD'] = SGD()
  23. optimizers['Momentum'] = Momentum()
  24. optimizers['AdaGrad'] = AdaGrad()
  25. optimizers['Adam'] = Adam()
  26. #optimizers['RMSprop'] = RMSprop()
  27.  
  28. networks = {}
  29. train_loss = {}
  30. for key in optimizers.keys():
  31.     networks[key] = MultiLayerNet(
  32.         input_size=784, hidden_size_list=[100, 100, 100, 100],
  33.         output_size=10)
  34.     train_loss[key] = []    
  35.  
  36.  
  37. # 2:开始训练==========
  38. for i in range(max_iterations):
  39.     batch_mask = np.random.choice(train_size, batch_size)
  40.     x_batch = x_train[batch_mask]
  41.     t_batch = t_train[batch_mask]
  42.     
  43.     for key in optimizers.keys():
  44.         grads = networks[key].gradient(x_batch, t_batch)
  45.         optimizers[key].update(networks[key].params, grads)
  46.     
  47.         loss = networks[key].loss(x_batch, t_batch)
  48.         train_loss[key].append(loss)
  49.     
  50.     if i % 100 == 0:
  51.         print( "===========" + "iteration:" + str(i) + "===========")
  52.         for key in optimizers.keys():
  53.             loss = networks[key].loss(x_batch, t_batch)
  54.             print(key + ":" + str(loss))
  55.  
  56.  
  57. # 3.绘制图形==========
  58. markers = {"SGD": "o", "Momentum": "x", "AdaGrad": "s", "Adam": "D"}
  59. x = np.arange(max_iterations)
  60. for key in optimizers.keys():
  61.     plt.plot(x, smooth_curve(train_loss[key]), marker=markers[key], markevery=100, label=key)
  62. plt.xlabel("iterations")
  63. plt.ylabel("loss")
  64. plt.ylim(0, 1)
  65. plt.legend()
  66. plt.show()

注：本文转载自blog.csdn.net的HBU_David的文章"https://blog.csdn.net/qq_38975453/article/details/128025693"。版权归原作者所有，此博客不拥有其著作权，亦不承担相应法律责任。如有侵权，请联系我们删除。
复制链接
相关推荐
发表评论

评论记录：

未查询到任何数据！