随机梯度下降，批量梯度下降，小批量梯度下降

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import warnings
warnings.filterwarnings('ignore')
# 设置字符集，防止中文乱码
mpl.rcParams['font.sans-serif'] = [u'simHei']
mpl.rcParams['axes.unicode_minus'] = False

EXAMPLE_NUM = 100#训练总数
BATCH_SIZE = 10#mini_batch训练集大小
TRAIN_STEP = 150#训练次数
LEARNING_RATE = 0.0001#学习率
X_INPUT = np.arange(EXAMPLE_NUM) * 0.1#生成输入数据X
Y_OUTPUT_CORRECT = 5 * X_INPUT#生成训练正确输出数据

##构造训练的函数
def train_func(X, K):
    result = K * X
    return result

SGD_k=0
SGD_ks=[]
for step in range(TRAIN_STEP):
    index=np.random.randint(len(X_INPUT))
    sum_sgd=(train_func(X_INPUT[index],SGD_k)-Y_OUTPUT_CORRECT[index])*X_INPUT[index]
    SGD_k-=LEARNING_RATE*sum_sgd
    SGD_ks.append(SGD_k)
print(SGD_ks[-1])

BGD_k=0
BGD_ks=[]
for step in range(TRAIN_STEP):
    sum_bgd=0
    for index in range(len(X_INPUT)):
        sum_bgd+=(train_func(X_INPUT[index],BGD_k)-Y_OUTPUT_CORRECT[index])*X_INPUT[index]
    # sum_bgd=sum_bgd/len(X_INPUT)
    BGD_k-=LEARNING_RATE*sum_bgd
    BGD_ks.append(BGD_k)
print(BGD_ks[-1])

MBGD_k=0
MBGD_ks=[]
for step in range(TRAIN_STEP):
    sum_mbgd=0
    index_start=np.random.randint(len(X_INPUT)-BATCH_SIZE)
    for index in np.arange(index_start,index_start+BATCH_SIZE):
        sum_mbgd+=(train_func(X_INPUT[index],MBGD_k)-Y_OUTPUT_CORRECT[index])*X_INPUT[index]
    # sum_mbgd=sum_mbgd/BATCH_SIZE
    MBGD_k-=LEARNING_RATE*sum_mbgd
    MBGD_ks.append(MBGD_k)
print(MBGD_ks[-1])

plt.plot(np.arange(TRAIN_STEP),np.array(SGD_ks),'r',label='SGD')
plt.plot(np.arange(TRAIN_STEP),np.array(BGD_ks),'g',label='BGD')
plt.plot(np.arange(TRAIN_STEP),np.array(MBGD_ks),'b',label='MBGD')
plt.xlabel('step')
plt.ylabel('K')
plt.legend(loc='best')
plt.show()