最近为了对比神经网络在不同语言中的运行效率，于是手搓了一个简单模型在不同语言中的代码，在这儿记录一下。
基于MINIST手写数字数据集，采用3层全连接网络，由Pytorch、Python、C++、CUDA四个版本实现。
- 实验结果
- Pytorch版
- Python版
2.1.Numpy版
2.2.纯Python版 - C++版
- CUDA版
- Python版本的实现主要基于Numpy
- 代码及相关算法原理和公式参照《深度学习入门 基于Python的理论与实现》，百度云盘链接：https://pan.baidu.com/s/1YNks6c4QsG9z-3BDc_QY1Q?pwd=uzpq 提取码：uzpq。这本书写的很详细，代码中有看不懂的地方可以参照该书。
- 完整代码，Github：
- 参考：https://github.com/hamlinzheng/mnist
# coding: utf-8
try:
import urllib.request
except ImportError:
raise ImportError('You should use Python 3.x')
import os.path
import gzip
import pickle
import os
import numpy as np
url_base = 'http://yann.lecun.com/exdb/mnist/'
key_file = {
'train_img': 'train-images-idx3-ubyte.gz',
'train_label': 'train-labels-idx1-ubyte.gz',
'test_img': 't10k-images-idx3-ubyte.gz',
'test_label': 't10k-labels-idx1-ubyte.gz'
}
dataset_dir = os.path.dirname(os.path.abspath(__file__))
save_file = dataset_dir + "/mnist.pkl"
train_num = 60000
test_num = 10000
img_dim = (1, 28, 28)
img_size = 784
def _download(file_name):
file_path = dataset_dir + "/" + file_name
if os.path.exists(file_path):
return
print("Downloading " + file_name + " ... ")
urllib.request.urlretrieve(url_base + file_name, file_path)
print("Done")
def download_mnist():
for v in key_file.values():
_download(v)
def _load_label(file_name):
file_path = dataset_dir + "/" + file_name
print("Converting " + file_name + " to NumPy Array ...")
with gzip.open(file_path, 'rb') as f:
labels = np.frombuffer(f.read(), np.uint8, offset=8)
print("Done")
return labels
def _load_img(file_name):
file_path = dataset_dir + "/" + file_name
print("Converting " + file_name + " to NumPy Array ...")
with gzip.open(file_path, 'rb') as f:
data = np.frombuffer(f.read(), np.uint8, offset=16)
data = data.reshape(-1, img_size)
print("Done")
return data
def _convert_numpy():
dataset = {}
dataset['train_img'] = _load_img(key_file['train_img'])
dataset['train_label'] = _load_label(key_file['train_label'])
dataset['test_img'] = _load_img(key_file['test_img'])
dataset['test_label'] = _load_label(key_file['test_label'])
return dataset
def init_mnist():
'''
Note：已将数据集下载至本地，第一次加载会将数据集保存成pickle
'''
# download_mnist()
dataset = _convert_numpy()
print("Creating pickle file ...")
with open(save_file, 'wb') as f:
pickle.dump(dataset, f, -1)
print("Done!")
def _change_one_hot_label(X):
T = np.zeros((X.size, 10))
for idx, row in enumerate(T):
row[X[idx]] = 1
return T
def load_mnist(normalize=True, flatten=True, one_hot_label=False):
"""读入MNIST数据集
Parameters
----------
normalize : 将图像的像素值正规化为0.0~1.0
one_hot_label :
one_hot_label为True的情况下，标签作为one-hot数组返回
one-hot数组是指[0,0,1,0,0,0,0,0,0,0]这样的数组
flatten : 是否将图像展开为一维数组
Returns
-------
(训练图像, 训练标签), (测试图像, 测试标签)
"""
if not os.path.exists(save_file):
init_mnist()
with open(save_file, 'rb') as f:
dataset = pickle.load(f)
if normalize:
for key in ('train_img', 'test_img'):
dataset[key] = dataset[key].astype(np.float32)
dataset[key] /= 255.0
if one_hot_label:
dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
dataset['test_label'] = _change_one_hot_label(dataset['test_label'])
if not flatten:
for key in ('train_img', 'test_img'):
dataset[key] = dataset[key].reshape(-1, 1, 28, 28)
print('训练图片数量：', dataset['train_img'].shape[0], '测试图片数量：', dataset['test_img'].shape[0])
return (dataset['train_img'], dataset['train_label']), (dataset['test_img'], dataset['test_label'])
if __name__ == '__main__':
# init_mnist()
data = load_mnist(normalize=True, flatten=True, one_hot_label=True)
print(data[0][0].shape)
layers.py
import numpy as np
class FC:
def __init__(self,w,b):
self.w=w
self.b=b
self.x=None
self.dw=None
self.db=None
def forward(self,x):
self.original_x_shape = x.shape
x = x.reshape(x.shape[0], -1)
self.x=x
out=np.dot(x,self.w)+self.b
return out
def backward(self,dout):
dx=np.dot(dout,self.w.T)
self.dw=np.dot(self.x.T,dout)
self.db=np.sum(dout,axis=0)
dx = dx.reshape(*self.original_x_shape)
return dx
class Relu:
def __init__(self):
self.mask=None
def forward(self,x):
self.mask=(x<=0)
out=x.copy()
out[self.mask]=0
return out
def backward(self,dout):
dout[self.mask]=0
dx=dout
return dx
class SoftmaxWithLoss:
def __init__(self):
self.y=None
self.loss=None
self.t=None
def softmax(self,x):
if x.ndim == 2:
x = x.T
x = x - np.max(x, axis=0)
y = np.exp(x) / np.sum(np.exp(x), axis=0)
return y.T
x = x - np.max(x) # 溢出对策
return np.exp(x) / np.sum(np.exp(x))
def CrossEntropyError(self,y,t):
'''
:param y:神经网络的输出
:param t:标签，onehot表示
:return:注意输出都是正数
'''
if y.ndim == 1:
t = t.reshape(1, t.size)
y = y.reshape(1, y.size)
# 监督数据是one-hot-vector的情况下，转换为正确解标签的索引
if t.size == y.size:
t = t.argmax(axis=1)
batch_size = y.shape[0]
return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size
def forward(self,x,t):
'''
:param x: input
:param t: 标签，onehot表示
:return:
'''
self.t=t
self.y=self.softmax(x)
self.loss=self.CrossEntropyError(self.y,self.t)
return self.loss
def backward(self,dout=1):
batch_size = self.t.shape[0]
if self.t.size == self.y.size: # 监督数据是one-hot-vector的情况
dx = (self.y - self.t) / batch_size
else:
dx = self.y.copy()
dx[np.arange(batch_size), self.t] -= 1
dx = dx / batch_size
return dx
main.py
import random
import numpy as np
from dataset.dataloader import *
from layers import *
import time,cv2
class model:
def __init__(self,size_list=(784,1024,1024,10)):
self.size_list=size_list
np.random.seed(123)
self.params = {}
self.params['w1']=0.01*np.random.randn(self.size_list[0],self.size_list[1])
self.params['b1']=np.zeros(self.size_list[1])
self.params['w2']=0.01*np.random.randn(self.size_list[1],self.size_list[2])
self.params['b2']=np.zeros(size_list[2])
self.params['w3']=0.01*np.random.randn(self.size_list[2],self.size_list[3])
self.params['b3']=np.zeros(self.size_list[3])
self.layer1=FC(self.params['w1'],self.params['b1'])
self.layer1_act=Relu()
self.layer2=FC(self.params['w2'],self.params['b2'])
self.layer2_act=Relu()
self.layer3=FC(self.params['w3'],self.params['b3'])
self.layer4=SoftmaxWithLoss()
def forward(self,x):
x = self.layer1.forward(x)
x = self.layer1_act.forward(x)
x = self.layer2.forward(x)
x = self.layer2_act.forward(x)
x = self.layer3.forward(x)
return x
def loss(self,x,t):
'''
:param x:forward输出的x
:param t: teacher标签（onehot）
:return:
'''
loss_=self.layer4.forward(x,t)
return loss_
def gradient(self,x,t):
#前向部分
x=self.forward(x)
loss_=self.loss(x,t)
#反向部分
# dout=1
dout=self.layer4.backward()
dout=self.layer3.backward(dout)
dout=self.layer2_act.backward(dout)
dout=self.layer2.backward(dout)
dout=self.layer1_act.backward(dout)
dout=self.layer1.backward(dout)
grads_={} # 用于存放梯度的字典
grads_['w1'],grads_['b1']=self.layer1.dw,self.layer1.db
grads_['w2'],grads_['b2']=self.layer2.dw,self.layer2.db
grads_['w3'],grads_['b3']=self.layer3.dw,self.layer3.db
return grads_,loss_
def SGD(params,grads):
for key in params.keys():
params[key] -= 0.01 * grads[key]
DNN=model((784,1024,1024,10))
def train(epochs=10,batch_size=1):
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, flatten=True, one_hot_label=True)
train_size=x_train.shape[0]
test_size=x_test.shape[0]
loss_list=[]
# optimizer=SG
for epoch in range(epochs):
t1=time.time()
for i in range(int(train_size/batch_size)):#一个epoch
batch_mask = np.random.choice(train_size, batch_size) # 从0到self.train_size中随机抽取出self.batch_size个数
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
grads,loss_=DNN.gradient(x_batch,t_batch)#gradient中包含了前向传播部分
SGD(DNN.params,grads)
loss_list.append(loss_)
if i%500==0:
loss_avg=sum(loss_list)/len(loss_list)
print('Epoch:',epoch,'Iter:',i,'loss_avg:',loss_avg)
loss_list=[]
t2=time.time()
print("---Time:",t2-t1)
#一个Epoch结束后执行一次测试
acc=0.0
for i_t in range(0,test_size,batch_size):
x_batch_t=x_test[i_t:i_t+batch_size]
t_batch_t=t_test[i_t:i_t+batch_size]
t_batch_t = np.argmax(t_batch_t, axis=1)#将onehot转换回数值
y_t = DNN.forward(x_batch_t)
y_t=np.argmax(y_t,axis=1)
acc+=np.sum(y_t==t_batch_t)
print("---Test Finished, ACC=",acc/test_size)
if epoch==6: #test
img=cv2.imread('D:\v3\dataset\testData\0.jpg',0)
img=img.flatten()
img=img.astype(np.float32)
img=img/255.0
img=np.expand_dims(img, axis=0)
y_c = DNN.forward(img)
print(y_c.shape,'--')
y_c=np.argmax(y_c,axis=1)
print('测试，预测值',y_c)
if __name__ == '__main__':
train()
