系统
本系统使用Python语言编写，基于pytorch框架开发，主要使用transformer作为系统模型，使用PYQT5做界面设计。
本系统基于image caption技术实现，看图说话（Image Caption）任务是结合CV和NLP两个领域的一种比较综合的任务，Image Caption模型的输入是一幅图像，输出是对该幅图像进行描述的一段文字。这项任务要求模型可以识别图片中的物体、理解物体间的关系，并用一句自然语言表达出来。
本系统主要基于CNN+Transformer的总体框架实现的，系统框架如图所示。
介绍
视障辅助系统是一个为视力障碍的人开发的一个系统。本系统可以帮助有视力障碍的人“听”到周围的环境，为用户实时讲述身边的环境，包括人、建筑物等，该功能通过识别与分析图像信息并进行语音播报，为视障用户提供从“听觉”到“视觉”的多模态交互方式，让盲人也能够“看”见周边的环境。
本系统可将视觉世界变成一种可听见的语音体验。用户只需要移动设备上的摄像头，在四周移动，并点击按钮，系统会截取当前摄像头画面，经过扫描识别图片信息并生成自然语言描述，再通过语音快速告知用户当前图片的具体内容，用户就可“听到”周围的环境的实际情况。这样一来，视障用户就可以顺利接收图片信息，在脑海中对图片内容有所想象，从而打开新“视”界。
上图为“基于孟加拉语的图像描述论文中模型图”
界面图
左边为摄像头画面，画面内容为口罩遮挡
不足;
系统不足：
系统的模型训练轮次不足，描述精度不够。
系统检测时间过长，响应慢。几乎30s
主代码：yuce.py
# here put the import lib
import sys
import pyttsx3
from PIL import Image
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QApplication, QMainWindow,QLabel
from PyQt5.QtCore import QTimer
from PyQt5.QtGui import QImage, QPixmap
from PyQt5.QtWidgets import QMessageBox
from shijian.gui import Ui_mainWindow
import numpy as np
import cv2
import time
from random import uniform
from PyQt5.Qt import *
from predict_ import predict
class Open_Camera(QtWidgets.QMainWindow, Ui_mainWindow):
def __init__(self):
super(Open_Camera, self).__init__()
self.setupUi(self) # 创建窗体对象
self.cap = cv2.VideoCapture() # 初始化摄像头
self.photo_flag = 0
self.label.setScaledContents(True) # 图片自适应
self.label_2.setScaledContents(True) # 图片自适应
self.init()
def init(self):
# 定时器让其定时读取显示图片
self.camera_timer = QTimer()
self.camera_timer.timeout.connect(self.show_image)
# 打开摄像头
self.show()
self.open_camera()
# 拍照
self.pushButton_3.clicked.connect(self.taking_pictures)
#self.show()
'''开启摄像头'''
def open_camera(self):
self.cap = cv2.VideoCapture(0) # 摄像头
self.camera_timer.start(40) # 每40毫秒读取一次，即刷新率为25帧
self.show_image()
'''显示图片'''
def show_image(self):
flag, self.image = self.cap.read() # 从视频流中读取图片
image_show = cv2.resize(self.image, (640, 480)) # 把读到的帧的大小重新设置为 600*360
# image_show = self.image
width, height = image_show.shape[:2] # 行:宽，列:高
image_show = cv2.cvtColor(image_show, cv2.COLOR_BGR2RGB) # opencv读的通道是BGR,要转成RGB
image_show = cv2.flip(image_show, 1) # 水平翻转，因为摄像头拍的是镜像的。
# 把读取到的视频数据变成QImage形式(图片数据、高、宽、RGB颜色空间，三个通道各有2**8=256种颜色)
self.showImage = QtGui.QImage(image_show.data, height, width, QImage.Format_RGB888)
self.label.setPixmap(QPixmap.fromImage(self.showImage)) # 往显示视频的Label里显示QImage
'''拍照'''
def taking_pictures(self):
if self.cap.isOpened():
FName = fr"images/cap{time.strftime('%Y%m%d%H%M%S', time.localtime())}"
print(FName)
self.label_2.setPixmap(QtGui.QPixmap.fromImage(self.showImage))
# self.showImage.save(FName + ".jpg", "JPG", 100)
save_picture = './images/1.jpg'
self.showImage.save(save_picture)
a = predict(save_picture)
self.textEdit.setText(a)
engine = pyttsx3.init()
rate = engine.getProperty('rate') # 获取当前语速属性的值
engine.setProperty('rate', rate - 80)
engine.say(a)
engine.runAndWait()
else:
QMessageBox.critical(self, '错误', '摄像头未打开！')
return None
def paintEvent(self, a0: QtGui.QPaintEvent) -> None:
painter = QPainter(self)
pixmap = QPixmap("images/back.jpg")
painter.drawPixmap(self.rect(), pixmap)
# def run(self,picture_path):
# img=picture_path
# a= predict(img)
# self.textEdit.setText(a)
#
if __name__ == '__main__':
from PyQt5 import QtCore
QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling) # 自适应分辨率
app = QtWidgets.QApplication(sys.argv)
camera = Open_Camera()
sys.exit(app.exec_())
gui.py
# -*- coding: utf-8 -*-
# Form implementation generated from reading ui file 'gui.ui'
#
# Created by: PyQt5 UI code generator 5.9.2
#
# WARNING! All changes made in this file will be lost!
from PyQt5 import QtCore, QtGui, QtWidgets
class Ui_mainWindow(object):
def setupUi(self, mainWindow):
mainWindow.setObjectName("mainWindow")
mainWindow.resize(618, 537)
mainWindow.setStyleSheet("")
self.centralwidget = QtWidgets.QWidget(mainWindow)
self.centralwidget.setObjectName("centralwidget")
self.gridLayout = QtWidgets.QGridLayout(self.centralwidget)
self.gridLayout.setObjectName("gridLayout")
self.horizontalLayout = QtWidgets.QHBoxLayout()
self.horizontalLayout.setObjectName("horizontalLayout")
self.label = QtWidgets.QLabel(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.label.sizePolicy().hasHeightForWidth())
self.label.setSizePolicy(sizePolicy)
self.label.setFrameShape(QtWidgets.QFrame.Box)
self.label.setTextFormat(QtCore.Qt.PlainText)
self.label.setScaledContents(False)
self.label.setIndent(-1)
self.label.setObjectName("label")
self.horizontalLayout.addWidget(self.label)
self.pushButton_3 = QtWidgets.QPushButton(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.pushButton_3.sizePolicy().hasHeightForWidth())
self.pushButton_3.setSizePolicy(sizePolicy)
self.pushButton_3.setObjectName("pushButton_3")
self.horizontalLayout.addWidget(self.pushButton_3)
self.label_2 = QtWidgets.QLabel(self.centralwidget)
self.label_2.setFrameShape(QtWidgets.QFrame.Box)
self.label_2.setObjectName("label_2")
self.horizontalLayout.addWidget(self.label_2)
self.gridLayout.addLayout(self.horizontalLayout, 0, 0, 1, 1)
self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.pushButton_2.sizePolicy().hasHeightForWidth())
self.pushButton_2.setSizePolicy(sizePolicy)
self.pushButton_2.setObjectName("pushButton_2")
self.gridLayout.addWidget(self.pushButton_2, 5, 0, 1, 1)
self.textEdit = QtWidgets.QTextEdit(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.textEdit.sizePolicy().hasHeightForWidth())
self.textEdit.setSizePolicy(sizePolicy)
self.textEdit.setMaximumSize(QtCore.QSize(16777215, 192))
self.textEdit.setObjectName("textEdit")
self.gridLayout.addWidget(self.textEdit, 4, 0, 1, 1)
self.label_3 = QtWidgets.QLabel(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(self.label_3.sizePolicy().hasHeightForWidth())
self.label_3.setSizePolicy(sizePolicy)
self.label_3.setObjectName("label_3")
self.gridLayout.addWidget(self.label_3, 1, 0, 1, 1)
mainWindow.setCentralWidget(self.centralwidget)
self.menubar = QtWidgets.QMenuBar(mainWindow)
self.menubar.setGeometry(QtCore.QRect(0, 0, 618, 23))
self.menubar.setObjectName("menubar")
mainWindow.setMenuBar(self.menubar)
self.statusbar = QtWidgets.QStatusBar(mainWindow)
self.statusbar.setObjectName("statusbar")
mainWindow.setStatusBar(self.statusbar)
self.retranslateUi(mainWindow)
QtCore.QMetaObject.connectSlotsByName(mainWindow)
def retranslateUi(self, mainWindow):
_translate = QtCore.QCoreApplication.translate
mainWindow.setWindowTitle(_translate("mainWindow", "视力障碍辅助"))
self.label.setText(_translate("mainWindow", "摄像头"))
self.pushButton_3.setText(_translate("mainWindow", "截图"))
self.label_2.setText(_translate("mainWindow", "截取图片"))
self.pushButton_2.setText(_translate("mainWindow", "语音播放"))
self.label_3.setText(_translate("mainWindow", "生成描述："))
pridect_.py
import torch
from torch import nn
from transformers import BertTokenizer
from PIL import Image
import argparse
from models.caption import build_model
from models import caption
from datasets import coco, utils
from configuration import Config
import os
def create_caption_and_mask(start_token, max_length):
caption_template = torch.zeros((1, max_length), dtype=torch.long)
mask_template = torch.ones((1, max_length), dtype=torch.bool)
caption_template[:, 0] = start_token
mask_template[:, 0] = False
return caption_template, mask_template
@torch.no_grad()
def evaluate(model,config,image,caption,cap_mask):
model.eval()
for i in range(config.max_position_embeddings - 1):
predictions = model(image, caption, cap_mask)
predictions = predictions[:, i, :]
predicted_id = torch.argmax(predictions, axis=-1)
if predicted_id[0] == 102:
return caption
caption[:, i+1] = predicted_id[0]
cap_mask[:, i+1] = False
return caption
def predict(image_path):
config = Config()
image_path = image_path # args.path
version = '' # args.v
checkpoint_path = './checkpoint.pth'
model, _ =build_model(config)
#model, _ = caption.build_model(config)
print("Loading Checkpoint...")
checkpoint = torch.load(checkpoint_path, map_location='cpu')
model.load_state_dict(checkpoint['model'])
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
start_token = tokenizer.convert_tokens_to_ids(tokenizer._cls_token)
end_token = tokenizer.convert_tokens_to_ids(tokenizer._sep_token) # 分隔输入中两个不同句子
image = Image.open(image_path)
image = coco.val_transform(image) # totensor normalize
image = image.unsqueeze(0)
caption, cap_mask = create_caption_and_mask(
start_token, config.max_position_embeddings)
output = evaluate(model,config,image,caption,cap_mask)
result = tokenizer.decode(output[0].tolist(), skip_special_tokens=True)
return result.capitalize()
#image_path ="niuniu.jpg"
#result=predict(image_path)
#print(result)
pridect_.py及cnn+transformer模型及代码来自github上一个博主分享的CNN+TRANSFORMER代码
