涉及到三个模型 静默活体检测模型<2M，人脸检测模型<2M ，人脸识别<5M(模型大小)
至于NCNN不必多说,全C++实现，不依赖第三方库实现，在第三方移动端CPU运行最快。
首先，这是三者结合的推理，这意味着从训练到转ncnn模型全部完成且作用效果的精度达到了不错的要求。
训练在此就省略了，复现的都是都是GitHub上的原项目，然后使用其最终模型进行C++上的推理。
实现的功能：一个demo实现对单个人的识别。（因为未涉及C++的太多知识，比如通过序列化对象转储到文件并从文件中读取，或者是通过连接数据库进行数据的转储，所以测试的推理的整个流程针对单人。）
步骤:先进行静默活体检测->人脸检测->人脸识别。后面的步骤都是在前面的步骤上实现，识别完后进行静默活体检测开始循环反复。
main函数
#include#include #include #include "platform.h" #include #include #include #include "iostream" #include "net.h" #if NCNN_VULKAN #include "gpu.h" #endif // NCNN_VULKAN #define IMAGE_SIZE 80 #define CV_RGB(r,g,b) cvScalar((b),(g),(r),0) bool JudgeReal(cv::Mat bgr, ncnn::Extractor ex, ncnn::Net & SilentFaceSpoofing); cv::Mat GetMyMat(cv::Mat img, const std::string& s);//获取人脸特征 cv::Mat GetMat(const std::string & path);//获取人脸框 cv::Mat GetMat(cv::Mat img);//重载 float* getFeatByMobileFaceNetNCNN(cv::Mat img);//获取特征向量 float CalcSimilarity_1(float* fc1,//比较出相识度 float* fc2, long dim); int main(){ cv::Mat sample = GetMat("./pic/yzh1.jpg"); float *sam= getFeatByMobileFaceNetNCNN( sample); static ncnn::Net SilentFaceSpoofing; SilentFaceSpoofing.load_param("./model/2.7_80x80_MiniFASNetV2_sim.param"); SilentFaceSpoofing.load_model("./model/2.7_80x80_MiniFASNetV2_sim.bin"); static ncnn::Extractor ex = SilentFaceSpoofing.create_extractor(); //打开摄像头 cv::VideoCapture capture(0);//创建VideoCapture对象 if (!capture.isOpened())//判断是否打开摄像头，打开isOpened返回ture return 1; bool stop(false);//定义一个用来停止循环的变量 cv::Mat frame;//用来存放读取的视频序列，承载每一帧的图像 ，Mat类是用于保存图像以及其他矩阵数据的数据结构 cv::namedWindow("Camera");//创建一个窗口，显示每一帧的窗口 while (!stop) { if (!capture.read(frame))//如果没有读取到就中断 { break; } bool result; result = JudgeReal(frame, ex, SilentFaceSpoofing);//判断真假脸 if (result) {//活体 cv::Mat frame2= GetMat(frame);//获取人脸框 float * fr= getFeatByMobileFaceNetNCNN(sample); float res= CalcSimilarity_1(fr, sam, 112); delete[]fr; if (res > 0.3f) { //认为是同一个人 cv::Mat imgs = GetMyMat(frame, "yuezhenhao"); putText(imgs, "realface", cv::Point(0, 100), cv::FONT_HERSHEY_SIMPLEX, 2, cv::Scalar(123, 0, 255), 4, 4); cv::imshow("Camera", imgs);//正常显示，把获取的视频填充到窗口中 } else { cv::Mat imgs = GetMyMat(frame, "unknow"); putText(imgs, "realface", cv::Point(0, 100), cv::FONT_HERSHEY_SIMPLEX, 2, cv::Scalar(123, 0, 255), 4, 4); cv::imshow("Camera", imgs);//正常显示，把获取的视频填充到窗口中 } } else { putText(frame, "FalseFace", cv::Point(0, 100), cv::FONT_HERSHEY_SIMPLEX, 2, cv::Scalar(123, 0, 255), 4, 4); cv::imshow("Camera", frame);//正常显示，把获取的视频填充到窗口中 std::cout << result << std::endl; } char c = cvWaitKey(33); if (c == 32)break; //使用空格键来停止ASCII 为32 关闭摄像头 } capture.release();//释放 return 0; }
获取人的特征向量以及比较人的相识度（mobilefacenet网络实现识别）
#include#include "net.h" #include using namespace std; static int flag = 0; //Mobilefacenet 初始化 float simd_dot_1(const float* x, const float* y, const long& len) { float inner_prod = 0.0f; __m128 X, Y; // 128-bit values __m128 acc = _mm_setzero_ps(); // set to (0, 0, 0, 0) float temp[4]; long i; for (i = 0; i + 4 < len; i += 4) { X = _mm_loadu_ps(x + i); // load chunk of 4 floats Y = _mm_loadu_ps(y + i); acc = _mm_add_ps(acc, _mm_mul_ps(X, Y)); } _mm_storeu_ps(&temp[0], acc); // store acc into an array inner_prod = temp[0] + temp[1] + temp[2] + temp[3]; // add the remaining values for (; i < len; ++i) { inner_prod += x[i] * y[i]; } return inner_prod; } float CalcSimilarity_1( float* fc1, float* fc2, long dim) { return simd_dot_1(fc1, fc2, dim) / (sqrt(simd_dot_1(fc1, fc1, dim)) * sqrt(simd_dot_1(fc2, fc2, dim))); } float* getFeatByMobileFaceNetNCNN( cv::Mat img) { static ncnn::Net Recognet; if (!flag) { Recognet.load_param("./model/mobilefacenet.param"); Recognet.load_model("./model/mobilefacenet.bin"); } ncnn::Extractor ex= Recognet.create_extractor(); ex.set_light_mode(true); ex.set_num_threads(4); //cout << "getFeatByMobileFaceNetNCNN" << endl; float* feat = new float[512]; ncnn::Mat in = ncnn::Mat::from_pixels_resize(img.data, ncnn::Mat::PIXEL_BGR, img.cols, img.rows, 112, 112); ex.input("data", in); ncnn::Mat out; ex.extract("fc1", out); //std::vector cls_scores; for (int j = 0; j < out.w; j++) { feat[j] = out[j]; } return feat; }
静默活体识别引用函数（实现预处理）
#include#include #include #include #include #include "iostream" #include "platform.h" #include "net.h" #if NCNN_VULKAN #include "gpu.h" #endif // NCNN_VULKAN #define IMAGE_SIZE 80 bool JudgeReal(cv::Mat bgr, ncnn::Extractor ex, ncnn::Net& SilentFaceSpoofing) { std::vector cls_scores; ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, bgr.rows, IMAGE_SIZE, IMAGE_SIZE); fprintf(stderr, "input shape: %d %d %d %dn", in.dims, in.h, in.w, in.c); ex.input("input", in);//input 是 .param文件中输入节点名称 ncnn::Mat out; ex.extract("output", out); { ncnn::Layer* softmax = ncnn::create_layer("Softmax"); ncnn::ParamDict pb; softmax->load_param(pb); softmax->forward_inplace(out, SilentFaceSpoofing.opt); delete softmax; } out = out.reshape(out.h * out.w * out.c); fprintf(stderr, "output shape: %d %d %d %dn", out.dims, out.h, out.w, out.c); cls_scores.resize(out.w); for (int j = 0; j < out.w; j++) { cls_scores[j] = out[j]; printf("cls_scores[%d]=%fn", j, cls_scores[j]); } std::cout << std::endl; auto itMax = max_element(cls_scores.begin(), cls_scores.end()); std::cout << "the max:" << *itMax << " the location:" << distance(cls_scores.begin(), itMax) << std::endl; if (distance(cls_scores.begin(), itMax) == 1) { std::cout << "Image " << distance(cls_scores.begin(), itMax) << " is Real Face. Score: " << *itMax << std::endl; return true; } else { std::cout << "Image " << distance(cls_scores.begin(), itMax) << " is Fake Face. Score: " << *itMax << std::endl; return false; } }
获取图像中的人脸框及其特征点 （retinaface网络实现检测，这里的boxsize是图片中检测的人脸个数，由于本次demo只针对单人，所以返回的总是最后一个人的检测框）
#include#include #include #include #include #include #include #include "FaceDetector.h" using namespace std; static bbox mybox; //cv::Mat GetMat() cv::Mat GetMat(const std::string & path) { string imgPath = path; static string param = "./model/face.param"; static string bin = "./model/face.bin"; static const int max_side = 320; static Detector detector(param, bin, false); // retinaface // Detector detector(param, bin, true); Timer timer; cv::Mat img = cv::imread(imgPath.c_str(), CV_LOAD_IMAGE_COLOR); // scale float long_side = max(img.cols, img.rows); float scale = max_side/long_side; cv::Mat img_scale; cv::Size size = cv::Size(img.cols*scale, img.rows*scale); cv::resize(img, img_scale, cv::Size(img.cols*scale, img.rows*scale)); if (img.empty()) { fprintf(stderr, "cv::imread %s failedn", imgPath.c_str()); } std::vector boxes; timer.tic(); detector.Detect(img_scale, boxes); timer.toc("----total timer:"); if (!boxes.size()) { cv::Mat temp; cv::resize(img, temp, cv::Size(112, 112)); return temp; } cv::Mat temps; // draw image 裁剪出图片中人脸框 for (int j = 0; j < boxes.size(); ++j) { cv::Rect rect(boxes[j].x1 / scale, boxes[j].y1 / scale, boxes[j].x2 / scale - boxes[j].x1 / scale, boxes[j].y2 / scale - boxes[j].y1 / scale); cv::rectangle(img, rect, cv::Scalar(0, 0, 255), 1, 8, 0); // char test[80]; cv::Mat m = img(rect); cv::resize(m, temps, cv::Size(112, 112)); mybox = boxes[j]; } return temps; } cv::Mat GetMat( cv::Mat img) { static string param = "./model/face.param"; static string bin = "./model/face.bin"; static const int max_side = 320; static Detector detector(param, bin, false); // retinaface // Detector detector(param, bin, true); Timer timer; // scale float long_side = max(img.cols, img.rows); float scale = max_side / long_side; cv::Mat img_scale; cv::Size size = cv::Size(img.cols * scale, img.rows * scale); cv::resize(img, img_scale, cv::Size(img.cols * scale, img.rows * scale)); if (img.empty()) { std::cout << "empty img" << std::endl; } std::vector boxes; timer.tic(); detector.Detect(img_scale, boxes); timer.toc("----total timer:"); if (!boxes.size()) { cv::Mat temp; cv::resize(img, temp, cv::Size(112, 112)); return temp; } cv::Mat temps; // draw image 裁剪出图片中人脸框 for (int j = 0; j < boxes.size(); ++j) { cv::Rect rect(boxes[j].x1 / scale, boxes[j].y1 / scale, boxes[j].x2 / scale - boxes[j].x1 / scale, boxes[j].y2 / scale - boxes[j].y1 / scale); cv::rectangle(img, rect, cv::Scalar(0, 0, 255), 1, 8, 0); // char test[80]; cv::Mat m = img(rect); cv::resize(m, temps, cv::Size(112, 112)); mybox = boxes[j]; } return temps; } cv::Mat GetMyMat( cv::Mat img, const std::string &s) { float long_side = max(img.cols, img.rows); float scale = 320 / long_side; cv::Rect rect(mybox.x1 / scale, mybox.y1 / scale, mybox.x2 / scale - mybox.x1 / scale, mybox.y2 / scale - mybox.y1 / scale); cv::rectangle(img, rect, cv::Scalar(0, 0, 255), 3, 8, 0); cv::putText(img, s, cv::Size((mybox.x1 / scale), mybox.y1 / scale), cv::FONT_HERSHEY_COMPLEX, 0.5, cv::Scalar(0, 255, 255)); cv::circle(img, cv::Point(mybox.point[0]._x / scale, mybox.point[0]._y / scale), 1, cv::Scalar(0, 0, 225), 4); cv::circle(img, cv::Point(mybox.point[1]._x / scale, mybox.point[1]._y / scale), 1, cv::Scalar(0, 255, 225), 4); cv::circle(img, cv::Point(mybox.point[2]._x / scale, mybox.point[2]._y / scale), 1, cv::Scalar(255, 0, 225), 4); cv::circle(img, cv::Point(mybox.point[3]._x / scale, mybox.point[3]._y / scale), 1, cv::Scalar(0, 255, 0), 4); cv::circle(img, cv::Point(mybox.point[4]._x / scale, mybox.point[4]._y / scale), 1, cv::Scalar(255, 0, 0), 4); return img; }
#include
//#include "omp.h"
#include "FaceDetector.h"
using namespace std;
Detector::Detector():
_nms(0.4),
_threshold(0.6),
_mean_val{104.f, 117.f, 123.f},
_retinaface(false),
Net(new ncnn::Net())
{
}
inline void Detector::Release(){
if (Net != nullptr)
{
delete Net;
Net = nullptr;
}
}
Detector::Detector(const std::string &model_param, const std::string &model_bin, bool retinaface):
_nms(0.4),
_threshold(0.6),
_mean_val{104.f, 117.f, 123.f},
_retinaface(retinaface),
Net(new ncnn::Net())
{
Init(model_param, model_bin);
}
void Detector::Init(const std::string &model_param, const std::string &model_bin)
{
int ret = Net->load_param(model_param.c_str());
ret = Net->load_model(model_bin.c_str());
}
void Detector::Detect(cv::Mat& bgr, std::vector& boxes)
{
Timer timer;
timer.tic();
ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, bgr.rows, bgr.cols, bgr.rows);
in.substract_mean_normalize(_mean_val, 0);
timer.toc("precoss:");
timer.tic();
ncnn::Extractor ex = Net->create_extractor();
ex.set_light_mode(true);
ex.set_num_threads(4);
ex.input(0, in);
ncnn::Mat out, out1, out2;
// loc
ex.extract("output0", out);
// class
ex.extract("530", out1);
//landmark
ex.extract("529", out2);
timer.toc("det:");
std::vector anchor;
timer.tic();
if (_retinaface)
create_anchor_retinaface(anchor, bgr.cols, bgr.rows);
else
create_anchor(anchor, bgr.cols, bgr.rows);
timer.toc("anchor:");
std::vector total_box;
float *ptr = out.channel(0);
float *ptr1 = out1.channel(0);
float *landms = out2.channel(0);
// #pragma omp parallel for num_threads(2)
for (int i = 0; i < anchor.size(); ++i)
{
if (*(ptr1+1) > _threshold)
{
box tmp = anchor[i];
box tmp1;
bbox result;
// loc and conf
tmp1.cx = tmp.cx + *ptr * 0.1 * tmp.sx;
tmp1.cy = tmp.cy + *(ptr+1) * 0.1 * tmp.sy;
tmp1.sx = tmp.sx * exp(*(ptr+2) * 0.2);
tmp1.sy = tmp.sy * exp(*(ptr+3) * 0.2);
result.x1 = (tmp1.cx - tmp1.sx/2) * in.w;
if (result.x1<0)
result.x1 = 0;
result.y1 = (tmp1.cy - tmp1.sy/2) * in.h;
if (result.y1<0)
result.y1 = 0;
result.x2 = (tmp1.cx + tmp1.sx/2) * in.w;
if (result.x2>in.w)
result.x2 = in.w;
result.y2 = (tmp1.cy + tmp1.sy/2)* in.h;
if (result.y2>in.h)
result.y2 = in.h;
result.s = *(ptr1 + 1);
// landmark
for (int j = 0; j < 5; ++j)
{
result.point[j]._x =( tmp.cx + *(landms + (j<<1)) * 0.1 * tmp.sx ) * in.w;
result.point[j]._y =( tmp.cy + *(landms + (j<<1) + 1) * 0.1 * tmp.sy ) * in.h;
}
total_box.push_back(result);
}
ptr += 4;
ptr1 += 2;
landms += 10;
}
std::sort(total_box.begin(), total_box.end(), cmp);
nms(total_box, _nms);
printf("%dn", (int)total_box.size());
for (int j = 0; j < total_box.size(); ++j)
{
boxes.push_back(total_box[j]);
}
}
inline bool Detector::cmp(bbox a, bbox b) {
if (a.s > b.s)
return true;
return false;
}
inline void Detector::SetDefaultParams(){
_nms = 0.4;
_threshold = 0.6;
_mean_val[0] = 104;
_mean_val[1] = 117;
_mean_val[2] = 123;
Net = nullptr;
}
Detector::~Detector(){
Release();
}
void Detector::create_anchor(std::vector &anchor, int w, int h)
{
// anchor.reserve(num_boxes);
anchor.clear();
std::vector > feature_map(4), min_sizes(4);
float steps[] = {8, 16, 32, 64};
for (int i = 0; i < feature_map.size(); ++i) {
feature_map[i].push_back(ceil(h/steps[i]));
feature_map[i].push_back(ceil(w/steps[i]));
}
std::vector minsize1 = {10, 16, 24};
min_sizes[0] = minsize1;
std::vector minsize2 = {32, 48};
min_sizes[1] = minsize2;
std::vector minsize3 = {64, 96};
min_sizes[2] = minsize3;
std::vector minsize4 = {128, 192, 256};
min_sizes[3] = minsize4;
for (int k = 0; k < feature_map.size(); ++k)
{
std::vector min_size = min_sizes[k];
for (int i = 0; i < feature_map[k][0]; ++i)
{
for (int j = 0; j < feature_map[k][1]; ++j)
{
for (int l = 0; l < min_size.size(); ++l)
{
float s_kx = min_size[l]*1.0/w;
float s_ky = min_size[l]*1.0/h;
float cx = (j + 0.5) * steps[k]/w;
float cy = (i + 0.5) * steps[k]/h;
box axil = {cx, cy, s_kx, s_ky};
anchor.push_back(axil);
}
}
}
}
}
void Detector::create_anchor_retinaface(std::vector &anchor, int w, int h)
{
// anchor.reserve(num_boxes);
anchor.clear();
std::vector > feature_map(3), min_sizes(3);
float steps[] = {8, 16, 32};
for (int i = 0; i < feature_map.size(); ++i) {
feature_map[i].push_back(ceil(h/steps[i]));
feature_map[i].push_back(ceil(w/steps[i]));
}
std::vector minsize1 = {10, 20};
min_sizes[0] = minsize1;
std::vector minsize2 = {32, 64};
min_sizes[1] = minsize2;
std::vector minsize3 = {128, 256};
min_sizes[2] = minsize3;
for (int k = 0; k < feature_map.size(); ++k)
{
std::vector min_size = min_sizes[k];
for (int i = 0; i < feature_map[k][0]; ++i)
{
for (int j = 0; j < feature_map[k][1]; ++j)
{
for (int l = 0; l < min_size.size(); ++l)
{
float s_kx = min_size[l]*1.0/w;
float s_ky = min_size[l]*1.0/h;
float cx = (j + 0.5) * steps[k]/w;
float cy = (i + 0.5) * steps[k]/h;
box axil = {cx, cy, s_kx, s_ky};
anchor.push_back(axil);
}
}
}
}
}
void Detector::nms(std::vector &input_boxes, float NMS_THRESH)
{
std::vectorvArea(input_boxes.size());
for (int i = 0; i < int(input_boxes.size()); ++i)
{
vArea[i] = (input_boxes.at(i).x2 - input_boxes.at(i).x1 + 1)
* (input_boxes.at(i).y2 - input_boxes.at(i).y1 + 1);
}
for (int i = 0; i < int(input_boxes.size()); ++i)
{
for (int j = i + 1; j < int(input_boxes.size());)
{
float xx1 = max(input_boxes[i].x1, input_boxes[j].x1);
float yy1 = max(input_boxes[i].y1, input_boxes[j].y1);
float xx2 = min(input_boxes[i].x2, input_boxes[j].x2);
float yy2 = min(input_boxes[i].y2, input_boxes[j].y2);
float w = max(float(0), xx2 - xx1 + 1);
float h = max(float(0), yy2 - yy1 + 1);
float inter = w * h;
float ovr = inter / (vArea[i] + vArea[j] - inter);
if (ovr >= NMS_THRESH)
{
input_boxes.erase(input_boxes.begin() + j);
vArea.erase(vArea.begin() + j);
}
else
{
j++;
}
}
}
}
效果:
真脸
最后:静默活体检测还存在一定误差。感谢各位大佬的开源，让作为菜鸟的我完成了缝合。
参考:38、静默活体检测测试及ncnn、mnn部署_sxj731533730的博客-CSDN博客_ncnn和mnn
deepinsight/insightface: State-of-the-art 2D and 3D Face Analysis Project (github.com)
GitHub - biubug6/Pytorch_Retinaface: Retinaface get 80.99% in widerface hard val using mobilenet0.25.
