YOLOX从训练到NCNN部署全流程(6)——模型部署

YOLOX从训练到NCNN部署全流程(1)——环境配置
YOLOX从训练到NCNN部署全流程(2)——代码训练
YOLOX从训练到NCNN部署全流程(3)——模型转化与量化
YOLOX从训练到NCNN部署全流程(4)——模型修改
YOLOX从训练到NCNN部署全流程(5)——创建demo工程
YOLOX从训练到NCNN部署全流程(6)——模型部署
YOLOX从训练到NCNN部署全流程(7)——模型部署测试
YOLOX从训练到NCNN部署全流程(8)——生成so库与jar包
YOLOX从训练到NCNN部署全流程(9)——使用so库和jar包

YOLOX从训练到NCNN部署全流程（6）——模型部署

1. 新建库module
2 为库模块添加c++支持
- 2.1 修改Deeplearning的gradle脚本
- 2.2 准备CMakeLists文件
- 2.3 接口JNI编写
- 2.4 模型推理及后处理

这个部分的目的就是新建一个库，专门用来调用深度学习模型，然后将库打包起来供别人从java层调用这个库。因为jni中的函数名是有特殊要求的，它会指定jni的java接口的路径，如果不提供jar包，那么使用者就要按照我们在jni头文件中声明的函数名来建立java文件，这是几乎不可能的事情。

1. 新建库module

在刚才新建的那个工程中，新建一个module（Android Studio的组织结构是一个project可以包含多个module，每个module都可以独立编译为一个apk）。这个module就是调用深度学习模型的库模块。
File > New > New module > Android Library
填好相关信息等就可以finish了。目前在用的是AndroidStudio3.4.1，在新建工程的时候已经找不到添加c++ support的选项了，因此一会儿也免不了要自己去改build.gradle文件来添加c++支持。
新建完后的工程结构如图:
可以看到我们DeepLearning模块和app是同级的。

2 为库模块添加c++支持

可以新建一个支持c++的工程，然后复制这个工程的cpp文件夹过来就好了，或者可以参考以前有NDK的工程。

2.1 修改Deeplearning的gradle脚本

此处修改的是Deeplearning的build.gradle配置文件。在使用Android studio 创建Android项目时，会默认创建两个build.gradle文件，第一个build.gradle文件不用太多关注，但是这里有个坑。
在老版Android Studio 是把 jcenter作为gradle的默认配置，但是 jcenter 服务即将停止，需要迁移项目库（比如mavenCentral），这意味着几乎所有使用了jcenter的项目将要面临失败构建（包含Android开发中build.gradle 和其它开源的二进制依赖）。
生成代码如下：

// Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
    repositories {
        google()
        jcenter()
    }
    dependencies {
        classpath "com.android.tools.build:gradle:7.0.3"

        // NOTE: Do not place your application dependencies here; they belong
        // in the individual module build.gradle files
    }
}

allprojects {
    repositories {
        jcenter ()
        google()
    }
}
task clean(type: Delete) {
    delete rootProject.buildDir
}

在Android Studio 4.2 Google已经把Maven Central 作为gradle的默认配置了，因此需要把Android Studio 更新到4.2，甚至最新版本。更新后生成的代码如下：

/ Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
    repositories {
        google()
        mavenCentral()
    }
    dependencies {
        classpath "com.android.tools.build:gradle:7.0.3"

        // NOTE: Do not place your application dependencies here; they belong
        // in the individual module build.gradle files
    }
}

task clean(type: Delete) {
    delete rootProject.buildDir
}

第二个build.gradle文件的相关设置如下：在defaultConfig节点下指定SDK版本（此处注意：与app的build.gradle文件中的版本要一致）；同样在defaultConfig节点下指定要编译的cpu架构；在android节点下指定CMakeLists的位置。最后文件如下：

plugins {
    id 'com.android.library'
}

android {
    compileSdk 32

    defaultConfig {
        minSdk 25
        targetSdk 32

        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
        consumerProguardFiles "consumer-rules.pro"
        //=================增加部分======================
        ndk{
            moduleName "ncnn" //编译后会生成 .so
            ldLibs "log"
            abiFilters "armeabi-v7a","arm64-v8a"
        }
        //==============================================
    }

    buildTypes {
        release {
            minifyEnabled false
            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
        }
    }
    compileOptions {
        sourceCompatibility JavaVersion.VERSION_1_8
        targetCompatibility JavaVersion.VERSION_1_8
    }
    //这里指定CMakeLists的位置
    externalNativeBuild {
        cmake {
            path file('src/main/cpp/CMakeLists.txt')   //这个路径一定要填对
            version '3.10.2'
        }
    }
}

dependencies {

    implementation 'androidx.appcompat:appcompat:1.3.0'
    implementation 'com.google.android.material:material:1.4.0'
    testImplementation 'junit:junit:4.13.2'
    androidTestImplementation 'androidx.test.ext:junit:1.1.3'
    androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
}

2.2 准备CMakeLists文件

下载官网编译好的ncnn库，下图1中文件夹是支持GPU的版本，下图2中是CPU版本。将其下载下来并分别重命名为ncnnvulkan和ncnn。下载地址
对于静态库(.a) 来说需要结合头文件(.h) 一起使用，头文件中负责声明函数，静态库中负责实现函数的主体。对于动态库(.so) 来说，不需要头文件，通过 loadLibrary 来加载库，调用里面暴露的方法。
ncnn提供的是静态库(.a) 所以需要结合头文件来使用，将ncnn安装目录下install 的 include目录拷贝到cppncnnvulkan中，如下图所示。
在src/main/ 路径下新建一个cpp文件夹，与java文件夹同级，将ncnnvulkan文件夹放在cpp文件夹中，并在cpp文件夹中新建CMakeLists文件；具体代码如下所示：

# Sets the minimum version of CMake required to build the native library.
cmake_minimum_required(VERSION 3.10.2)     #cmake最小版本
project("deeplearning")              #项目名称

# 这一句用来导出编译生成的so库
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/jniLibs/${ANDROID_ABI})

###========================编译NCNN和YOLOX================================###
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp")
if(DEFINED ANDROID_NDK_MAJOR AND ${ANDROID_NDK_MAJOR} GREATER 20)
    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-openmp")
endif()

#------------------创建YOLOX动态库---------------------#
aux_source_directory(. SRC_LIST)     #搜索当前目录下的所有.cpp文件
add_library(
        yolox
        SHARED
        ${SRC_LIST}
)
#-------------添加NCNN的include路径--------------------#
include_directories(
        ncnnvulkan/include
)

#---------------查找指定的预编译库----------------------#
find_library(log-lib log)           #将它的路径存储在变量中
find_library(android-lib android)
find_library(vulkan-lib vulkan)
find_library(jnigraphics-lib jnigraphics)

##==================导入ncnn静态库====================##
add_library( ncnn STATIC IMPORTED )        #创建静态库
set_target_properties(                     # 将导入的静态库作为静态库
        ncnn                               #库名
        PROPERTIES IMPORTED_LOCATION       #设定参数名
        ${CMAKE_SOURCE_DIR}/ncnnvulkan/${ANDROID_ABI}/libncnn.a )     #导入库的路径

#--------------------ncnnvulkan----------------------#
add_library(glslang STATIC IMPORTED)
add_library(OGLCompiler STATIC IMPORTED)
add_library(OSDependent STATIC IMPORTED)
add_library(SPIRV STATIC IMPORTED)
set_target_properties(glslang PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/ncnnvulkan/${ANDROID_ABI}/libglslang.a)
set_target_properties(OGLCompiler PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/ncnnvulkan/${ANDROID_ABI}/libOGLCompiler.a)
set_target_properties(OSDependent PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/ncnnvulkan/${ANDROID_ABI}/libOSDependent.a)
set_target_properties(SPIRV PROPERTIES IMPORTED_LOCATION ${CMAKE_SOURCE_DIR}/ncnnvulkan/${ANDROID_ABI}/libSPIRV.a)

#------------------------------------------------------#
# NCNN 使用no-RTTI，而opencv必须使用RTTI，如何解决？
# disable rtti and exceptions
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -fno-exceptions")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
# enable rtti and exceptions
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti")
##======================================================##


#target_link_libraries( # Specifies the target library.
#        yolox
#
#        # Links the target library to the log library
#        # included in the NDK.
#        ${log-lib}
#        ${vulkan-lib}
#        ${android-lib}
#        ${jnigraphics-lib}
#        )
##libopencv_java4)

target_link_libraries( # Specifies the target library.
        yolox

        # 添加opencv_java4
        #opencv_java4

        # Links the target library to the log library
        # included in the NDK.
        ${log-lib}
        ${vulkan-lib}
        ${android-lib}
        ${jnigraphics-lib}
        ncnn
        glslang SPIRV OGLCompiler OSDependent )
###==========================================================###

注意第51行那条注释：NCNN 使用no-RTTI，而opencv必须使用RTTI，如何解决？
这是NCNN部署中的一个问题，NCNN 和Opencv的存在冲突，目前不知道解决办法，因此导致不能在c++脚本中使用opencv函数来做一些前处理或后处理，而是选择在java脚本中处理。如果cmake成功文件结构如下所示：

2.3 接口JNI编写

1、在DeepLearning库中java目录下中创建YOLOX类，其中定义 Init 和 Detect 两个函数来与 yolox_jni.cpp里面函数对应，以供其它函数进行调用。

package com.lqy.deeplearning;

import android.content.res.AssetManager;
import android.graphics.Bitmap;

public class YOLOX {
    static { System.loadLibrary("yolox");}   //加载cmakelist中创建的yolox静态库
    public static native void  Init(AssetManager manager);    //初始化函数
    public static native Box[] Detect(Bitmap bitmap, double threshold, double nms_threshold);  //检测函数
}

2、在java中创建Box类，定义一些输入输出的数据，用来与接口函数进行数据交互。

package com.lqy.deeplearning;

import android.graphics.Color;
import android.graphics.RectF;
import java.util.Random;

public class Box {
    public float x0, y0, w, h;
    public int label;
    public float score;

    private static String[] labels = {"iris", "pipul", "shut-eye"};

    public Box(float x,float y, float w, float h, int label, float score) {
        this.x0 = x;
        this.y0 = y;
        this.w = w;
        this.h = h;
        this.label = label;
        this.score = score;
    }


    //得出Bbox ，但需要输入的是左上角和右下角两点坐标
    public RectF getRect(){
        return new RectF(x0,y0,w,h);
    }
    public String getLabel(){
        return labels[label];
    }
    public float getScore(){
        return score;
    }
    public int getColor(){
        Random random = new Random(label);
        return Color.argb(255,random.nextInt(256),random.nextInt(256),random.nextInt(256));
    }
}

3、JNI接口函数编写

数据类型

在JNI中如果需要接受来自java的数据，在对应的数据类型前加一个 j，例如java中的 boolean类型，对应JNI中的 jboolean，其他类型同理。

函数命名规则

JNI中函数需要对应java中的函数，对应规则如下：
Java_com_lqy_yolox_Yolox_Init
Java ：表示是java的包
com_lqy_yolox：表示路径comlqyyolox
Yolox ：表示类名
Init ：表示函数名
如果想要使用ncnn的GPU推理，需要添加 JNI_OnLoad 和 JNI_OnUnLoad函数，如果只是用CPU可以不添加下面的函数。
JNI_OnLoad ：Java 调用System.loadLibrary() 加载库的时候，会自动调用该函数
JNI_OnUnLoad：当GC回收了加载这个库的ClassLoader时，会自动调用该函数。

#include 
#include 

//包含以下头文件
#include 
#include 

#include "yolox.h"

JNIEXPORT jint JNI_OnLoad(JavaVM *vm,void *reserved){
ncnn::create_gpu_instance();
if (ncnn::get_gpu_count() > 0){
yolox::hasGPU = true;
}
return JNI_VERSION_1_6;
}

JNIEXPORT void JNI_OnUnload(JavaVM *vm, void *reserved) {
    ncnn::destroy_gpu_instance();
    delete yolox::detector;

}
extern "C" JNIEXPORT void JNICALL
Java_com_lqy_deeplearning_YOLOX_Init(JNIEnv *env, jclass, jobject assetManager) {
if (yolox::detector != nullptr) {
    delete yolox::detector;
    yolox::detector = nullptr;}
if (yolox::detector == nullptr) {
    AAssetManager *mgr = AAssetManager_fromJava(env, assetManager);
    yolox::detector = new yolox(mgr, "5920_nano.param", "5920_nano.bin", true);}
}

extern "C" JNIEXPORT jobjectArray JNICALL
Java_com_lqy_deeplearning_YOLOX_Detect(JNIEnv *env, jclass, jobject image, jdouble threshold, jdouble nms_threshold)
{
    auto result = yolox::detector->detect(env, image,threshold,nms_threshold);

    auto box_cls = env->FindClass("com/lqy/deeplearning/Box");
    auto cid = env->GetMethodID(box_cls, "", "(FFFFIF)V");
    jobjectArray ret = env->NewObjectArray(result.size(), box_cls, nullptr);
    int i = 0;
    for (auto &box:result)
    {
        env->PushLocalFrame(1);
        jobject obj = env->NewObject(box_cls, cid, box.x, box.y, box.w, box.h, box.label, box.score);
        obj = env->PopLocalFrame(obj);
        env->SetObjectArrayElement(ret, i++, obj);
    }
    return ret;
}

2.4 模型推理及后处理

1、cpp文件夹中创建yolox类，用来实现模型推理和数据后处理。程序如下：
yolox.h文件

//
// Created by lqy on 2022/06/27.
//
#ifndef YOLOX_YOLOX_H
#define YOLOX_YOLOX_H

#include 

namespace yolocv {
    typedef struct {
        int width;
        int height;
    } YoloSize;
}

struct BoxInfo {
    float x;
    float y;
    float w;
    float h;
    float score;
    int label;
} ;

struct GridAndStride
{
    int grid0;
    int grid1;
    int stride;
};

class yolox {
public:
    yolox(AAssetManager *mgr, const char *param, const char *bin, bool useGPU);

    ~yolox();

    std::vector detect(JNIEnv *env, jobject image, jdouble threshold, jdouble nms_threshold);
    std::vector labels{"iris", "pipul", "shut-eye"};
private:

    ncnn::Net *Net;
    int input_size  = 256;
    int num_class = 3;

public:
    static yolox *detector;
    static bool hasGPU;
    static bool toUseGPU;

};

#endif //YOLOX_YOLOX_H

yolox.cpp文件

//
// Created by lqy on 2022/1/6.
//

#include "yolox.h"

//=======================自定义focus层=============================
class YoloV5Focus : public ncnn::Layer {
public:
    YoloV5Focus();

    virtual int forward(const ncnn::Mat &bottom_blob, ncnn::Mat &top_blob, const ncnn::Option &opt) const;
};

DEFINE_LAYER_CREATOR(YoloV5Focus)

YoloV5Focus::YoloV5Focus() {
    one_blob_only = true;
}

int YoloV5Focus::forward(const ncnn::Mat &bottom_blob, ncnn::Mat &top_blob, const ncnn::Option &opt) const {
    int w = bottom_blob.w;
    int h = bottom_blob.h;
    int channels = bottom_blob.c;

    int outw = w / 2;
    int outh = h / 2;
    int outc = channels * 4;

    top_blob.create(outw, outh, outc, 4u, 1, opt.blob_allocator);
    if (top_blob.empty())
        return -100;

#pragma omp parallel for num_threads(opt.num_threads)
    for (int p = 0; p < outc; p++) {
        const float *ptr = bottom_blob.channel(p % channels).row((p / channels) % 2) + ((p / channels) / 2);
        float *outptr = top_blob.channel(p);

        for (int i = 0; i < outh; i++) {
            for (int j = 0; j < outw; j++) {
                *outptr = *ptr;

                outptr += 1;
                ptr += 2;
            }

            ptr += w;
        }
    }

    return 0;
}
// ===================================================================================================

static inline float intersection_area(const BoxInfo &a, const BoxInfo &b) {
    if (a.x > b.x + b.w || a.x + a.w < b.x || a.y > b.y + b.h || a.y + a.h < b.y) {
        // no intersection
        return 0.f;
    }
    float inter_width = std::min(a.x + a.w, b.x + b.w) - std::max(a.x, b.x);
    float inter_height = std::min(a.y + a.h, b.y + b.h) - std::max(a.y, b.y);

    return inter_width * inter_height;
}

static void qsort_descent_inplace(std::vector &faceobjects, int left, int right) {
    int i = left;
    int j = right;
    float p = faceobjects[(left + right) / 2].score;

    while (i <= j) {
        while (faceobjects[i].score > p)
            i++;

        while (faceobjects[j].score < p)
            j--;

        if (i <= j) {
            // swap
            std::swap(faceobjects[i], faceobjects[j]);

            i++;
            j--;
        }
    }

#pragma omp parallel sections
    {
#pragma omp section
        {
            if (left < j) qsort_descent_inplace(faceobjects, left, j);
        }
#pragma omp section
        {
            if (i < right) qsort_descent_inplace(faceobjects, i, right);
        }//
// Created by HG06 on 2022/1/6.
//

#include "yolox.h"

class YoloV5Focus : public ncnn::Layer {
public:
    YoloV5Focus();

    virtual int forward(const ncnn::Mat &bottom_blob, ncnn::Mat &top_blob, const ncnn::Option &opt) const;
};

DEFINE_LAYER_CREATOR(YoloV5Focus)

YoloV5Focus::YoloV5Focus() {
    one_blob_only = true;
}

int YoloV5Focus::forward(const ncnn::Mat &bottom_blob, ncnn::Mat &top_blob, const ncnn::Option &opt) const {
    int w = bottom_blob.w;
    int h = bottom_blob.h;
    int channels = bottom_blob.c;

    int outw = w / 2;
    int outh = h / 2;
    int outc = channels * 4;

    top_blob.create(outw, outh, outc, 4u, 1, opt.blob_allocator);
    if (top_blob.empty())
        return -100;

#pragma omp parallel for num_threads(opt.num_threads)
    for (int p = 0; p < outc; p++) {
        const float *ptr = bottom_blob.channel(p % channels).row((p / channels) % 2) + ((p / channels) / 2);
        float *outptr = top_blob.channel(p);

        for (int i = 0; i < outh; i++) {
            for (int j = 0; j < outw; j++) {
                *outptr = *ptr;

                outptr += 1;
                ptr += 2;
            }

            ptr += w;
        }
    }

    return 0;
}

// ===================================================================================================

static inline float intersection_area(const BoxInfo &a, const BoxInfo &b) {
    if (a.x > b.x + b.w || a.x + a.w < b.x || a.y > b.y + b.h || a.y + a.h < b.y) {
        // no intersection
        return 0.f;
    }
    float inter_width = std::min(a.x + a.w, b.x + b.w) - std::max(a.x, b.x);
    float inter_height = std::min(a.y + a.h, b.y + b.h) - std::max(a.y, b.y);

    return inter_width * inter_height;
}

static void qsort_descent_inplace(std::vector &faceobjects, int left, int right) {
    int i = left;
    int j = right;
    float p = faceobjects[(left + right) / 2].score;

    while (i <= j) {
        while (faceobjects[i].score > p)
            i++;

        while (faceobjects[j].score < p)
            j--;

        if (i <= j) {
            // swap
            std::swap(faceobjects[i], faceobjects[j]);

            i++;
            j--;
        }
    }

#pragma omp parallel sections
    {
#pragma omp section
        {
            if (left < j) qsort_descent_inplace(faceobjects, left, j);
        }
#pragma omp section
        {
            if (i < right) qsort_descent_inplace(faceobjects, i, right);
        }
    }
}

static void qsort_descent_inplace(std::vector &bbox) {
    if (bbox.empty()) {
        return;
    }
    qsort_descent_inplace(bbox, 0, bbox.size() - 1);
}

static void nms_sorted_bboxes(const std::vector& bbox, std::vector& picked, float nms_threshold)
{
    picked.clear();

    const int n = bbox.size();

    std::vector areas(n);
    for (int i = 0; i < n; i++)
    {
        areas[i] = bbox[i].w * bbox[i].h;
    }

    for (int i = 0; i < n; i++)
    {
        const BoxInfo& a = bbox[i];

        int keep = 1;
        for (int j = 0; j < (int)picked.size(); j++)
        {
            const BoxInfo& b = bbox[picked[j]];

            // intersection over union
            float inter_area = intersection_area(a, b);
            float union_area = areas[i] + areas[picked[j]] - inter_area;
            // float IoU = inter_area / union_area
            if (inter_area / union_area > nms_threshold)
                keep = 0;
        }

        if (keep)
            picked.push_back(i);
    }
}



static void generate_grids_and_stride(const int target_size, std::vector& strides, std::vector& grid_strides)
{
    for (auto stride : strides)
    {
        int num_grid = target_size / stride;
        for (int g1 = 0; g1 < num_grid; g1++)
        {
            for (int g0 = 0; g0 < num_grid; g0++)
            {
                grid_strides.push_back((GridAndStride){g0, g1, stride});
            }
        }
    }
}



static inline float sigmoid(float x) {
    return static_cast(1.f / (1.f + exp(-x)));
}

static void generate_yolox_proposals(std::vector grid_strides, const ncnn::Mat& feat_blob, float prob_threshold, std::vector& objects)
{
    const int num_grid = feat_blob.h;
    fprintf(stderr, "output height: %d, width: %d, channels: %d, dims:%dn", feat_blob.h, feat_blob.w, feat_blob.c, feat_blob.dims);

    const int num_class = feat_blob.w - 5;

    const int num_anchors = grid_strides.size();

    const float* feat_ptr = feat_blob.channel(0);
    for (int anchor_idx = 0; anchor_idx < num_anchors; anchor_idx++)
    {
        const int grid0 = grid_strides[anchor_idx].grid0;
        const int grid1 = grid_strides[anchor_idx].grid1;
        const int stride = grid_strides[anchor_idx].stride;

        // yolox/models/yolo_head.py decode logic
        //  outputs[..., :2] = (outputs[..., :2] + grids) * strides
        //  outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides
        float x_center = (feat_ptr[0] + grid0) * stride;
        float y_center = (feat_ptr[1] + grid1) * stride;
        float w = exp(feat_ptr[2]) * stride;
        float h = exp(feat_ptr[3]) * stride;
        float x0 = x_center - w * 0.5f;
        float y0 = y_center - h * 0.5f;

        float box_objectness = feat_ptr[4];
        for (int class_idx = 0; class_idx < num_class; class_idx++)
        {
            float box_cls_score = feat_ptr[5 + class_idx];
            float box_prob = sigmoid(box_objectness) * sigmoid(box_cls_score);   // 计算置信度必须要sigmoid处理
            if (box_prob > prob_threshold)
            {
                BoxInfo obj;
                obj.x = x0;
                obj.y = y0;
                obj.w = w;
                obj.h = h;
                obj.label = class_idx;
                obj.score = sigmoid(box_cls_score);

                objects.push_back(obj);
            }

        } // class loop
        feat_ptr += feat_blob.w;

    } // point anchor loop
}


bool yolox::hasGPU = true;
bool yolox::toUseGPU = true;
yolox *yolox::detector = nullptr;

yolox::yolox(AAssetManager *mgr, const char *param, const char *bin, bool useGPU) {
    hasGPU = ncnn::get_gpu_count() > 0;
    toUseGPU = hasGPU && useGPU;

    Net = new ncnn::Net();
    // opt 需要在加载前设置
    Net->opt.use_vulkan_compute = toUseGPU;  // gpu
    Net->opt.use_fp16_arithmetic = true;  // fp16运算加速

    // 注册自定义层
    Net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator);

    Net->load_param(mgr, param);
    Net->load_model(mgr, bin);
}

yolox::~yolox() {
    Net->clear();
    delete Net;
}

std::vector yolox::detect(JNIEnv *env, jobject image,jdouble threshold, jdouble nms_threshold) {
    AndroidBitmapInfo img_size;
    AndroidBitmap_getInfo(env, image, &img_size);
    int w = img_size.width;
    int h = img_size.height;

    //设置置信度阈值、nms阈值、strides和目标尺寸
//    const float threshold = 0.3f;
//    const float nms_threshold = 0.5f;
    std::vector strides = {8,16,32};

    //resize到目标尺寸
    float scale = 1.f;
    if (w > h) {
        scale = (float) input_size / w;
        w = input_size;
        h = h * scale;
    } else {
        scale = (float) input_size / h;
        h = input_size;
        w = w * scale;
    }
    ncnn::Mat in_net = ncnn::Mat::from_android_bitmap_resize(env, image, ncnn::Mat::PIXEL_RGBA2RGB, w, h);

    int wpad = input_size - w;
    int hpad = input_size - h;
    ncnn::Mat in_pad;
    ncnn::copy_make_border(in_net, in_pad, 0, hpad, 0, wpad , ncnn::BORDER_CONSTANT,128.f); //填充数值与训练时一致

    //in_pad = Calibration(in_pad);
//    float mean[3] = {0, 0, 0};
//    float norm[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f};
//    in_pad.substract_mean_normalize(mean, norm);

    std::vector objects;
    {
        auto ex = Net->create_extractor();
        ex.set_light_mode(true);        
        ex.set_num_threads(1);     
        if (toUseGPU) {  // 消除提示
            ex.set_vulkan_compute(toUseGPU);
        }
        ex.input("input", in_pad);
        std::vector proposals;
        {
            ncnn::Mat out;
            ex.extract("output", out);
            std::vector grid_strides;
            generate_grids_and_stride(input_size, strides, grid_strides);
            generate_yolox_proposals(grid_strides, out, threshold, proposals);
        }

        qsort_descent_inplace(proposals);

        std::vector picked;
        nms_sorted_bboxes(proposals, picked, nms_threshold);
        int count = picked.size();

        objects.resize(count);
        for (int i = 0; i < count; i++) {
            objects[i] = proposals[picked[i]];

            // adjust offset to original unpadded
            float x0 = (objects[i].x) / scale;
            float y0 = (objects[i].y) / scale;
            float x1 = (objects[i].x + objects[i].w) / scale;
            float y1 = (objects[i].y + objects[i].h) / scale;

            // clip
            x0 = std::max(std::min(x0, (float) (img_size.width - 1)), 0.f);
            y0 = std::max(std::min(y0, (float) (img_size.height - 1)), 0.f);
            x1 = std::max(std::min(x1, (float) (img_size.width - 1)), 0.f);
            y1 = std::max(std::min(y1, (float) (img_size.height - 1)), 0.f);

            objects[i].x = x0;
            objects[i].y = y0;
            objects[i].w = x1 - x0;
            objects[i].h = y1 - y0;
        }

    }
    return objects;

}


    }
}

static void qsort_descent_inplace(std::vector &faceobjects) {
    if (faceobjects.empty()) {
        return;
    }
    qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1);
}

static void nms_sorted_bboxes(const std::vector& faceobjects, std::vector& picked, float nms_threshold)
{
    picked.clear();

    const int n = faceobjects.size();

    std::vector areas(n);
    for (int i = 0; i < n; i++)
    {
        areas[i] = faceobjects[i].w * faceobjects[i].h;
    }

    for (int i = 0; i < n; i++)
    {
        const BoxInfo& a = faceobjects[i];

        int keep = 1;
        for (int j = 0; j < (int)picked.size(); j++)
        {
            const BoxInfo& b = faceobjects[picked[j]];

            // intersection over union
            float inter_area = intersection_area(a, b);
            float union_area = areas[i] + areas[picked[j]] - inter_area;
            // float IoU = inter_area / union_area
            if (inter_area / union_area > nms_threshold)
                keep = 0;
        }

        if (keep)
            picked.push_back(i);
    }
}

static void generate_grids_and_stride(const int target_size, std::vector& strides, std::vector& grid_strides)
{
    for (auto stride : strides)
    {
        int num_grid = target_size / stride;
        for (int g1 = 0; g1 < num_grid; g1++)
        {
            for (int g0 = 0; g0 < num_grid; g0++)
            {
                grid_strides.push_back((GridAndStride){g0, g1, stride});
            }
        }
    }
}

static inline float sigmoid(float x) {
    return static_cast(1.f / (1.f + exp(-x)));
}

static void generate_yolox_proposals(std::vector grid_strides, const ncnn::Mat& feat_blob, float prob_threshold, std::vector& objects)
{
    const int num_grid = feat_blob.h;
    fprintf(stderr, "output height: %d, width: %d, channels: %d, dims:%dn", feat_blob.h, feat_blob.w, feat_blob.c, feat_blob.dims);

    const int num_class = feat_blob.w - 5;

    const int num_anchors = grid_strides.size();

    const float* feat_ptr = feat_blob.channel(0);
    for (int anchor_idx = 0; anchor_idx < num_anchors; anchor_idx++)
    {
        const int grid0 = grid_strides[anchor_idx].grid0;
        const int grid1 = grid_strides[anchor_idx].grid1;
        const int stride = grid_strides[anchor_idx].stride;

        // yolox/models/yolo_head.py decode logic
        //  outputs[..., :2] = (outputs[..., :2] + grids) * strides
        //  outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides
        float x_center = (feat_ptr[0] + grid0) * stride;
        float y_center = (feat_ptr[1] + grid1) * stride;
        float w = exp(feat_ptr[2]) * stride;
        float h = exp(feat_ptr[3]) * stride;
        float x0 = x_center - w * 0.5f;
        float y0 = y_center - h * 0.5f;

        float box_objectness = feat_ptr[4];
        for (int class_idx = 0; class_idx < num_class; class_idx++)
        {
            float box_cls_score = feat_ptr[5 + class_idx];
            //float box_prob = box_objectness * box_cls_score;
            float box_prob = sigmoid(box_objectness) * sigmoid(box_cls_score);   // 计算置信度必须要sigmoid处理
            if (box_prob > prob_threshold)
            {
                BoxInfo obj;
                obj.x = x0;
                obj.y = y0;
                obj.w = w;
                obj.h = h;
                obj.label = class_idx;
                obj.score = box_prob;

                objects.push_back(obj);
            }
        } // class loop
        feat_ptr += feat_blob.w;
    } // point anchor loop
}


bool yolox::hasGPU = true;
bool yolox::toUseGPU = true;
yolox *yolox::detector = nullptr;

yolox::yolox(AAssetManager *mgr, const char *param, const char *bin, bool useGPU) {
    hasGPU = ncnn::get_gpu_count() > 0;
    toUseGPU = hasGPU && useGPU;

    Net = new ncnn::Net();
    // opt 需要在加载前设置
    Net->opt.use_vulkan_compute = toUseGPU;  // gpu
    Net->opt.use_fp16_arithmetic = true;  // fp16运算加速
#if ENABLE_CUSTOM_LAYER
    // 注册自定义层
    Net->register_custom_layer("YoloV5Focus", YoloV5Focus_layer_creator);
#endif
    Net->load_param(mgr, param);
    Net->load_model(mgr, bin);
}

yolox::~yolox() {
    Net->clear();
    delete Net;
}

std::vector yolox::detect(JNIEnv *env, jobject image) {
    AndroidBitmapInfo img_size;
    AndroidBitmap_getInfo(env, image, &img_size);
    int w = img_size.width;
    int h = img_size.height;

    //设置置信度阈值、nms阈值、strides和目标尺寸
    const float threshold = 0.3f;
    const float nms_threshold = 0.5f;
    std::vector strides = {8,16,32};
    const int target_size = 256;

    //resize到目标尺寸
    float scale = 1.f;
    if (w > h) {
        scale = (float) input_size / w;
        w = input_size;
        h = h * scale;
    } else {
        scale = (float) input_size / h;
        h = input_size;
        w = w * scale;
    }
    ncnn::Mat in_net = ncnn::Mat::from_android_bitmap_resize(env, image, ncnn::Mat::PIXEL_RGBA2RGB, w, h);

    int wpad = target_size - w;
    int hpad = target_size - h;
    ncnn::Mat in_pad;
    ncnn::copy_make_border(in_net, in_pad, 0, hpad, 0, wpad , ncnn::BORDER_CONSTANT,128.f); //填充数值与训练时一致

//    float mean[3] = {0, 0, 0};
//    float norm[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f};
//    in_pad.substract_mean_normalize(mean, norm);

    std::vector objects;
    {
        auto ex = Net->create_extractor();
        ex.set_light_mode(true);
        ex.set_num_threads(4);
        if (toUseGPU) {  // 消除提示
            ex.set_vulkan_compute(toUseGPU);
        }
        ex.input("input", in_pad);
        std::vector proposals;
        {
            ncnn::Mat out;
            ex.extract("output", out);
            std::vector grid_strides;
            generate_grids_and_stride(target_size, strides, grid_strides);
            generate_yolox_proposals(grid_strides, out, threshold, proposals);
        }

        qsort_descent_inplace(proposals);

        std::vector picked;
        nms_sorted_bboxes(proposals, picked, nms_threshold);
        int count = picked.size();

        objects.resize(count);
        for (int i = 0; i < count; i++) {
            objects[i] = proposals[picked[i]];

            // adjust offset to original unpadded
            float x0 = (objects[i].x - (wpad / 2)) / scale;
            float y0 = (objects[i].y - (hpad / 2)) / scale;
            float x1 = (objects[i].x + objects[i].w - (wpad / 2)) / scale;
            float y1 = (objects[i].y + objects[i].h - (hpad / 2)) / scale;

            // clip
            x0 = std::max(std::min(x0, (float) (img_size.width - 1)), 0.f);
            y0 = std::max(std::min(y0, (float) (img_size.height - 1)), 0.f);
            x1 = std::max(std::min(x1, (float) (img_size.width - 1)), 0.f);
            y1 = std::max(std::min(y1, (float) (img_size.height - 1)), 0.f);

            objects[i].x = x0;
            objects[i].y = y0;
            objects[i].w = x1 - x0;
            objects[i].h = y1 - y0;
        }
    }
}

至此，c++及jni接口部分已经完成，DeepLearning库中可以通过创建新的java类，来调用YOLOX类中的Detect函数来调用深度学习模型。

写博客不易，看官给个赞再走呗，哈哈哈哈！！！！

暑期编程PK赛得CSDN机械键盘等精美礼品！

YOLOX从训练到NCNN部署全流程(6)——模型部署

Java相关栏目本月热门文章