YOLOX从训练到NCNN部署全流程(1)——环境配置
YOLOX从训练到NCNN部署全流程(2)——代码训练
YOLOX从训练到NCNN部署全流程(3)——模型转化与量化
YOLOX从训练到NCNN部署全流程(4)——模型修改
YOLOX从训练到NCNN部署全流程(5)——创建demo工程
YOLOX从训练到NCNN部署全流程(6)——模型部署
YOLOX从训练到NCNN部署全流程(7)——模型部署测试
YOLOX从训练到NCNN部署全流程(8)——生成so库与jar包
YOLOX从训练到NCNN部署全流程(9)——使用so库和jar包
上述步骤已经把库的部分完成了，但是还是要测试一下能不能正常运行。这里选择直接把库的module作为app module的项目依赖，这样一来，每次我们运行app module时，都是以DeepLearning 库作为依赖实时编译的，不必我们每次都要导出so库和jar包。方便修改。
在app module下已经有一个初始化的MainActivity。直接用它显示DeepLearning库处理后的图像。接下来要将DeepLearning作为app的依赖。在app上点击右键，点击Open module settings，依次点击：Dependencies > app > + >Module Dependency。
选择Module Dependency后，会出来目前这个工程里的所有module，选择DeepLearning这个module。
点击OK之后，再点击OK，这个时候依赖就添加好了。那么到底什么地方发生了改变呢？其实就是在app的build.gradle文件中的dependencies节点下加了一句：
dependencies {
implementation 'androidx.appcompat:appcompat:1.3.0'
implementation 'com.google.android.material:material:1.4.0'
implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
implementation project(path: ':DeepLearning') //就是这句
testImplementation 'junit:junit:4.13.2'
androidTestImplementation 'androidx.test.ext:junit:1.1.3'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
如果不想这么麻烦也可以直接把这句加进去在sync一下gradle就可以了。
然后，在DeepLearning库中创建Box类和Detection类用于定义Box和检测过程，同样也可创建其他类用于更复杂的前处理和后处理。
package com.lqy.deeplearning;
import android.graphics.Color;
import android.graphics.RectF;
import java.util.Random;
public class Box {
public float x0, y0, w, h;
public int label;
public float score;
private static String[] labels = {"iris", "pipul", "shut-eye"};
public Box(float x,float y, float w, float h, int label, float score) {
this.x0 = x;
this.y0 = y;
this.w = w;
this.h = h;
this.label = label;
this.score = score;
}
//得出Bbox ，但需要输入的是左上角和右下角两点坐标
public RectF getRect(){
return new RectF(x0,y0,w,h);
}
public String getLabel(){
return labels[label];
}
public float getScore(){
return score;
}
public int getColor(){
Random random = new Random(label);
return Color.argb(255,random.nextInt(256),random.nextInt(256),random.nextInt(256));
}
}
package com.lqy.deeplearning;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import java.util.List;
public class Detection {
public static int width;
public static int height;
public static double threshold = 0.6;
public static double nms_threshold = 0.3;
public static Bitmap Location(Bitmap bitmap) {
width = bitmap.getWidth();
height = bitmap.getHeight();
//------------------------------------------------------------------------
final Bitmap mutableBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true); // 用于显示到控件
Canvas canvas = new Canvas(mutableBitmap); // 用于画出BOX
Bitmap left_bitmap = Bitmap.createBitmap(bitmap, 0, 0, width, height);
Box[] result_l = YOLOX.Detect(left_bitmap, threshold, nms_threshold); // 得到检测结果
for (Box box : result_l) {
DrawBox(canvas, width, height, box); // 画BOX
}
return mutableBitmap;
}
//=================绘制box=====================//
public static void DrawBox(Canvas canvas, int width, int height, Box box){
//================================================
Paint boxPaint = new Paint(); // 画检测框
boxPaint.setStyle(Paint.Style.STROKE);
boxPaint.setStrokeWidth(4 * width / 800);
boxPaint.setTextSize(40 * width / 800);
Paint textbgpaint = new Paint(); // 填充label和scale的背景
textbgpaint.setColor(Color.WHITE);
textbgpaint.setStyle(Paint.Style.FILL);
textbgpaint.setStrokeWidth(4 * width / 800);
textbgpaint.setTextSize(40 * width / 800);
Paint textpaint = new Paint(); // 画label和scale
textpaint.setColor(Color.BLACK);
textpaint.setTextAlign(Paint.Align.LEFT);
textpaint.setStrokeWidth(4 * width / 800);
textpaint.setTextSize(20 * width / 800);
Paint signpaint = new Paint(); // 画图像中心和iris中心
signpaint.setColor(Color.RED);
signpaint.setStyle(Paint.Style.STROKE);
signpaint.setStrokeWidth(width / 800);
signpaint.setTextSize(40 * width / 800);
// // 画图像中心
//左图
canvas.drawLine(width/4-10,height/2,width/4+10,height/2, signpaint);
canvas.drawLine(width/4,height/2-10,width/4,height/2+10, signpaint);
canvas.drawArc(width/4-10,height/2-10,width/4+10,height/2+10,0,360,false,signpaint);
//右图
canvas.drawLine(width*3/4-10,height/2,width*3/4+10,height/2, signpaint);
canvas.drawLine(width*3/4,height/2-10,width*3/4,height/2+10, signpaint);
canvas.drawArc(width*3/4-10,height/2-10,width*3/4+10,height/2+10,0,360,false,signpaint);
// 画iris中心圆圈
float left = box.x0 + box.w/2 - 2;
float top = box.y0 + box.h/2 - 2;
float right = box.x0 + box.w/2 + 2;
float bottom = box.y0 + box.h/2 + 2;
canvas.drawArc(left,top,right,bottom,0,360,false,signpaint);
boxPaint.setColor(box.getColor());
canvas.drawRect(box.x0, box.y0, box.x0 + box.w, box.y0 + box.h, boxPaint);
// draw filled text inside image
{
String text = box.getLabel() + " = " + String.format("%.1f", box.score * 100) + "%";
float text_width = textpaint.measureText(text);
float text_height = - textpaint.ascent() + textpaint.descent();
float x = box.x0;
float y = box.y0 - text_height;
if (y < 0)
y = 0;
if (x + text_width > width)
x = width - text_width;
canvas.drawRect(x, y, x + text_width, y + text_height, textbgpaint);
canvas.drawText(text, x, y - textpaint.ascent(), textpaint);
}
}
}
最后在app的MainActivity里就可以调用DeepLearning里YOLOX里的 Detect方法检测图像。
package com.lqy.demo;
import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.camera2.interop.Camera2Interop;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Preview;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.camera.view.PreviewView;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import android.Manifest;
import android.annotation.SuppressLint;
import android.content.pm.PackageManager;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.YuvImage;
import android.hardware.camera2.CaptureRequest;
import android.os.Bundle;
import android.util.Range;
import android.util.Size;
import android.widget.ImageView;
import com.google.common.util.concurrent.ListenableFuture;
import com.lqy.deeplearning.Detection; // DeepLearning库中的类
import com.lqy.deeplearning.YOLOX; // DeepLearning库中的类
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class MainActivity extends AppCompatActivity {
private static final int REQUEST_CAMERA = 1;
private static String[] PERMISSIONS_CAMERA = {
Manifest.permission.CAMERA }; // 相机使用权限
// 用于图像预览
private PreviewView previewView;
private ImageView resultImageView;
private ProcessCameraProvider cameraProvider = null;
private ExecutorService cameraExecutor;
private ListenableFuture cameraProviderFuture;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
// 相机权限申请
int permission = ActivityCompat.checkSelfPermission(this, Manifest.permission.CAMERA);
if (permission != PackageManager.PERMISSION_GRANTED) {
ActivityCompat.requestPermissions(
this,
PERMISSIONS_CAMERA,
REQUEST_CAMERA
);
finish();
}
YOLOX.Init(getAssets());
previewView = findViewById(R.id.view_preview); // 绑定控件
resultImageView = findViewById(R.id.imageView);
//==================调用bindPreview函数，bindPreview函数包括预览、图像处理等===================
cameraProviderFuture = ProcessCameraProvider.getInstance(this);
cameraProviderFuture.addListener(() -> {
try {
ProcessCameraProvider cameraProvider = cameraProviderFuture.get();
cameraExecutor= Executors.newSingleThreadExecutor();
bindPreview(cameraProvider);
} catch (ExecutionException | InterruptedException e) {
// No errors need to be handled for this Future.
// This should never be reached.
}
}, ContextCompat.getMainExecutor(this));
//=======================================================================================
}
//==========================================================================
@SuppressLint("UnsafeOptInUsageError")
private void bindPreview(@NonNull ProcessCameraProvider cameraProvider) { // 创建一个相机的管理器，相当于一个主管道。
//1 图像预览接口
Preview preview= new Preview.Builder()
.setTargetResolution(new Size(1600,600))
.build();
//2 选择相机（前置后置）
CameraSelector cameraSelector = new CameraSelector.Builder()
.requireLensFacing(CameraSelector.LENS_FACING_FRONT)
.build();
//3 图像分析接口
ImageAnalysis.Builder builder = new ImageAnalysis.Builder();
Camera2Interop.Extender ext = new Camera2Interop.Extender<>(builder); // Camera2扩展API
//ext.setCaptureRequestOption(CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_OFF); // 自动曝光设置
ext.setCaptureRequestOption(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, new Range(30, 60)); // 设置相机获取图像的帧率范围
ext.setCaptureRequestOption(CaptureRequest.CONTROL_AE_ANTIBANDING_MODE, CaptureRequest.CONTROL_AE_ANTIBANDING_MODE_50HZ); // 设置电力线频率 50Hz、60Hz
builder.setTargetResolution(new Size(1600,600));
//builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888); //设置图像格式
builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); // 操作模式：非阻塞
//builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_BLOCK_PRODUCER); // 操作模式：阻塞
//builder.setImageQueueDepth(5); // 操作模式：阻塞、图像队列深度为6
ImageAnalysis myAnalyzer = builder.build();
myAnalyzer.setAnalyzer(cameraExecutor, new ImageAnalysis.Analyzer() {
@Override
public void analyze(@NonNull ImageProxy imageProxy) {
int rotationDegrees = imageProxy.getImageInfo().getRotationDegrees();
// 处理图像的函数
AnalyzeImg(imageProxy,rotationDegrees);
// after done, release the ImageProxy object
imageProxy.close();
}
});
cameraProvider.unbindAll();
//4 把相机的信息还得告诉预览窗口以做好准备显示数据,将 surfaceProvider 绑定至 preview 用例来启动预览
preview.setSurfaceProvider(previewView.getSurfaceProvider());
//5 把我们需要的这三个接口安装到相机管理器的主线路上，实现截取数据的目的
cameraProvider.bindToLifecycle(MainActivity.this, cameraSelector, myAnalyzer, preview);
}
private void AnalyzeImg(@NonNull ImageProxy image, final int rotationDegrees) {
Matrix matrix = new Matrix();
matrix.postRotate(rotationDegrees);
matrix.postScale(-1,1);
// ===========在此处编写图像处理代码===========//
// 图像格式转换
final Bitmap bitmapsrc = imageToBitmap(image); // 格式转换
int width = bitmapsrc.getWidth();
int height = bitmapsrc.getHeight();
Bitmap bitmap = Bitmap.createBitmap(bitmapsrc, 0, 0, width, height, matrix, false);
Bitmap mutableBitmap = Detection.Location(bitmap);
runOnUiThread(new Runnable() {
@Override
public void run() {
resultImageView.setImageBitmap(mutableBitmap);
}
});
}
@Override
protected void onDestroy() {
cameraProvider.unbindAll();
super.onDestroy();
}
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
for (int result : grantResults) {
if (result != PackageManager.PERMISSION_GRANTED) {
this.finish();
}
}
}
//==============image转Bitmap==================//
public static Bitmap imageToBitmap(ImageProxy image) {
ImageProxy.PlaneProxy[] planes = image.getPlanes();
ImageProxy.PlaneProxy y = planes[0];
ImageProxy.PlaneProxy u = planes[1];
ImageProxy.PlaneProxy v = planes[2];
ByteBuffer yBuffer = y.getBuffer();
ByteBuffer uBuffer = u.getBuffer();
ByteBuffer vBuffer = v.getBuffer();
int ySize = yBuffer.remaining();
int uSize = uBuffer.remaining();
int vSize = vBuffer.remaining();
byte[] nv21 = new byte[ySize + uSize + vSize];
// U and V are swapped
yBuffer.get(nv21, 0, ySize);
vBuffer.get(nv21, ySize, vSize);
uBuffer.get(nv21, ySize + vSize, uSize);
YuvImage yuvImage = new YuvImage(nv21, ImageFormat.NV21, image.getWidth(), image.getHeight(), null);
ByteArrayOutputStream out = new ByteArrayOutputStream();
yuvImage.compressToJpeg(new Rect(0, 0, yuvImage.getWidth(), yuvImage.getHeight()), 100, out);
byte[] imageBytes = out.toByteArray();
return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);
}
}
然后在cpp同级目录中创建一个assets文件夹，其中放转换好的深度学习模型文件（.param， .bin）。千万别忘了！！！
运行后显示检测框的图像，就说明成功了。
写博客不易，看官给个赞再走呗，哈哈哈哈！！！！
