Java 调用Python+Opencv实现图片定位

做过移动端ui自动化的小伙伴，就会发现很多控件的元素是一样的或者是找不到的，为了解决这个痛点，于是通过图片灰度处理返回坐标x，y找到控件的位置。再结合pytest+接口+UI断言整体项目思路。
1.接下来我们主要说一下基于opencv图片识别寻找控件坐标
2. 我们使用两个图，一个是移动端截图，一个是控件的图，

Java代码如下

    public static void main(String[] args) {
        run_opencv("D:/Search.png", "D:/Setting.png",50,50);
    }

    
    public static HashMap run_opencv(String picturePath,String PagePicturePath,int xPercent,int yPercent) {
        HashMap location = new HashMap<>();
        try {
            //x,y = get_center_location('D:/Battery.png', 'D:/Setting.png',0,0)
            
            String cmds = String.format("python D:\Project\Program\PythonWorkspace\myProject\python_project\apptest\myopencv\other_case\get_location_by_opencv.py %s %s %d %d", picturePath,PagePicturePath,xPercent,yPercent);

            System.out.println("Executing python script for picture location.");
            Process pcs = Runtime.getRuntime().exec(cmds);
            pcs.waitFor();
            Thread.sleep(1000);
            
            // 定义Python脚本的返回值
            String result = null;
            // 获取CMD的返回流
            BufferedInputStream in = new BufferedInputStream(pcs.getInputStream());// 字符流转换字节流
            BufferedReader br = new BufferedReader(new InputStreamReader(in));// 这里也可以输出文本日志
            String lineStr = null;
            while ((lineStr = br.readLine()) != null) {
                result = lineStr;//Python 代码中print的数据就是返回值
                //xLocation: 147
                //yLocation: 212
                if(lineStr.contains("xLocation")) {
                    int x =  Integer.parseInt(lineStr.split(":")[1].trim());
                    location.put("x", x);
                }
                if(lineStr.contains("yLocation")) {
                    int x =  Integer.parseInt(lineStr.split(":")[1].trim());
                    location.put("y", x);
                }
            }
            // 关闭输入流
            br.close();
            in.close();
            System.out.println(location.toString());
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return location;
    }

Python代码：

# -*- encoding=utf-8 -*-

__author__ = 'Jeff.xie'


import cv2
import os
import sys
import time

#获取移动端图片
def screencap():
    cmd = "adb root"
    cmd1 = "adb shell /system/bin/screencap -p /sdcard/da.png"
    cmd2 = "adb pull /sdcard/da.png "
    os.system(cmd)
    time.sleep(1)
    os.system(cmd1)
    time.sleep(2)
    os.system(cmd2)


def _tran_canny(image):
    """消除噪声"""
    image = cv2.GaussianBlur(image, (3, 3), 0)
    return cv2.Canny(image, 50, 150)


def get_center_location(img_slider_path,image_background_path,x_percent,y_percent):
    """get_center_location"""

    # print("img_slider_path: "+img_slider_path)
    # print("image_background_path: "+image_background_path)
    # print("x_percent: "+str(x_percent))
    # print("y_percent: "+str(y_percent))
    # java传递过来的参数都是str类型，所以需要强转成int类型
    xper = int(x_percent)
    yper = int(y_percent)

    # # 参数0是灰度模式

    image = cv2.imread(img_slider_path, 0)
    template = cv2.imread(image_background_path, 0)

    # 寻找最佳匹配
    res = cv2.matchTemplate(_tran_canny(image), _tran_canny(template), cv2.TM_CCOEFF_NORMED)
    # 最小值，最大值，并得到最小值, 最大值的索引
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)

    #获得背景图像高和宽
    src_img = cv2.imread(image_background_path,cv2.IMREAD_GRAYSCALE)
    h,w = src_img.shape
    # print("src_img_h:",h)
    # print("src_img_w:",w)


    #获得需要寻找图像高和宽
    des_img = cv2.imread(img_slider_path,cv2.IMREAD_GRAYSCALE)
    des_img_h,des_img_w = des_img.shape
    # print("des_img_h:",des_img_h)
    # print("des_img_w:",des_img_w)

    trows,tcols = image.shape[:2]  #获得图片的宽度,两种方式都可以
    # print(trows)
    # print(tcols)

    top_left = max_loc[0]  # 横坐标
    # 展示圈出来的区域
    x, y = max_loc
    # max_loc这个是最大值，所以获取的是x,y位置坐标，小图片右下角的位置，左上角的要用min_loc
    # print("x:",x)
    # print("y:",y)
    xLocation = x + int(des_img_w*xper/100)
    yLocation = y + int(des_img_h*yper/100)
    print("xLocation: "+str(xLocation))
    print("yLocation: "+str(yLocation))
    # print(max_loc)
    # print(min_loc)
    # print(min_val)
    # print(max_val)
    return xLocation,yLocation

    # w, h = image.shape[::-1]  # 宽高
    # cv2.rectangle(template, (x, y), (x + w, y + h), (7, 249, 151), 2)
    # return top_left


if __name__ == '__main__':
     # x,y = get_center_location('D:/Battery.png', 'D:/Setting.png',40,39)
    img_slider_path = sys.argv[1]
    image_background_path = sys.argv[2]
    x_percent = sys.argv[3]
    y_percent = sys.argv[4]
    get_center_location(img_slider_path, image_background_path,x_percent,y_percent)

    # 0%
    # getx: 29
    # gety: 1390

    # 50%
    # getx: 49
    # gety: 1415

    # 100%
    # getx: 69
    # gety: 1441