智能虚拟鼠标

MediaPipe Holistic分别利用MediaPipe Pose，MediaPipe Face Mesh和MediaPipe Hands中的姿势，面部和手界标模型来生成总共543个界标（每手33个姿势界标，468个脸部界标和21个手界标）。本文将用MediaPipe Hands来完成AI虚拟鼠标。

引入库

import cv2
import numpy as np
import HandTrackingModule as htm
import autopy

其中 HandTrackingModule是大神编写的库，用于获取手指的坐标和检查手指是否伸出，调用autopy库实现鼠标点击功能。

HandTrackingModule

Youtube: http://www.youtube.com/c/MurtazasWorkshopRoboticsandAI

Website: https://www.computervision.zone

1. 检测手部得到手指关键点的坐标
success, img = cap.read()
img = detector.findHands(img)
lmList, bbox = detector.findPosition(img)

2,得到食指和中指的部分坐标
if len(lmList)!= 0:
    x1, y1 = lmList[8][1:]
    x2, y2 = lmList[12][1:]
    #print(x1, y1, x2, y2)

3. 检查哪个手指伸出
fingers = detector.fingersUp()

 4. 只有食指伸出时移动鼠标
if fingers[1]==1 and fingers[2] == 0:

 5. 转换坐标
x3 = np.interp(x1, (frameR, wCam-frameR),(0,wScr))
y3 = np.interp(y1, (frameR, hCam-frameR), (0, hScr))

7. 移动鼠标
autopy.mouse.move(wScr-x3, y3)
cv2.circle(img, (x1, y1), 15, (255, 0,255),cv2.FILLED)
plocX, plocY = clocX, clocY

8. 当食指和中指同时伸出时，鼠标不动
if fingers[1] == 1 and fingers[2] == 1:
9. 中指和食指之间的距离
length, img, lineInfo = detector.findDistance(8, 12, img)

10. 当两指间的距离小于限定值时点击
if length < 40:  # 当距离小于40个像素点时为左键
    cv2.circle(img, (lineInfo[4], lineInfo[5]), 15, (0, 255, 0), cv2.FILLED)
    autopy.mouse.click()

代码展示

import cv2
import numpy as np
import HandTrackingModule as htm
import time
import autopy

wCam, hCam = 640,480
frameR = 100
smoothening = 5
pTime = 0
plocX, plocY = 0, 0
clocX, clocY = 0, 0
cap = cv2.VideoCapture(0)
cap.set(3, wCam)
cap.set(4, hCam)

detector = htm.handDetector(maxHands=1)
wScr, hScr = autopy.screen.size()

while True:
    
    success, img = cap.read()
    img = detector.findHands(img)
    lmList, bbox = detector.findPosition(img)

 
    if len(lmList)!= 0:
        x1, y1 = lmList[8][1:]
        x2, y2 = lmList[12][1:]      
   
        finger= detector.fingersUp()
        print(fingers)
        cv2.rectangle(img, (frameR, frameR), (wCam - frameR, hCam - frameR), (255, 0, 255), 2)
        
        if fingers[1]==1 and fingers[2] == 0:
        
            x3 = np.interp(x1, (frameR, wCam-frameR),(0,wScr))
            y3 = np.interp(y1, (frameR, hCam-frameR), (0, hScr))
        
            clocX = plocX +(x3 - plocX) / smoothening
            clocY = plocY + (y3 - plocY) / smoothening

            
            autopy.mouse.move(wScr-x3, y3)
            cv2.circle(img, (x1, y1), 15, (255, 0,255),cv2.FILLED)
            plocX, plocY = clocX, clocY

        
        if fingers[1] == 1 and fingers[2] == 1:          
            length, img, lineInfo = detector.findDistance(8, 12, img)         
            if length < 40:  
                cv2.circle(img, (lineInfo[4], lineInfo[5]), 15, (0, 255, 0), cv2.FILLED)
                autopy.mouse.click()


    cTime = time.time()
    fps = 1 / (cTime - pTime)
    pTime = cTime
    cv2.putText(img, str(int(fps)), (20, 50), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 255), 3)



    cv2.imshow("Img", img)
    if cv2.waitKey(1) == ord('q'):  
        break

调试结果

20220109_182406_哔哩哔哩_bilibili