class_15_1_hand_count_mouse.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center> 👉 class_15_1 » _Gesture Detection/Recognition & Mouse_ </center>   \n",
    "\n",
    "\n",
    "# Gesture Detection and Recognition w/ MediaPipe\n",
    "\n",
    "https://github.com/ANANTH-SWAMY/NUMBER-DETECTION-WITH-MEDIAPIPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import mediapipe\n",
    "import time\n",
    "\n",
    "ctime=0\n",
    "ptime=0\n",
    "\n",
    "cap=cv2.VideoCapture(0)\n",
    "\n",
    "medhands=mediapipe.solutions.hands\n",
    "hands=medhands.Hands(max_num_hands=1,min_detection_confidence=0.7)\n",
    "draw=mediapipe.solutions.drawing_utils\n",
    "\n",
    "while True:\n",
    "    success, img=cap.read()\n",
    "    img = cv2.flip(img,1)\n",
    "    imgrgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)    \n",
    "    res = hands.process(imgrgb)\n",
    "    \n",
    "    lmlist=[]\n",
    "    tipids=[4,8,12,16,20] #list of all landmarks of the tips of fingers\n",
    "    \n",
    "    cv2.rectangle(img,(20,350),(90,440),(0,255,204),cv2.FILLED)\n",
    "    cv2.rectangle(img,(20,350),(90,440),(0,0,0),5)\n",
    "    \n",
    "    if res.multi_hand_landmarks:\n",
    "        for handlms in res.multi_hand_landmarks:\n",
    "            for id,lm in enumerate(handlms.landmark):\n",
    "                \n",
    "                h,w,c= img.shape\n",
    "                cx,cy=int(lm.x * w) , int(lm.y * h)\n",
    "                lmlist.append([id,cx,cy])\n",
    "                if len(lmlist) != 0 and len(lmlist)==21:\n",
    "                    fingerlist=[]\n",
    "                    \n",
    "                    #thumb and dealing with flipping of hands\n",
    "                    if lmlist[12][1] > lmlist[20][1]:\n",
    "                        if lmlist[tipids[0]][1] > lmlist[tipids[0]-1][1]:\n",
    "                            fingerlist.append(1)\n",
    "                        else:\n",
    "                            fingerlist.append(0)\n",
    "                    else:\n",
    "                        if lmlist[tipids[0]][1] < lmlist[tipids[0]-1][1]:\n",
    "                            fingerlist.append(1)\n",
    "                        else:\n",
    "                            fingerlist.append(0)\n",
    "                    \n",
    "                    #others\n",
    "                    for id in range (1,5):\n",
    "                        if lmlist[tipids[id]][2] < lmlist[tipids[id]-2][2]:\n",
    "                            fingerlist.append(1)\n",
    "                        else:\n",
    "                            fingerlist.append(0)\n",
    "                    \n",
    "                    \n",
    "                    if len(fingerlist)!=0:\n",
    "                        fingercount=fingerlist.count(1)\n",
    "                    \n",
    "                    \n",
    "                    cv2.putText(img,str(fingercount),(25,430),cv2.FONT_HERSHEY_PLAIN,6,(0,0,0),5)\n",
    "                    \n",
    "                #change color of points and lines\n",
    "                draw.draw_landmarks(img,handlms,medhands.HAND_CONNECTIONS,draw.DrawingSpec(color=(0,255,204),thickness=2,circle_radius=2),draw.DrawingSpec(color=(0,0,0),thickness=2,circle_radius=3))\n",
    "    \n",
    "    #fps counter\n",
    "    ctime = time.time()\n",
    "    fps=1/(ctime-ptime)\n",
    "    ptime=ctime\n",
    "    \n",
    "    #fps display\n",
    "    cv2.putText(img,f'FPS:{str(int(fps))}',(0,12),cv2.FONT_HERSHEY_PLAIN,1,(0,255,0),1)\n",
    "          \n",
    "    cv2.imshow(\"hand gestures\",img)\n",
    "    \n",
    "    #press q to quit\n",
    "    if cv2.waitKey(1) == ord('q'):\n",
    "        break\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# American-Sign-Language-Recognition  \n",
    "https://github.com/Nam-H-Pham/American-Sign-Language-Recognition  \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Touchless Mouse w/ Hand Gesture  \n",
    "\n",
    "https://github.com/AadrianLeo/AI_hand_gester_mouse/tree/main  \n",
    "    \n",
    "AI hand gester mouse project is a AI based project in which you can assess the mouse with gestures without physically touching the mouse.   \n",
    "This project detects the hand and counts the fingers with the first finger up and all fingers down you can move the mouse and with first finger and middle finger you can click. The project is made using python language.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyautogui in c:\\users\\owner\\appdata\\roaming\\python\\python37\\site-packages (0.9.54)\n",
      "Requirement already satisfied: pyscreeze>=0.1.21 in c:\\users\\owner\\appdata\\roaming\\python\\python37\\site-packages (from pyautogui) (0.1.30)\n",
      "Requirement already satisfied: pygetwindow>=0.0.5 in c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages (from pyautogui) (0.0.9)\n",
      "Requirement already satisfied: pymsgbox in c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages (from pyautogui) (1.0.9)\n",
      "Requirement already satisfied: pytweening>=1.0.4 in c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages (from pyautogui) (1.0.7)\n",
      "Requirement already satisfied: mouseinfo in c:\\users\\owner\\appdata\\roaming\\python\\python37\\site-packages (from pyautogui) (0.1.3)\n",
      "Requirement already satisfied: pyrect in c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages (from pygetwindow>=0.0.5->pyautogui) (0.2.0)\n",
      "Requirement already satisfied: Pillow>=9.2.0 in c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages (from pyscreeze>=0.1.21->pyautogui) (9.5.0)\n",
      "Requirement already satisfied: pyperclip in c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages (from mouseinfo->pyautogui) (1.8.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -illow (c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -illow (c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -illow (c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -illow (c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -illow (c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -illow (c:\\users\\owner\\anaconda\\envs\\dl(2023)_yolo\\lib\\site-packages)\n"
     ]
    }
   ],
   "source": [
    "pip install pyautogui --user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import mediapipe as mp\n",
    "import time\n",
    "import math\n",
    "import pyautogui\n",
    "import numpy as np\n",
    "\n",
    "class handDetector():\n",
    "    def __init__(self, mode=False, maxHands=2, modelComplexity=1, detectionCon=0.5, trackCon=0.5):\n",
    "        self.mode = mode\n",
    "        self.maxHands = maxHands\n",
    "        self.modelComplex = modelComplexity\n",
    "        self.detectionCon = detectionCon\n",
    "        self.trackCon = trackCon\n",
    "\n",
    "        self.mpHands = mp.solutions.hands\n",
    "        self.hands = self.mpHands.Hands(self.mode, self.maxHands, self.modelComplex, self.detectionCon, self.trackCon)\n",
    "        self.mpDraw = mp.solutions.drawing_utils\n",
    "\n",
    "        self.tipIds = [4, 8, 12, 16, 20]\n",
    "\n",
    "    def findHands(self, img, draw=True):\n",
    "        imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        self.results = self.hands.process(imgRGB)\n",
    "        # print(results.multi_hand_landmarks)\n",
    "\n",
    "        if self.results.multi_hand_landmarks:\n",
    "            for handLms in self.results.multi_hand_landmarks:\n",
    "                if draw:\n",
    "                    self.mpDraw.draw_landmarks(img, handLms,\n",
    "                                               self.mpHands.HAND_CONNECTIONS)\n",
    "        return img\n",
    "\n",
    "    def findPosition(self, img, handNo=0, draw=True):\n",
    "        xList = []\n",
    "        yList = []\n",
    "        bbox = []\n",
    "        self.lmList = []\n",
    "        if self.results.multi_hand_landmarks:\n",
    "            myHand = self.results.multi_hand_landmarks[handNo]\n",
    "            for id, lm in enumerate(myHand.landmark):\n",
    "                # print(id, lm)\n",
    "                h, w, c = img.shape\n",
    "                cx, cy = int(lm.x * w), int(lm.y * h)\n",
    "                xList.append(cx)\n",
    "                yList.append(cy)\n",
    "                # print(id, cx, cy)\n",
    "                self.lmList.append([id, cx, cy])\n",
    "                if draw:\n",
    "                    cv2.circle(img, (cx, cy), 5, (255, 0, 0), cv2.FILLED)\n",
    "            xmin, xmax = min(xList), max(xList)\n",
    "            ymin, ymax = min(yList), max(yList)\n",
    "            bbox = xmin, ymin, xmax, ymax\n",
    "\n",
    "            if draw:\n",
    "                cv2.rectangle(img, (bbox[0] - 20, bbox[1] - 20), (bbox[2] + 20, bbox[3] + 20), (0, 225, 0), 2)\n",
    "\n",
    "        return self.lmList, bbox\n",
    "\n",
    "    def fingersUp(self):\n",
    "        fingers = []\n",
    "        # Thumb\n",
    "        if self.lmList[self.tipIds[0]][1] > self.lmList[self.tipIds[0] - 1][1]:\n",
    "            fingers.append(1)\n",
    "        else:\n",
    "            fingers.append(0)\n",
    "\n",
    "        # 4 fingers\n",
    "        for id in range(1, 5):\n",
    "            if self.lmList[self.tipIds[id]][2] < self.lmList[self.tipIds[id] - 2][2]:\n",
    "                fingers.append(1)\n",
    "            else:\n",
    "                fingers.append(0)\n",
    "        return fingers\n",
    "\n",
    "    def findDistance(self, p1, p2, img, draw = True):\n",
    "\n",
    "        x1, y1 = self.lmList[p1][1], self.lmList[p1][2]\n",
    "        x2, y2 = self.lmList[p2][1], self.lmList[p2][2]\n",
    "        cx, cy = (x1 + x2) // 2, (y1 + y2) // 2\n",
    "\n",
    "        if draw:\n",
    "            cv2.circle(img, (x1, y1), 15, (225, 0, 225), cv2.FILLED)\n",
    "            cv2.circle(img, (x2, y2), 15, (225, 0, 225), cv2.FILLED)\n",
    "            cv2.line(img, (x1, y1), (x2, y2), (225, 0, 225), 3)\n",
    "            cv2.circle(img, (cx, cy), 15, (225, 0, 225), cv2.FILLED)\n",
    "\n",
    "        length = math.hypot(x2 - x1, y2 - y1)\n",
    "        return length, img, [x1, y1, x2, y2, cx, cy]\n",
    "    \n",
    "    def findAngle(self, p1, p2, p3, img, draw=True):\n",
    "        # Get the landmarks\n",
    "        x1, y1 = self.lmList[p1][1:]\n",
    "        x2, y2 = self.lmList[p2][1:]\n",
    "        x3, y3 = self.lmList[p3][1:]\n",
    "        # Calculate the Angle\n",
    "        angle = math.degrees(math.atan2(y3 - y2, x3 - x2) - math.atan2(y1 - y2, x1 - x2))\n",
    "        if angle < 0:\n",
    "            angle += 360\n",
    "\n",
    "        # print(angle)\n",
    "\n",
    "        # Draw\n",
    "        if draw:\n",
    "            cv2.line(img, (x1, y1), (x2, y2), (255, 255, 255), 3)\n",
    "            cv2.line(img, (x3, y3), (x2, y2), (255, 255, 255), 3)\n",
    "            cv2.circle(img, (x1, y1), 15, (0, 0, 255), cv2.FILLED)\n",
    "            cv2.circle(img, (x1, y1), 15, (0, 0, 255), cv2.FILLED)\n",
    "            cv2.circle(img, (x2, y2), 15, (0, 0, 255), cv2.FILLED)\n",
    "            # cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 2)\n",
    "\n",
    "        return angle  \n",
    "\n",
    "hd = handDetector(detectionCon=0.7)\n",
    "\n",
    "wCam, hCam = 780, 640\n",
    "frameR = 100\n",
    "smoothening = 7\n",
    " \n",
    "pTime = 0\n",
    "plocX, plocY = 0, 0\n",
    "clocX, clocY = 0, 0\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "cap.set(3, wCam)\n",
    "cap.set(4, hCam)\n",
    "\n",
    "detector = handDetector(detectionCon=0.7)\n",
    "wSrc, hSrc = pyautogui.size()\n",
    "# print(wSrc, hSrc)\n",
    "\n",
    "while True:\n",
    "    # 1. Find hand landmarks\n",
    "    success, img = cap.read()\n",
    "    img = detector.findHands(img)\n",
    "    lmList , bbox = detector.findPosition(img)\n",
    "    output = img.copy()\n",
    "    \n",
    "    if len(lmList) != 0:\n",
    "        # print(lmList[4], lmList[8])\n",
    "        x1, y1 = lmList[8][1:]\n",
    "        x2, y2 = lmList[12][1:]\n",
    " \n",
    "        fingers = detector.fingersUp()\n",
    "        # print(fingers)\n",
    "        cv2.rectangle(img, (frameR, frameR), (wCam - frameR, hCam - frameR), (205, 250, 255), -1)\n",
    "        img = cv2.addWeighted(img, 0.5, output, 1 - .5, 0, output)\n",
    " \n",
    "        # Only Index Finger : Moving Mode\n",
    "        if fingers[1] == 1 and fingers[2] == 0:\n",
    "            # Convert Coordinates\n",
    "            x3 = np.interp(x1, (frameR, wCam - frameR), (0, wSrc))\n",
    "            y3 = np.interp(y1, (frameR, hCam - frameR), (0, hSrc))\n",
    " \n",
    "            # Smoothen Values\n",
    "            clocX = plocX + (x3 - plocX) / smoothening\n",
    "            clocY = plocY + (y3 - plocY) / smoothening\n",
    " \n",
    "            # Move Mouse\n",
    "            pyautogui.moveTo(wSrc - clocX, clocY)\n",
    "            cv2.circle(img, (x1, y1), 6, (255, 28, 0), cv2.FILLED)\n",
    "            plocX, plocY = clocX, clocY\n",
    "            # cv2.putText(img, 'Moving Mode', (20, 50), cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 2)\n",
    " \n",
    "        # Both Index and middle fingers are up : Clicking Mode\n",
    "        if fingers[1] == 1 and fingers[2] == 1:\n",
    "            # Find distance between fingers\n",
    "            length, img, lineInfo = detector.findDistance(8, 12, img)\n",
    " \n",
    "            # Click mouse if distance short\n",
    "            if length < 40:\n",
    "                cv2.circle(img, (lineInfo[4], lineInfo[5]), 6, (0, 255, 0), cv2.FILLED)\n",
    "                # cv2.putText(img, 'Click!!', (20, 50), cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 2)\n",
    "                pyautogui.click()\n",
    " \n",
    "    cTime = time.time()\n",
    "    fps = 1 / (cTime - pTime)\n",
    "    pTime = cTime\n",
    " \n",
    "    cv2.imshow(\"Vitual mouse monitor\", cv2.flip(img, 1))\n",
    "    cv2.setWindowProperty(\"Vitual mouse monitor\", cv2.WND_PROP_TOPMOST, 1)\n",
    "    if cv2.waitKey(1) == ord('q'):\n",
    "        break\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.7_YOLO",
   "language": "python",
   "name": "yolo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}