Gesture Controlled Robotic Arm Using Arduino And Opencv

About the project

This gesture controlled robotic ar;m,it tracks the movement of the hand and moves the robotic arm.It uses opencv to track the movement of the hand and uses serial communication to send the coordinates to the arduino which in fact moves the servo motors

Project info

Difficulty: Expert

Platforms: Arduino

Estimated time: 3 days

License: GNU General Public License, version 3 or later (GPL3+)

Items used in this project

Hardware components

	Arduino Uno - R3	x 1
	MG995 SERVO MOTOR	x 4

Software apps and online services

	ARDUINO IDE
	PYCHARM

Story

It is a gesture-controlled robotic arm that I recently made a laser turret it basically tracks the movements of the hand and moves the turret.A laser is attached to it so it moves the laser. Later an idea strike me how about controlling a robotic arm that would be very cool.So I managed to get a robotic arm from my cousin. He bought it from Alibaba 4 years ago I think. I took that from my brother and converted it into gesture-controlled arm.

Here is the working video of the project

Code

HandtrackingModule

"""
Hand Tracing Module
By: Murtaza Hassan
Youtube: http://www.youtube.com/c/MurtazasWorkshopRoboticsandAI
Website: https://www.computervision.zone/
"""

import cv2
import mediapipe as mp
import time
import math
import numpy as np


class handDetector():
    def __init__(self, mode=False, maxHands=2, detectionCon=0.5, trackCon=0.5):
        self.mode = mode
        self.maxHands = maxHands
        self.detectionCon = detectionCon
        self.trackCon = trackCon

        self.mpHands = mp.solutions.hands
        self.hands = self.mpHands.Hands(self.mode, self.maxHands,
                                        self.detectionCon, self.trackCon)
        self.mpDraw = mp.solutions.drawing_utils
        self.tipIds = [4, 8,9,0, 12, 16, 20]

    def findHands(self, img, draw=True):
        imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        self.results = self.hands.process(imgRGB)
        # print(results.multi_hand_landmarks)

        if self.results.multi_hand_landmarks:
            for handLms in self.results.multi_hand_landmarks:
                if draw:
                    self.mpDraw.draw_landmarks(img, handLms,
                                               self.mpHands.HAND_CONNECTIONS)

        return img

    def findPosition(self, img, handNo=0, draw=True):
        xList = []
        yList = []
        bbox = []
        self.lmList = []
        if self.results.multi_hand_landmarks:
            myHand = self.results.multi_hand_landmarks[handNo]
            for id, lm in enumerate(myHand.landmark):
                # print(id, lm)
                h, w, c = img.shape
                cx, cy = int(lm.x * w), int(lm.y * h)
                xList.append(cx)
                yList.append(cy)
                # print(id, cx, cy)
                self.lmList.append([id, cx, cy])
                if draw:
                    cv2.circle(img, (cx, cy), 5, (255, 0, 255), cv2.FILLED)

            xmin, xmax = min(xList), max(xList)
            ymin, ymax = min(yList), max(yList)
            bbox = xmin, ymin, xmax, ymax

            if draw:
                cv2.rectangle(img, (xmin - 20, ymin - 20), (xmax + 20, ymax + 20),
                              (0, 255, 0), 2)

        return self.lmList, bbox

    def fingersUp(self):
        fingers = []
        # Thumb
        if self.lmList[self.tipIds[0]][1] > self.lmList[self.tipIds[0] - 1][1]:
            fingers.append(1)
        else:
            fingers.append(0)

        # Fingers
        for id in range(1, 5):

            if self.lmList[self.tipIds[id]][2] < self.lmList[self.tipIds[id] - 2][2]:
                fingers.append(1)
            else:
                fingers.append(0)

        # totalFingers = fingers.count(1)

        return fingers
    def getpoint(self,p1,p2,img,draw=True,r=15,t=3):
        x1, y1 = self.lmList[p1][1:]
        x2, y2 = self.lmList[p2][1:]
        cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
        length =[cx,cy]
        return length
    def findDistance(self, p1, p2, img, draw=True, r=15, t=3):
        x1, y1 = self.lmList[p1][1:]
        x2, y2 = self.lmList[p2][1:]
        cx, cy = (x1 + x2) // 2, (y1 + y2) // 2

        if draw:
            cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), t)
            cv2.circle(img, (x1, y1), r, (255, 0, 255), cv2.FILLED)
            cv2.circle(img, (x2, y2), r, (255, 0, 255), cv2.FILLED)
            cv2.circle(img, (cx, cy), r, (0, 0, 255), cv2.FILLED)
        length = math.hypot(x2 - x1, y2 - y1)

        return length, img, [x1, y1, x2, y2, cx, cy]


def main():
    pTime = 0
    cTime = 0
    cap = cv2.VideoCapture(0)
    detector = handDetector()
    while True:
        success, img = cap.read()
        img = detector.findHands(img)
        lmList, bbox = detector.findPosition(img)
        if len(lmList) != 0:
            print(lmList[4])

        cTime = time.time()
        fps = 1 / (cTime - pTime)
        pTime = cTime

        cv2.putText(img, str(int(fps)), (10, 70), cv2.FONT_HERSHEY_PLAIN, 3,
                    (255, 0, 255), 3)

        cv2.imshow("Image", img)
        cv2.waitKey(1)


if __name__ == "__main__":
    main()

hand tracking python code

import cv2
import numpy as np
import HandTrackingModule as htm
import time
import autopy
import serial, time

##########################
wCam, hCam = 1280, 720
frameR = 100  # Frame Reduction
smoothening = 9
#########################

pTime = 0
plocX, plocY = 0, 0
clocX, clocY = 0, 0

cap = cv2.VideoCapture(0)
cap.set(3, wCam)
cap.set(4, hCam)
detector = htm.handDetector(maxHands=1)
wScr, hScr = autopy.screen.size()
# print(wScr)
# print(hScr)
arduino = serial.Serial('COM11', 500000, timeout=1)
time.sleep(1.8)
# print(wScr, hScr)

###################################################################
###################################################################

while True:
    # 1. Find hand Landmarks
    success, img = cap.read()
    img = detector.findHands(img)


    # img = detector.findHands(img)
    lmList, bbox = detector.findPosition(img)
    # 2. Get the tip of the index and middle fingers
    if len(lmList) != 0:
        x1, y1 = lmList[8][1:]
        x2, y2 = lmList[12][1:]
        # print(x1, y1, x2, y2)

    # 3. Check which fingers are up
        fingers = detector.fingersUp()
        # print(fingers)
        cv2.rectangle(img, (frameR, frameR), (wCam - frameR, hCam - frameR),
                  (255, 0, 255), 2)
    # 4. Only Index Finger : Moving Mode

        if fingers[1] == 1:
        # 5. Convert Coordinates
            x3 = np.interp(x1, (frameR, wCam - frameR), (0, wScr))
            y3 = np.interp(y1, (frameR, hCam - frameR), (0, hScr))
        # 6. Smoothen Values
            clocX = plocX + (x3 - plocX) / smoothening
            clocY = plocY + (y3 - plocY) / smoothening

        # 7. Move Mouse
            x =int(((wScr-clocX)/(wScr/180)))
            x =str(x)
            y =int(((clocY)/(hScr/180)))

            z, img, lineInfo2 = detector.findDistance(4,8, img)
            z = int(z)

            data ="X{}Y{}Z{}".format(x,y,z)
            print(data)
            arduino.write(str.encode(data))
            # arduino.close()
            # print(str.encode(data))
            # chara =str.encode(x)
            # print(chara.decode())
            autopy.mouse.move(wScr - clocX, clocY)
            cv2.circle(img, (x1, y1), 15, (255, 0, 255), cv2.FILLED)
            plocX, plocY = clocX, clocY

    # 8. Both Index and middle fingers are up : Clicking Mode
    #     if fingers[1] == 1:
    #     # 9. Find distance between fingers
    #         length, img, lineInfo = detector.findDistance(8, 12, img)
    #         print(length)
            # data ="X{}Y{}".format(length,length)
            # arduino.write(str.encode(data))

            # print(detector.getpoint(9,img))
        # 10. Click mouse if distance short

                # autopy.mouse.click()

    # 11. Frame Rate
    cTime = time.time()
    # fps = 1 / (cTime - pTime)
    pTime = cTime
    # cv2.putText(img, str(int(fps)), (20, 50), cv2.FONT_HERSHEY_PLAIN, 3,
    #             (255, 0, 0), 3)
    # 12. Display
    cv2.imshow("Image", img)
    cv2.waitKey(1)

Arduino code

#include<Servo.h>

Servo serX;
Servo serY;
Servo serZ;
Servo serL;




String serialData,data;
int datax,datay;

void setup() {

  serX.attach(7);
  serY.attach(3);
  serZ.attach(5);
  serL.attach(8);

  Serial.begin(500000);
  Serial.setTimeout(10);
}

void loop() {
  //lol
  
}

void serialEvent() {
  serialData = Serial.readString();
  data =String(serialData);
  serX.write(parseDataX(data));
  serY.write(parseDataY(data));
  serZ.write(parseDataZ(data));
//  serL.write(parseDataZ(data));


}

int parseDataX(String data){

  data.remove(data.indexOf("Y"));
  data.remove(data.indexOf("X"), 1);
  return data.toInt();
}

int parseDataY(String data){
  data.remove(0,data.indexOf("Y") + 1);
  return data.toInt();
}
int parseDataZ(String data){
  data.remove(0,data.indexOf("Z") + 1);
  return data.toInt();
}
int parseDataL(String data){
  data.remove(0,data.indexOf("Z") + 1);
  return data.toInt();
}

Shop

Project Hub

Video

Blog

Shop home

3D printing

Single Board Computers

Development kits

Sensors

Internet of things

Robotics

Accessories

Product of the Week

Electromaker Educator

The Electromaker Show

The Electromaker Podcast

Gesture Controlled Robotic Arm Using Arduino And Opencv

About the project

Project info

Items used in this project

Hardware components

Software apps and online services

Story

Code

HandtrackingModule

hand tracking python code

Arduino code

Credits

athul-krishna

Leave your feedback...

Shop

Project Hub

Video

Blog

Shop home

3D printing

Single Board Computers

Development kits

Sensors

Internet of things

Robotics

Accessories

Product of the Week

Electromaker Educator

The Electromaker Show

The Electromaker Podcast

Gesture Controlled Robotic Arm Using Arduino And Opencv

About the project

Project info

Items used in this project

Hardware components

Software apps and online services

Story

Code

HandtrackingModule

hand tracking python code

Arduino code

Credits

athul-krishna

Leave your feedback...

Related products