Moving 3D object in unity using hand coordinates

57 Views Asked by At

I'm working on a 3D game in unity that I want to take to the "next level" by having the character move using a hand tracking program that I made using python. I'm using sockets to send the coordinates from my hand tracking program to my unity game but my character is still not moving. I verified that the connection between the two programs is good and it is, I also verified if the coordinates are being sent for every frame and they are but the character is still not moving. If u guys have any ideas onto what I did wrong please help and ty <3.

Here is my hand tracking program in python

import socket
import cv2
import mediapipe as mp

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles

mphands = mp.solutions.hands

UDP_IP = "127.0.0.1"
UDP_PORT = 5005

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

cap = cv2.VideoCapture(0)
hand = mphands.Hands()

while True:
    data, image = cap.read()
    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
    results = hand.process(image)
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

    if results.multi_hand_landmarks:
        for hand_landmarks in results.multi_hand_landmarks:
            for landmark in hand_landmarks.landmark:
                h, w, c = image.shape
                cx, cy = int(landmark.x * w), int(landmark.y * h)           
                message = f"{cx},{cy}".encode()
                sock.sendto(message, (UDP_IP, UDP_PORT))

            mp_drawing.draw_landmarks(
                image,
                hand_landmarks, mphands.HAND_CONNECTIONS)

    cv2.imshow('Handtracker', image)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

And here is the code from my unity project

using UnityEngine;
using System.Net;
using System.Net.Sockets;
using System;

public class PlayerBehavior : MonoBehaviour
{
    public float movementSpeed = 5f;
    private UdpClient udpClient;
    private int port = 5005;
    private Vector3 targetPosition; // Store the target position for smooth movement

    void Start()
    {
        udpClient = new UdpClient(port);
        udpClient.BeginReceive(ReceiveData, null); // Begin receiving data
    }

    private void ReceiveData(IAsyncResult result)
    {
        IPEndPoint source = new IPEndPoint(IPAddress.Any, 0);
        byte[] receivedBytes = udpClient.EndReceive(result, ref source);
        string receivedText = System.Text.Encoding.ASCII.GetString(receivedBytes);

        // Parse received text to get coordinates
        string[] coordinates = receivedText.Split(',');
        float x = float.Parse(coordinates[0]);
        float y = float.Parse(coordinates[1]);

        Debug.Log($"Received coordinates: ({x}, {y}) from {source.Address}:{source.Port}");

        // Update the target position for smooth movement
        targetPosition = new Vector3(x, y, transform.position.z);

        // Begin receiving again to listen for the next message
        udpClient.BeginReceive(ReceiveData, null);
    }

    void Update()
    {
        // Move the character smoothly towards the target position
        transform.position = Vector3.Lerp(transform.position, targetPosition, Time.deltaTime * movementSpeed);
    }

    void OnApplicationQuit()
    {
        udpClient.Close();
    }
}
0

There are 0 best solutions below