I'm working on a 3D game in unity that I want to take to the "next level" by having the character move using a hand tracking program that I made using python. I'm using sockets to send the coordinates from my hand tracking program to my unity game but my character is still not moving. I verified that the connection between the two programs is good and it is, I also verified if the coordinates are being sent for every frame and they are but the character is still not moving. If u guys have any ideas onto what I did wrong please help and ty <3.
Here is my hand tracking program in python
import socket
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mphands = mp.solutions.hands
UDP_IP = "127.0.0.1"
UDP_PORT = 5005
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
cap = cv2.VideoCapture(0)
hand = mphands.Hands()
while True:
data, image = cap.read()
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
results = hand.process(image)
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
for landmark in hand_landmarks.landmark:
h, w, c = image.shape
cx, cy = int(landmark.x * w), int(landmark.y * h)
message = f"{cx},{cy}".encode()
sock.sendto(message, (UDP_IP, UDP_PORT))
mp_drawing.draw_landmarks(
image,
hand_landmarks, mphands.HAND_CONNECTIONS)
cv2.imshow('Handtracker', image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
And here is the code from my unity project
using UnityEngine;
using System.Net;
using System.Net.Sockets;
using System;
public class PlayerBehavior : MonoBehaviour
{
public float movementSpeed = 5f;
private UdpClient udpClient;
private int port = 5005;
private Vector3 targetPosition; // Store the target position for smooth movement
void Start()
{
udpClient = new UdpClient(port);
udpClient.BeginReceive(ReceiveData, null); // Begin receiving data
}
private void ReceiveData(IAsyncResult result)
{
IPEndPoint source = new IPEndPoint(IPAddress.Any, 0);
byte[] receivedBytes = udpClient.EndReceive(result, ref source);
string receivedText = System.Text.Encoding.ASCII.GetString(receivedBytes);
// Parse received text to get coordinates
string[] coordinates = receivedText.Split(',');
float x = float.Parse(coordinates[0]);
float y = float.Parse(coordinates[1]);
Debug.Log($"Received coordinates: ({x}, {y}) from {source.Address}:{source.Port}");
// Update the target position for smooth movement
targetPosition = new Vector3(x, y, transform.position.z);
// Begin receiving again to listen for the next message
udpClient.BeginReceive(ReceiveData, null);
}
void Update()
{
// Move the character smoothly towards the target position
transform.position = Vector3.Lerp(transform.position, targetPosition, Time.deltaTime * movementSpeed);
}
void OnApplicationQuit()
{
udpClient.Close();
}
}