React Three Fiber Player Component stop moving after a moment of no movement

Question

I have a React Three Fiber application, and the user can move the player around in a FPS like fashion. It works great until you stop moving for more than like 3 seconds, then if you try to move again, the player won't move, I can still move the camera around, but none of the keyboard controls work. Here is my code Home.jsx (where the canvas is created)

import { KeyboardControls, PointerLockControls, Sky } from '@react-three/drei'
import { Canvas } from '@react-three/fiber'
import { Physics, RigidBody } from '@react-three/rapier'
import React, { useEffect } from 'react'
import { Player } from '../components/3d/Player'
import '../css/Home.css'
import Scene from '../components/3d/Scene'

const Home = () => {

    useEffect(() => {
        // Add event listener for the 'Escape' key press
        const handleKeyDown = (event) => {
            if (event.key === 'Escape') {
                document.exitPointerLock(); // Release pointer lock
            }
        };

        document.addEventListener('keydown', handleKeyDown);

        return () => {
            // Clean up event listener
            document.removeEventListener('keydown', handleKeyDown);
        };
    }, []);

    
    return (
        <section >


            <div>
                <KeyboardControls
                    map={[
                        { name: "forward", keys: ["ArrowUp", "w", "W"] },
                        { name: "backward", keys: ["ArrowDown", "s", "S"] },
                        { name: "left", keys: ["ArrowLeft", "a", "A"] },
                        { name: "right", keys: ["ArrowRight", "d", "D"] },
                        { name: "jump", keys: ["Space"] },
                        { name: 'run', keys: ['Shift'] },
                        { name: 'crouch', keys: ['ControlLeft']}
                    ]}>
                    <Canvas id='canvas'shadows camera={{ fove: 50 }} className='canvas' onKeyDown={(e)=>{console.log(e)}}>
                        <Sky sunPosition={[100, 20, 100]} />
                        <ambientLight intensity={1} />
                        <pointLight castShadow intensity={0.8} position={[100, 100, 100]} />
                        <Physics gravity={[0, -30, 0]}>
                            <Scene scale={1}/>
                            
                            <Player />
                        </Physics>
                        <PointerLockControls />
                    </Canvas>
                </KeyboardControls>
            </div>
        </section>
    )
}

export default Home

Player.jsx

import * as THREE from "three"
import * as RAPIER from "@dimforge/rapier3d-compat"
import { useRef, useState } from "react"
import { useFrame } from "@react-three/fiber"
import { useKeyboardControls } from "@react-three/drei"
import { CapsuleCollider, RigidBody, useRapier } from "@react-three/rapier"

const SPEED = 5
const direction = new THREE.Vector3()
const frontVector = new THREE.Vector3()
const sideVector = new THREE.Vector3()
const rotation = new THREE.Vector3()

export function Player({ lerp = THREE.MathUtils.lerp }) {

    const playerRef = useRef()
    const grounded = useRef(false);
    const [, get] = useKeyboardControls()
    const [targetHeight, setTargetHeight] = useState(0.7); // Target height for smooth transition
    const [height, setHeight] = useState(0.7); // Current height
    const lerpFactor = 0.1; // Adjust this value for smoother or faster transitions
 

    useFrame((state) => {
        const velocity = playerRef.current.linvel()       
        const { forward, backward, left, right, jump, crouch } = get()
        const translation = playerRef.current.translation();
        state.camera.position.set(translation.x, translation.y, translation.z);
  
        frontVector.set(0, 0, backward - forward)
        sideVector.set(left - right, 0, 0)
        direction.subVectors(frontVector, sideVector).normalize().multiplyScalar(SPEED).applyEuler(state.camera.rotation) 
        playerRef.current.setLinvel({ x: direction.x, y: velocity.y, z: direction.z })
        
        // if (jump && grounded.current) {
        //     playerRef.current.setLinvel({ x: 0, y: 7, z: 0 })
        //     grounded.current = false;
        // }
        
    })
    return (
        <>
            <RigidBody  ref={playerRef} colliders={false} mass={1} type="dynamic" position={[2, 2, 0]} enabledRotations={[false, false, false]} onCollisionEnter={(e) => {
                
                // if(e.rigidBodyObject.id === "181"){
                //     grounded.current = true
                // }
            }}>
                <CapsuleCollider args={[0.75, height]} />
            </RigidBody>
        </>
    )
}

I have tried logging everywhere I can. Every variable associated with the keyboard controls work. the Forward and Side Vectors change when I press keys. There are no errors in the console either.