How do I make sure that previous AR game object is destroyed before instantiating a new one (I'm using ARCore in Unity)?

Question

I'm using AR Foundation, Google AR core on unity for a project. The basic goal is to display a different game object depending on the image scanned. However, the code I wrote sometimes works but sometimes, does not destroy the previous game object and overlays the next game object on top of the previous one.

using System.Collections.Generic;
using UnityEngine;
using UnityEngine.XR.ARFoundation;

[RequireComponent(typeof(ARTrackedImageManager))]
public class ImageTracking : MonoBehaviour
{
    [SerializeField]
    private GameObject[] arObjectsToPlace;

    [SerializeField]
    private Vector3 scaleFactor = new Vector3(0.1f, 0.1f, 0.1f);

    private ARTrackedImageManager m_TrackedImageManager;

    private Dictionary<string, GameObject> arObjects = new Dictionary<string, GameObject>();

    void Awake()
    {
        m_TrackedImageManager = GetComponent<ARTrackedImageManager>();
    }

    void OnEnable()
    {
        m_TrackedImageManager.trackedImagesChanged += OnTrackedImagesChanged;

        // Instantiate all AR objects and add them to the dictionary
        foreach (GameObject arObject in arObjectsToPlace)
        {
            GameObject newARObject = Instantiate(arObject, Vector3.zero, Quaternion.identity);
            newARObject.name = arObject.name;
            newARObject.SetActive(false); // Initially set them to inactive
            arObjects.Add(arObject.name, newARObject);
        }
    }

    void OnDisable()
    {
        m_TrackedImageManager.trackedImagesChanged -= OnTrackedImagesChanged;
    }

    void OnTrackedImagesChanged(ARTrackedImagesChangedEventArgs eventArgs)
    {
        foreach (ARTrackedImage trackedImage in eventArgs.added)
        {
            UpdateARImage(trackedImage);
        }

        foreach (ARTrackedImage trackedImage in eventArgs.updated)
        {
            UpdateARImage(trackedImage);
        }

        foreach (ARTrackedImage trackedImage in eventArgs.removed)
        {
            string imageName = trackedImage.referenceImage.name;

            // Disable the AR object associated with the removed image
            if (arObjects.ContainsKey(imageName))
            {
                GameObject arObject = arObjects[imageName];

                // Ensure the object is destroyed properly
                Destroy(arObject);
                arObjects.Remove(imageName);
            }
        }
    }

    private void UpdateARImage(ARTrackedImage trackedImage)
    {
        string imageName = trackedImage.referenceImage.name;

        if (arObjects.ContainsKey(imageName))
        {
            GameObject arObject = arObjects[imageName];
            arObject.SetActive(trackedImage.trackingState == UnityEngine.XR.ARSubsystems.TrackingState.Tracking);

            if (trackedImage.trackingState == UnityEngine.XR.ARSubsystems.TrackingState.Tracking)
            {
                arObject.transform.position = trackedImage.transform.position;
                arObject.transform.localScale = scaleFactor;
            }
        }

        Debug.Log($"trackedImage.referenceImage.name: {imageName}");
    }
}

This is my code. How do I make it so that it destroys the gameobject if the image is no longer in view and the next game object is shown so that the gameobjects don't overlay on top of each other.