PyTest-qt handle QFileDialog automatically

Question

I have a PySide6 application, for which I want to test that when cancelling the file dialog, nothing changes. Here is part of the function where the dialog is called:

def change_folder(self, side: int):
        self.dialog = QFileDialog(self)
        new_folder_name = (
            self.dialog.getExistingDirectory(
                self, caption="Select a folder"
            )
        )
...

And here part of the test for it.

@pytest.fixture
def main_window(qtbot):
    qApp = QApplication.instance()
    if qApp is None:
        app = QApplication([""])
    else:
        app = qApp
    window = MainWindow()
    qtbot.addWidget(window)
    window.show()
    return window

def test_change_folder(main_window, qtbot):
    def handle_dialog():
        dialog = main_window.dialog
        # alternative: dialog = QApplication.activeWindow()
        qtbot.addWidget(dialog)
        dialog.reject()
        # alternative: qtbot.mouseClick(dialog.buton(QFileDialog.Cancel), Qt.LeftButton, delay=1)

    QTimer.singleShot(500, handle_dialog)
    qtbot.mouseClick(main_window.change_folder_left_button, Qt.LeftButton, delay=1)
    assert not main_window.dialog.isVisible()
    ...

I followed the instructions from here, but i cant make it work, I always have to click the cancel button myself, if not the dialog just stays open. Why could that be?