QTableView - How to handle a lot of dataChanged signals?

Question

I have to program a table view that should be able to handle tens thousands of cells containing images (image is different for every cell). All in python using PySide2.

I implemented the loading of my images using a thread pool. The problem is that I have to asynchronusly notify the view that the image has been loaded for a given index so it can reload the display. Using the dataChanged signal works but there are just too many of them to process and the UI does not show up until all the indexes are processed by the thread pool.

I provide a working example below that reproduces the issue (no images, juste text). For now, I solved the problem by letting the threads sleep a little (just uncomment the time.sleep(1) line in the Work.run method) but it feels more like a dirty hack than a real solution to me.

I thought about the following solutions:

• Try to make dataChanged signal behave asynchronusly. I suspect the default connection between dataChanged and whatever slot that update the view to be a AutoConnection. Is there any way to do this?
• Gather the modified indexes in a buffer and update the view at regular time intervals. I would like to avoid this solution because finding a good time interval between two evaluation of the buffer is quite a hard task.

Do you have any other idea on how to avoid this blocking behavior?

Thank you for your advice!

import math
from random import choice
import string
import time

from PySide2.QtCore import QModelIndex
from PySide2.QtCore import Qt
from PySide2.QtCore import QObject
from PySide2.QtCore import Signal
from PySide2.QtCore import QThread
from PySide2.QtCore import QThreadPool
from PySide2.QtCore import QMutex
from PySide2.QtCore import QAbstractTableModel
from PySide2.QtWidgets import QTableView


class Notifier(QObject):

    finished = Signal(QModelIndex, str)


class Work(QThread):

    def __init__(self, *args, **kwargs):
        super(Work, self).__init__(*args, **kwargs)
        self.work = []
        self._stopped = False
        self._slept = False

    def run(self):

        while True:

            try:
                work = self.work.pop(0)
            except IndexError:
                work = None

            if not work:
                if self._slept:
                    break

                self.msleep(500)
                self._slept = True
                continue

            # Uncomment the following line to make the UI responsive
            # time.sleep(1)

            if work[0]:
                c = ''.join(choice(string.ascii_uppercase + string.digits)
                            for _ in range(6))
                work[0].finished.emit(work[1], c)

    def reset(self):
        self.work = []
        self._stopped = True


class WorkPool(object):

    def __init__(self):

        self.thread_count = QThreadPool().maxThreadCount()
        self.thread_pool = []
        self.thread_cpt = 0
        self.mutex = QMutex()

        for c in range(0, self.thread_count):
            self.thread_pool.append(Work())

    def add_work(self, notifier, index):

        new_thread = divmod(self.thread_cpt, self.thread_count)[1]
        thread = self.thread_pool[new_thread]
        self.thread_cpt += 1

        thread.work.append((notifier, index))

        if not thread.isRunning():
            thread.start()

    def terminate(self):
        self.mutex.lock()

        for t in self.thread_pool:
            t.reset()

        for t in self.thread_pool:
            t.wait()

        self.mutex.unlock()


class TableModel(QAbstractTableModel):

    def __init__(self, items, *args, **kwargs):
        super(TableModel, self).__init__(*args, **kwargs)

        self.items = items
        self.works = []
        self.loader = WorkPool()

    def index(self, row, column, parent=QModelIndex()):

        pos = row * self.columnCount() + column

        try:
            return self.createIndex(row, column,self.items[pos])

        except IndexError:
            return QModelIndex()


    def data(self, index, role):

        if not index.isValid():
            return None

        if role == Qt.DisplayRole:
            return index.internalPointer()

    def columnCount(self, parent=QModelIndex()):
        return 10

    def rowCount(self, parent=QModelIndex()):
        return int(math.ceil(float(len(self.items)) / self.columnCount()))

    def refresh_content(self):

        # Launch a thread to update the content of each index
        for r in range(0, self.rowCount()):
            for c in range(0, self.columnCount()):
                index = self.index(r, c)

                notifier = Notifier()
                notifier.finished.connect(self.setData)

                self.loader.add_work(notifier, index)

    def setData(self, index, value):

        if not index.isValid():
            return False

        self.items[index.row() * self.columnCount() + index.column()] = value
        self.dataChanged.emit(index, index)
        return True



class TableView(QTableView):

    def closeEvent(self, *args, **kwargs):
        self.model().loader.terminate()
        super(TableView, self).closeEvent(*args, **kwargs)


if __name__ == '__main__':

    from PySide2.QtWidgets import QApplication

    app = QApplication([])

    tv = TableView()

    model = TableModel([None] * 99999)
    tv.setModel(model)
    model.refresh_content()

    tv.show()

    app.exec_()