Django - is there a way to shuffle and run only a subset of tests?

Question

We are using Django with tests. We have in total about 6,000 tests which take about 40 minutes to run. Is there a way to shuffle the tests and run only 200 (randomly chosen) tests? This should be done with a command line parameter with the number 200 (which may change), since usually we run all the tests.

This can be used together with --shuffle. If the number of tests is less than 200, run all the tests (and not 200 tests).

Here is some of our code:

File https://github.com/speedy-net/speedy-net/blob/main/speedy/core/base/management/commands/test.py:

from django.core.management.commands import test


class Command(test.Command):
    def add_arguments(self, parser):
        super().add_arguments(parser=parser)
        parser.add_argument(
            "--test-all-languages",
            action="store_true",
            help="If run with this argument, test all languages, and don't skip languages.",
        )

File https://github.com/speedy-net/speedy-net/blob/main/speedy/core/base/test/models.py:

    class SiteDiscoverRunner(DiscoverRunner):
        def __init__(self, *args, **kwargs):
            assert (django_settings.TESTS is True)
            super().__init__(*args, **kwargs)
            self.test_all_languages = kwargs.get('test_all_languages', False)

        def build_suite(self, test_labels=None, extra_tests=None, **kwargs):
            if (not (test_labels)):
                # Default test_labels are all the relevant directories under "speedy". For example ["speedy.core", "speedy.net"].
                # Due to problems with templates, "speedy.match" label is not added to speedy.net tests, and "speedy.net" label is not added to speedy.match tests. # ~~~~ TODO: fix this bug and enable these labels, although the tests there are skipped.
                test_labels = []
                for label in django_settings.INSTALLED_APPS:
                    if (label.startswith('speedy.')):
                        label_to_test = '.'.join(label.split('.')[:2])
                        if (label_to_test == 'speedy.net'):
                            add_this_label = (django_settings.SITE_ID == django_settings.SPEEDY_NET_SITE_ID)
                        elif (label_to_test == 'speedy.match'):
                            add_this_label = (django_settings.SITE_ID == django_settings.SPEEDY_MATCH_SITE_ID)
                        elif (label_to_test == 'speedy.composer'):
                            add_this_label = (django_settings.SITE_ID == django_settings.SPEEDY_COMPOSER_SITE_ID)
                        elif (label_to_test == 'speedy.mail'):
                            add_this_label = (django_settings.SITE_ID == django_settings.SPEEDY_MAIL_SOFTWARE_SITE_ID)
                        else:
                            add_this_label = True
                        if (add_this_label):
                            if (not (label_to_test in test_labels)):
                                test_labels.append(label_to_test)
            print(test_labels)
            return super().build_suite(test_labels=test_labels, extra_tests=extra_tests, **kwargs)

        def setup_test_environment(self, **kwargs):
            super().setup_test_environment(**kwargs)
            django_settings.TEST_ALL_LANGUAGES = self.test_all_languages

        def teardown_test_environment(self, **kwargs):
            super().teardown_test_environment(**kwargs)
            del django_settings.TEST_ALL_LANGUAGES

There is more code which you can see on our repository on GitHub.

Answer 1

1. Accept and store int num in test.py Command:

class Command(test.Command):
    def add_arguments(self, parser):
        super().add_arguments(parser=parser)
        ...
        parser.add_argument(
            "--num",
            action="store",
            help="If run with this argument, run this number of tests.",
            type=int,
        )

2. Override test_suite in SiteDiscoverRunner:

    class SiteDiscoverRunner(DiscoverRunner):
        def __init__(self, *args, **kwargs):
            assert (django_settings.TESTS is True)
            super().__init__(*args, **kwargs)
            ...
            self.num = kwargs.get('num', None)

        ...

        def test_suite(self, tests=()):
            if (self.num is not None):
                tests = tests[:self.num]
            return super().test_suite(tests=tests)

Usage:

python manage.py test --shuffle --num 200

(Note: In your project, use with --test-all-languages, otherwise some tests may be skipped.)

Answer 2

In test.py add this :-

from django.core.management.commands import test

class Command(test.Command):
    def add_arguments(self, parser):
        super().add_arguments(parser=parser)
        parser.add_argument(
            "--test-all-languages",
            action="store_true",
            help="If run with this argument, test all languages, and don't skip languages.",
        )
        parser.add_argument(
            "--random-subset",
            type=int,
            help="If provided, randomly select this many tests to run.",
        )

In models.py add this:-

class SiteDiscoverRunner(DiscoverRunner):
    def __init__(self, *args, **kwargs):
        assert (django_settings.TESTS is True)
        super().__init__(*args, **kwargs)
        self.test_all_languages = kwargs.get('test_all_languages', False)

        
        random.shuffle(self.test_labels)

        
        if len(self.test_labels) < 200:
            self.test_labels = self.test_labels
        else:
            self.test_labels = self.test_labels[:200]

def build_suite(self, test_labels=None, extra_tests=None, **kwargs):
    if (not (test_labels)):
        test_labels = self.get_default_test_labels()

    
    random.shuffle(test_labels)

    
    self.random_subset = min(len(test_labels), 200) if len(test_labels) > 0 else len(test_labels)

    
    test_labels = test_labels[: self.random_subset]

    
    filtered_test_labels = []
    for label in django_settings.INSTALLED_APPS:
        if label.startswith('speedy.'):
            label_to_test = '.'.join(label.split('.')[:2])
            if label_to_test == 'speedy.net':
                add_this_label = (django_settings.SITE_ID == django_settings.SPEEDY_NET_SITE_ID)
            elif label_to_test == 'speedy.match':
                add_this_label = (django_settings.SITE_ID == django_settings.SPEEDY_MATCH_SITE_ID)
            else:
                add_this_label = True

            if add_this_label:
                filtered_test_labels.append(label)

    return super().build_suite(test_labels=filtered_test_labels, extra_tests=extra_tests, **kwargs)

        

  
  def handle(self, *args, **options):
        if options["random_subset"]:
                        
          random.shuffle(self.tests)
                
            self.tests = self.tests[: options["random_subset"]]

        super().handle(*args, **options)


    def setup_test_environment(self, **kwargs):
        super().setup_test_environment(**kwargs)
        django_settings.TEST_ALL_LANGUAGES = self.test_all_languages
        django_settings.TEST_RANDOM_SUBSET = self.random_subset

    def teardown_test_environment(self, **kwargs):
        super().teardown_test_environment(**kwargs)
        del django_settings.TEST_ALL_LANGUAGES
        del django_settings.TEST_RANDOM_SUBSET