twister: add options to shuffle tests across subsets

Add option to shuffle tests (randomly) before they are split into subsets.
This allow to fairly distribute tests across subsets,
so that long tests are not crowded in a single group.

This also allows to detect unwanted dependencies in test execution order.

Also, added option to provide custom seed to random generator
used to shuffle tests.
This will allow to reproduce certain test order if needed.
Used seed is printed at console.

Signed-off-by: Piotr Kosycarz <piotr.kosycarz@nordicsemi.no>

scripts/pylib/twister/twisterlib/environment.py

@@ -247,6 +247,17 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
              "This option is useful when running a large number of tests on "
              "different hosts to speed up execution time.")
 
+    parser.add_argument(
+        "--shuffle-tests", action="store_true", default=None,
+        help="""Shuffle test execution order to get randomly distributed tests across subsets.
+                Used only when --subset is provided.""")
+
+    parser.add_argument(
+        "--shuffle-tests-seed", action="store", default=None,
+        help="""Seed value for random generator used to shuffle tests.
+                If not provided, seed in generated by system.
+                Used only when --shuffle-tests is provided.""")
+
     parser.add_argument("-C", "--coverage", action="store_true",
                         help="Generate coverage reports. Implies "
                              "--enable-coverage.")
@@ -698,6 +709,14 @@ def parse_arguments(parser, args, options = None):
         logger.error("--device-flash-with-test requires --device-testing")
         sys.exit(1)
 
+    if options.shuffle_tests and options.subset is None:
+        logger.error("--shuffle-tests requires --subset")
+        sys.exit(1)
+
+    if options.shuffle_tests_seed and options.shuffle_tests is None:
+        logger.error("--shuffle-tests-seed requires --shuffle-tests")
+        sys.exit(1)
+
     if options.coverage_formats and (options.coverage_tool != "gcovr"):
         logger.error("""--coverage-formats can only be used when coverage
                         tool is set to gcovr""")

scripts/pylib/twister/twisterlib/testplan.py

@@ -15,6 +15,7 @@ from itertools import islice
 import logging
 import copy
 import shutil
+import random
 
 logger = logging.getLogger('twister')
 logger.setLevel(logging.DEBUG)
@@ -251,6 +252,17 @@ class TestPlan:
         else:
             self.instances = OrderedDict(sorted(self.instances.items()))
 
+        if self.options.shuffle_tests:
+            seed_value = int.from_bytes(os.urandom(8), byteorder="big")
+            if self.options.shuffle_tests_seed is not None:
+                seed_value = self.options.shuffle_tests_seed
+
+            logger.info(f"Shuffle tests with seed: {seed_value}")
+            random.seed(seed_value)
+            temp_list = list(self.instances.items())
+            random.shuffle(temp_list)
+            self.instances = OrderedDict(temp_list)
+
         # Do calculation based on what is actually going to be run and evaluated
         # at runtime, ignore the cases we already know going to be skipped.
         # This fixes an issue where some sets would get majority of skips and