WIP: Added support for repeating seeds from previous test run.

Author: LarsAsplund

vunit/test/bench.py

@@ -84,28 +84,57 @@ def create_tests(self, simulator_if, seed, elaborate_only, test_list=None):
         if test_list is None:
             test_list = TestList()
 
+        key = b"test_history"
+        if key not in self._database:
+            test_history = {}
+        else:
+            test_history = self._database[key]
+
         if self._individual_tests:
             for test_case in self._test_cases:
-                test_case.create_tests(simulator_if, seed, elaborate_only, test_list)
+                test_case.create_tests(simulator_if, seed, elaborate_only, test_list, test_history)
         elif self._implicit_test:
             for config in self._get_configurations_to_run():
+                actual_seed = seed
+                if seed == "repeat":
+                    # Test suite name and test name are the same for implicit tests
+                    test_suite_name = test_name = IndependentSimTestCase.get_name(self._implicit_test, config)
+
+                    if test_suite_name not in test_history:
+                        actual_seed = None
+                    elif test_name not in test_history[test_suite_name]:
+                        actual_seed = None
+                    else:
+                        actual_seed = test_history[test_name][test_name]["seed"]
+
                 test_list.add_test(
                     IndependentSimTestCase(
                         test=self._implicit_test,
                         config=config,
                         simulator_if=simulator_if,
-                        seed=seed,
+                        seed=actual_seed,
                         elaborate_only=elaborate_only,
                     )
                 )
         else:
             for config in self._get_configurations_to_run():
+                actual_seed = seed
+                if seed == "repeat":
+                    test_suite_name = SameSimTestSuite.get_name(config)
+
+                    if test_suite_name not in test_history:
+                        actual_seed = None
+                    else:
+                        # All tests in the test suite will have the same seed so take the first test
+                        test_name = next(iter(test_history[test_suite_name]))
+                        actual_seed = test_history[test_suite_name][test_name]["seed"]
+
                 test_list.add_suite(
                     SameSimTestSuite(
                         tests=[test.test for test in self._test_cases],
                         config=config,
                         simulator_if=simulator_if,
-                        seed=seed,
+                        seed=actual_seed,
                         elaborate_only=elaborate_only,
                     )
                 )
@@ -414,17 +443,31 @@ def _get_configurations_to_run(self):
             del configs[DEFAULT_NAME]
         return configs.values()
 
-    def create_tests(self, simulator_if, seed, elaborate_only, test_list=None):
+    def create_tests(self, simulator_if, seed, elaborate_only, test_list=None, test_history=None):
         """
         Create all tests from this test case which may be several depending on the number of configurations
         """
+        test_history = test_history if test_history else {}
+
         for config in self._get_configurations_to_run():
+            actual_seed = seed
+            if seed == "repeat":
+                # Test suite name and test name are the same for individual tests
+                test_suite_name = test_name = IndependentSimTestCase.get_name(self._test, config)
+
+                if test_suite_name not in test_history:
+                    actual_seed = None
+                elif test_name not in test_history[test_suite_name]:
+                    actual_seed = None
+                else:
+                    actual_seed = test_history[test_suite_name][test_name]["seed"]
+
             test_list.add_test(
                 IndependentSimTestCase(
                     test=self._test,
                     config=config,
                     simulator_if=simulator_if,
-                    seed=seed,
+                    seed=actual_seed,
                     elaborate_only=elaborate_only,
                 )
             )

vunit/test/list.py

@@ -104,3 +104,6 @@ def run(self, *args, **kwargs):
         """
         test_ok = self._test_case.run(*args, **kwargs)
         return {self._test_case.name: PASSED if test_ok else FAILED}
+
+    def get_seed(self):
+        return self._test_case.get_seed()

vunit/test/report.py

@@ -275,7 +275,7 @@ class TestResult(object):
     """
 
     def __init__(
-        self, name, status, time, output_file_name, *, test_suite_name, start_time
+        self, name, status, time, output_file_name, *, test_suite_name, start_time, seed
     ):  # pylint: disable=too-many-arguments
         assert status in (PASSED, FAILED, SKIPPED)
         self.name = name
@@ -284,6 +284,7 @@ def __init__(
         self._output_file_name = output_file_name
         self.test_suite_name = test_suite_name
         self.start_time = start_time
+        self.seed = seed
 
     @property
     def output(self):

vunit/test/runner.py

@@ -328,6 +328,7 @@ def _add_results(
         """
         runtime = ostools.get_time() - start_time
         time_per_test = runtime / len(results)
+        seed = test_suite.get_seed()
 
         for test_name in test_suite.test_names:
             status = results[test_name]
@@ -338,6 +339,7 @@ def _add_results(
                 output_file_name,
                 test_suite_name=test_suite.name,
                 start_time=start_time,
+                seed=seed
             )
             self._report.print_latest_status(total_tests=num_tests)
         print()

vunit/test/suites.py

@@ -21,17 +21,23 @@ class IndependentSimTestCase(object):
     A test case to be run in an independent simulation
     """
 
-    def __init__(self, test, config, simulator_if, seed, *, elaborate_only=False):
-        self._name = f"{config.library_name!s}.{config.design_unit_name!s}"
+    @staticmethod
+    def get_name(test, config):
+        name = f"{config.library_name!s}.{config.design_unit_name!s}"
 
         if not config.is_default:
-            self._name += "." + config.name
+            name += "." + config.name
 
         if test.is_explicit:
-            self._name += "." + test.name
+            name += "." + test.name
         elif config.is_default:
             # JUnit XML test reports wants three dotted name hierarchies
-            self._name += ".all"
+            name += ".all"
+
+        return name
+
+    def __init__(self, test, config, simulator_if, seed, *, elaborate_only=False):
+        self._name = self.get_name(test, config)
 
         self._configuration = config
 
@@ -74,19 +80,28 @@ def run(self, *args, **kwargs):
         Run the test case using the output_path
         """
         results = self._run.run(*args, **kwargs)
+
         return results[self._test.name] == PASSED
 
+    def get_seed(self):
+        return self._run.get_seed()
+
 
 class SameSimTestSuite(object):
     """
     A test suite where multiple test cases are run within the same simulation
     """
 
-    def __init__(self, tests, config, simulator_if, seed, *, elaborate_only=False):
-        self._name = f"{config.library_name!s}.{config.design_unit_name!s}"
-
+    @staticmethod
+    def get_name(config):
+        name = f"{config.library_name!s}.{config.design_unit_name!s}"
         if not config.is_default:
-            self._name += "." + config.name
+            name += "." + config.name
+
+        return name  
+
+    def __init__(self, tests, config, simulator_if, seed, *, elaborate_only=False):
+        self._name = self.get_name(config)
 
         self._configuration = config
 
@@ -98,7 +113,7 @@ def __init__(self, tests, config, simulator_if, seed, *, elaborate_only=False):
             test_suite_name=self._name,
             test_cases=[test.name for test in tests],
             seed=seed,
-        )
+        ) 
 
     @property
     def file_name(self):
@@ -146,8 +161,11 @@ def run(self, *args, **kwargs):
         """
         results = self._run.run(*args, **kwargs)
         results = {_full_name(self._name, test_name): result for test_name, result in results.items()}
+
         return results
 
+    def get_seed(self):
+        return self._run.get_seed()
 
 class TestRun(object):
     """
@@ -160,7 +178,19 @@ def __init__(self, *, simulator_if, config, elaborate_only, test_suite_name, tes
         self._elaborate_only = elaborate_only
         self._test_suite_name = test_suite_name
         self._test_cases = test_cases
-        self._args_seed = seed
+        self._seed = seed
+            
+    def get_seed(self):
+        if self._seed:
+            pass
+        elif "seed" in self._config.sim_options:
+            self._seed = self._config.sim_options["seed"]
+        else:
+            now_us = str(int(time() * 1e6)).encode()
+            thread_id = get_ident().to_bytes(8, byteorder="little")
+            self._seed = blake2b(now_us, digest_size=8, salt=thread_id).hexdigest()
+
+        return self._seed
 
     def set_test_cases(self, test_cases):
         self._test_cases = test_cases
@@ -223,15 +253,7 @@ def _simulate(self, output_path):
         """
 
         config = self._config.copy()
-
-        if self._args_seed:
-            seed = self._args_seed
-        elif "seed" in config.sim_options:
-            seed = config.sim_options["seed"]
-        else:
-            now_us = str(int(time() * 1e6)).encode()
-            thread_id = get_ident().to_bytes(8, byteorder="little")
-            seed = blake2b(now_us, digest_size=8, salt=thread_id).hexdigest()
+        seed = self.get_seed()
 
         print(f"Seed for {self._test_suite_name}: {seed}")
 

vunit/ui/__init__.py

@@ -163,7 +163,7 @@ def __init__(
 
         self._create_output_path(args.clean)
 
-        self._database_version = (10, sys.version)
+        self._database_version = (11, sys.version)
         self._pickled_database_version = (self._database_version[0], pickle.HIGHEST_PROTOCOL)
 
         self._database = self._create_database()
@@ -989,6 +989,7 @@ def _update_test_history(self, report, simulator_if):
             test_suite_data[test_result.test_suite_name][test_result.name]["skipped"] = test_result.skipped
             test_suite_data[test_result.test_suite_name][test_result.name]["failed"] = test_result.failed
             test_suite_data[test_result.test_suite_name][test_result.name]["start_time"] = test_result.start_time
+            test_suite_data[test_result.test_suite_name][test_result.name]["seed"] = test_result.seed
 
         test_history = self._get_test_history(simulator_if)
 

vunit/vunit_cli.py

@@ -35,8 +35,9 @@
 """
 
 import argparse
-from pathlib import Path
+import re
 import os
+from pathlib import Path
 from vunit.sim_if.factory import SIMULATOR_FACTORY
 from vunit.about import version
 
@@ -265,10 +266,28 @@ def _create_argument_parser(description=None, for_documentation=False):
 
     parser.add_argument("--version", action="version", version=version())
 
+    def valid_seed(value):
+        value = value.lower()
+
+        if value == "repeat":
+            return value
+
+        if not re.fullmatch(r'[0-9a-fA-F]+', value):
+            raise argparse.ArgumentTypeError(f"""Expecting a valid 16-digit hex number or "repeat".""")
+
+        if len(value) != 16:
+            raise argparse.ArgumentTypeError(f"Expecting 16 digits. {value} is {len(value)} digits.")
+
+        return value.lower()
+
     parser.add_argument(
         "--seed",
         default=None,
-        help="Base seed provided to the simulation. Must be 16 hex digits. Default seed is generated from system time.",
+        type=valid_seed,
+        metavar="<seed>|repeat",
+        help="16 hex digits base seed provided to the simulation or repeat the previous base seed(s)."
+        """ If no previous base seed has been recorded, "repeat" will use the default base seed which is"""
+        " generated from system time.",
     )
 
     SIMULATOR_FACTORY.add_arguments(parser)