Add support for specific test configuration. Closes #22.

Author: kraigher

examples/generate_tests/run.py

@@ -25,8 +25,7 @@ def post_check(output_path):
         This function recives the output_path of the test
         """
 
-        expected = ", ".join(["hello world",
-                              str(data_width),
+        expected = ", ".join([str(data_width),
                               str(sign).lower()]) + "\n"
 
         output_file = join(output_path, "generics.txt")
@@ -42,21 +41,21 @@ def post_check(output_path):
     return post_check
 
 
-def generate_data_width_and_sign_tests(entity):
+def generate_tests(obj, signs, data_widths):
     """
-    Generate test for entity by varying the data_width and sign generics
+    Generate test by varying the data_width and sign generics
     """
 
-    for sign, data_width in product([False, True], range(1, 5)):
+    for sign, data_width in product(signs, data_widths):
         # This configuration name is added as a suffix to the test bench name
         config_name = "data_width=%i,sign=%s" % (data_width, sign)
 
         # Add the configuration with a post check function to verify the output
-        entity.add_config(name=config_name,
-                          generics=dict(
-                              data_width=data_width,
-                              sign=sign),
-                          post_check=make_post_check(data_width, sign))
+        obj.add_config(name=config_name,
+                       generics=dict(
+                           data_width=data_width,
+                           sign=sign),
+                       post_check=make_post_check(data_width, sign))
 
 
 test_path = join(dirname(__file__), "test")
@@ -68,7 +67,17 @@ def generate_data_width_and_sign_tests(entity):
 tb_generated = lib.entity("tb_generated")
 
 # Just set a generic for a given test bench
-tb_generated.set_generic("message", "hello world")
-generate_data_width_and_sign_tests(tb_generated)
+tb_generated.set_generic("message", "set-for-entity")
+
+# Run all tests with signed/unsigned and data width in range [1,5[
+generate_tests(tb_generated, [False, True], range(1, 5))
+
+# Test 2 should only be run with signed width of 16
+test_2 = tb_generated.test("Test 2")
+generate_tests(test_2, [True], [16])
+
+# Just set a generic for a given test
+test_2.set_generic("message", "set-for-test")
+
 
 ui.main()

examples/generate_tests/test/tb_generated.vhd

@@ -21,14 +21,26 @@ end entity;
 architecture a of tb_generated is
 begin
   main : process
-   file fwrite : text;
-   variable l : line;
+    procedure dump_generics is
+      file fwrite : text;
+      variable l : line;
+    begin
+      file_open(fwrite, output_path & "/" & "generics.txt", write_mode);
+      write(l, to_string(data_width) & ", " & to_string(sign));
+      writeline(fwrite, l);
+      file_close(fwrite);
+    end procedure;
   begin
     test_runner_setup(runner, runner_cfg);
-    file_open(fwrite, output_path & "/" & "generics.txt", write_mode);
-    write(l, message & ", " & to_string(data_width) & ", " & to_string(sign));
-    writeline(fwrite, l);
-    file_close(fwrite);
+    while test_suite loop
+      if run("Test 1") then
+        assert message = "set-for-entity";
+        dump_generics;
+      elsif run("Test 2") then
+        assert message = "set-for-test";
+        dump_generics;
+      end if;
+    end loop;
     test_runner_cleanup(runner);
     wait;
   end process;

vunit/test/acceptance/artificial/run.py

@@ -18,4 +18,33 @@
 ui = VUnit.from_argv()
 lib = ui.add_library("lib")
 lib.add_source_files(join(root, "vhdl", "*.vhd"))
+
+
+def configure_tb_with_generic_config(ui):
+    """
+    Configure tb_with_generic_config test bench
+    """
+    bench = lib.entity("tb_with_generic_config")
+    tests = [bench.test("Test %i" % i) for i in range(5)]
+
+    bench.set_generic("set_generic", "set-for-entity")
+
+    tests[1].add_config("", generics=dict(config_generic="set-from-config"))
+
+    tests[2].set_generic("set_generic", "set-for-test")
+
+    tests[3].add_config("", generics=dict(set_generic="set-for-test",
+                                          config_generic="set-from-config"))
+
+    def post_check(output_path):
+        with open(join(output_path, "post_check.txt"), "r") as fptr:
+            return fptr.read() == "Test 4 was here"
+
+    tests[4].add_config("",
+                        generics=dict(set_generic="set-from-config",
+                                      config_generic="set-from-config"),
+                        post_check=post_check)
+
+configure_tb_with_generic_config(ui)
+
 ui.main()

vunit/test/acceptance/artificial/vhdl/tb_with_generic_config.vhd

@@ -0,0 +1,57 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at http://mozilla.org/MPL/2.0/.
+--
+-- Copyright (c) 2015, Lars Asplund lars.anders.asplund@gmail.com
+
+use std.textio.all;
+
+library vunit_lib;
+context vunit_lib.vunit_context;
+
+entity tb_with_generic_config is
+  generic (
+    runner_cfg : runner_cfg_t;
+    output_path : string;
+    set_generic : string := "default";
+    config_generic : string := "default");
+end entity;
+
+architecture tb of tb_with_generic_config is
+begin
+  test_runner : process
+    file fptr : text;
+  begin
+    test_runner_setup(runner, runner_cfg);
+
+    while test_suite loop
+      if run("Test 0") then
+        assert set_generic = "set-for-entity";
+        assert config_generic = "default";
+
+      elsif run("Test 1") then
+        assert set_generic = "set-for-entity";
+        assert config_generic = "set-from-config";
+
+      elsif run("Test 2") then
+        assert set_generic = "set-for-test";
+        assert config_generic = "default";
+
+      elsif run("Test 3") then
+        assert set_generic = "set-for-test";
+        assert config_generic = "set-from-config";
+
+      elsif run("Test 4") then
+        assert set_generic = "set-from-config";
+        assert config_generic = "set-from-config";
+        file_open(fptr, output_path & "post_check.txt", write_mode);
+        write(fptr, string'("Test 4 was here"));
+        file_close(fptr);
+      end if;
+    end loop;
+
+    test_runner_cleanup(runner);
+    wait;
+  end process;
+  test_runner_watchdog(runner, 50 ns);
+end architecture;

vunit/test/acceptance/test_artificial.py

@@ -41,33 +41,16 @@ def test_artificial_elaborate_only(self):
         self.check(self.artificial_run,
                    exit_code=1,
                    args=["--elaborate"])
-        check_report(self.report_file, [
-            ("passed", "lib.tb_pass"),
-            ("passed", "lib.tb_fail"),
-            ("passed", "lib.tb_infinite_events"),
-            ("passed", "lib.tb_fail_on_warning"),
-            ("passed", "lib.tb_no_fail_on_warning"),
-            ("passed", "lib.tb_two_architectures.pass"),
-            ("passed", "lib.tb_two_architectures.fail"),
-            ("passed", "lib.tb_with_vhdl_runner.pass"),
-            ("passed", "lib.tb_with_vhdl_runner.Test with spaces"),
-            ("passed", "lib.tb_with_vhdl_runner.fail"),
-            ("passed", "lib.tb_with_vhdl_runner.Test that timeouts"),
-            ("passed", "lib.tb_magic_paths"),
-            ("passed", "lib.tb_no_fail_after_cleanup"),
-            ("failed", "lib.tb_elab_fail"),
-
-            ("passed", "lib.tb_same_sim_all_pass.Test 1"),
-            ("passed", "lib.tb_same_sim_all_pass.Test 2"),
-            ("passed", "lib.tb_same_sim_all_pass.Test 3"),
-
-            ("passed", "lib.tb_same_sim_some_fail.Test 1"),
-            ("passed", "lib.tb_same_sim_some_fail.Test 2"),
-            ("passed", "lib.tb_same_sim_some_fail.Test 3"),
-
-            ("passed", "lib.tb_with_checks.Test passing check"),
-            ("passed", "lib.tb_with_checks.Test failing check"),
-            ("passed", "lib.tb_with_checks.Test non-stopping failing check")])
+
+        elab_expected_report = []
+        for status, name in EXPECTED_REPORT:
+            if name in ("lib.tb_elab_fail",):
+                status = "failed"
+            else:
+                status = "passed"
+            elab_expected_report.append((status, name))
+
+        check_report(self.report_file, elab_expected_report)
 
         self.check(self.artificial_run,
                    exit_code=0,
@@ -91,34 +74,7 @@ def _test_artificial(self, args=None):
         self.check(self.artificial_run,
                    exit_code=1,
                    args=args)
-        check_report(self.report_file, [
-            ("passed", "lib.tb_pass"),
-            ("failed", "lib.tb_fail"),
-            ("passed", "lib.tb_infinite_events"),
-            ("failed", "lib.tb_fail_on_warning"),
-            ("passed", "lib.tb_no_fail_on_warning"),
-            ("passed", "lib.tb_two_architectures.pass"),
-            ("failed", "lib.tb_two_architectures.fail"),
-            ("passed", "lib.tb_with_vhdl_runner.pass"),
-            ("passed", "lib.tb_with_vhdl_runner.Test with spaces"),
-            ("failed", "lib.tb_with_vhdl_runner.fail"),
-            ("failed", "lib.tb_with_vhdl_runner.Test that timeouts"),
-            ("passed", "lib.tb_magic_paths"),
-            ("passed", "lib.tb_no_fail_after_cleanup"),
-            ("failed", "lib.tb_elab_fail"),
-
-            # @TODO verify that these are actually run in separate simulations
-            ("passed", "lib.tb_same_sim_all_pass.Test 1"),
-            ("passed", "lib.tb_same_sim_all_pass.Test 2"),
-            ("passed", "lib.tb_same_sim_all_pass.Test 3"),
-
-            ("passed", "lib.tb_same_sim_some_fail.Test 1"),
-            ("failed", "lib.tb_same_sim_some_fail.Test 2"),
-            ("skipped", "lib.tb_same_sim_some_fail.Test 3"),
-
-            ("passed", "lib.tb_with_checks.Test passing check"),
-            ("failed", "lib.tb_with_checks.Test failing check"),
-            ("failed", "lib.tb_with_checks.Test non-stopping failing check")])
+        check_report(self.report_file, EXPECTED_REPORT)
 
     def test_run_selected_tests_in_same_sim_test_bench(self):
         self.check(self.artificial_run,
@@ -181,3 +137,39 @@ def check(self, run_file, args=None, persistent_sim=True, clean=True, exit_code=
                         "--xunit-xml=%s" % self.report_file] + args,
                        env=new_env)
         self.assertEqual(retcode, exit_code)
+
+
+EXPECTED_REPORT = (
+    ("passed", "lib.tb_pass"),
+    ("failed", "lib.tb_fail"),
+    ("passed", "lib.tb_infinite_events"),
+    ("failed", "lib.tb_fail_on_warning"),
+    ("passed", "lib.tb_no_fail_on_warning"),
+    ("passed", "lib.tb_two_architectures.pass"),
+    ("failed", "lib.tb_two_architectures.fail"),
+    ("passed", "lib.tb_with_vhdl_runner.pass"),
+    ("passed", "lib.tb_with_vhdl_runner.Test with spaces"),
+    ("failed", "lib.tb_with_vhdl_runner.fail"),
+    ("failed", "lib.tb_with_vhdl_runner.Test that timeouts"),
+    ("passed", "lib.tb_magic_paths"),
+    ("passed", "lib.tb_no_fail_after_cleanup"),
+    ("failed", "lib.tb_elab_fail"),
+
+    # @TODO verify that these are actually run in separate simulations
+    ("passed", "lib.tb_same_sim_all_pass.Test 1"),
+    ("passed", "lib.tb_same_sim_all_pass.Test 2"),
+    ("passed", "lib.tb_same_sim_all_pass.Test 3"),
+
+    ("passed", "lib.tb_same_sim_some_fail.Test 1"),
+    ("failed", "lib.tb_same_sim_some_fail.Test 2"),
+    ("skipped", "lib.tb_same_sim_some_fail.Test 3"),
+
+    ("passed", "lib.tb_with_checks.Test passing check"),
+    ("failed", "lib.tb_with_checks.Test failing check"),
+    ("failed", "lib.tb_with_checks.Test non-stopping failing check"),
+
+    ("passed", "lib.tb_with_generic_config.Test 0"),
+    ("passed", "lib.tb_with_generic_config.Test 1"),
+    ("passed", "lib.tb_with_generic_config.Test 2"),
+    ("passed", "lib.tb_with_generic_config.Test 3"),
+    ("passed", "lib.tb_with_generic_config.Test 4"))

vunit/test/acceptance/test_external_run_scripts.py

@@ -39,6 +39,16 @@ def test_check_example_project(self):
 
     def test_generate_tests_example_project(self):
         self.check(join(ROOT, "examples", "generate_tests", "run.py"))
+        check_report(self.report_file,
+                     [("passed", "lib.tb_generated.data_width=1,sign=False.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=1,sign=True.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=2,sign=False.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=2,sign=True.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=3,sign=False.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=3,sign=True.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=4,sign=False.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=4,sign=True.Test 1"),
+                      ("passed", "lib.tb_generated.data_width=16,sign=True.Test 2")])
 
     def test_array_example_project(self):
         self.check(join(ROOT, "examples", "array", "run.py"))