add code coverage support for ghdl (#627)

* sim_if: add supports_coverage method
* Expose supports_coverage to user
* Add code coverage support for ghdl
Simulation now puts the compile-time gcno with the compiled .o-files, and run-time gcda folders within the output folder for each test
Each test case produces a separate .gcda file.
* Hopefully make ghdl coverage work when using multiple threads
* Add merge_coverage method for GHDLInterface
* Update coverage example for GHDL
* Adapt for usage of Pathlib
* Better handling of empty glob results
* Correct comment
* Minor fixes after review by LukasVik
* Add enable_coverage for source files
* Fix linting
* Add coverage example to acceptance tests
* Updates after review
* Add documentation for the enable_coverage flags for GHDL

Co-authored-by: Lukas Vik <10241915+LukasVik@users.noreply.github.com>

Author: LudvigVidlid

docs/py/opts.rst

@@ -46,6 +46,11 @@ The following compilation options are known.
    Extra arguments passed to Active HDL ``vcom`` command.
    Must be a list of strings.
 
+``enable_coverage``
+   Enables compilation flags needed for code coverage and tells VUnit to handle
+   the coverage files created at compilation. Only used for coverage with GHDL.
+   Must be a boolean value. Default is False.
+
 .. note::
    Only affects source files added *before* the option is set.
 
@@ -73,17 +78,22 @@ The following simulation options are known.
   Must be a boolean value. Default is False.
 
   When coverage is enabled VUnit only takes the minimal steps required
-  to make the simulator creates an unique coverage file for the
-  simulation run. The VUnit users must still set :ref:`sim
+  to make the simulator create a unique coverage file for the
+  simulation run.
+
+  For RiverieraPRO and Modelsim/Questa, the VUnit users must still set :ref:`sim
   <sim_options>` and :ref:`compile <compile_options>` options to
   configure the simulator specific coverage options they want. The
   reason for this to allow the VUnit users maximum control of their
   coverage settings.
 
+  For GHDL with GCC backend there is less configurability for coverage, and all
+  necessary flags are set by the the ``enable_coverage`` sim and compile options.
+
   An example of a ``run.py`` file using coverage can be found
   :vunit_example:`here <vhdl/coverage>`.
 
-  .. note: Supported by RivieraPRO and Modelsim/Questa simulators.
+  .. note: Supported by GHDL with GCC backend, RivieraPRO and Modelsim/Questa simulators.
 
 
 ``pli``

examples/vhdl/coverage/run.py

@@ -6,21 +6,26 @@
 
 from pathlib import Path
 from vunit import VUnit
+from subprocess import call
 
 
 def post_run(results):
     results.merge_coverage(file_name="coverage_data")
+    if VU.get_simulator_name() == "ghdl":
+        call(["gcovr", "coverage_data"])
 
 
 VU = VUnit.from_argv()
 
 LIB = VU.add_library("lib")
 LIB.add_source_files(Path(__file__).parent / "*.vhd")
 
+LIB.set_sim_option("enable_coverage", True)
+
 LIB.set_compile_option("rivierapro.vcom_flags", ["-coverage", "bs"])
 LIB.set_compile_option("rivierapro.vlog_flags", ["-coverage", "bs"])
 LIB.set_compile_option("modelsim.vcom_flags", ["+cover=bs"])
 LIB.set_compile_option("modelsim.vlog_flags", ["+cover=bs"])
-LIB.set_sim_option("enable_coverage", True)
+LIB.set_compile_option("enable_coverage", True)
 
 VU.main(post_run=post_run)

tests/acceptance/test_external_run_scripts.py

@@ -121,6 +121,13 @@ def test_vhdl_third_party_integration_example_project(self):
     def test_vhdl_check_example_project(self):
         self.check(ROOT / "examples" / "vhdl" / "check" / "run.py")
 
+    @unittest.skipIf(
+        simulator_check(lambda simclass: not simclass.supports_coverage()),
+        "This simulator/backend does not support coverage",
+    )
+    def test_vhdl_coverage_example_project(self):
+        self.check(join(ROOT, "examples", "vhdl", "coverage", "run.py"))
+
     def test_vhdl_generate_tests_example_project(self):
         self.check(ROOT / "examples" / "vhdl" / "generate_tests" / "run.py")
         check_report(

vunit/sim_if/__init__.py

@@ -172,7 +172,14 @@ def has_valid_exit_code():
     @staticmethod
     def supports_vhpi():
         """
-        Return if the simulator supports VHPI
+        Returns True when the simulator supports VHPI
+        """
+        return False
+
+    @staticmethod
+    def supports_coverage():
+        """
+        Returns True when the simulator supports coverage
         """
         return False
 
@@ -216,7 +223,7 @@ def setup_library_mapping(self, project):
         Implemented by specific simulators
         """
 
-    def __compile_source_file(self, source_file, printer):
+    def _compile_source_file(self, source_file, printer):
         """
         Compiles a single source file and prints status information
         """
@@ -297,7 +304,7 @@ def compile_source_files(
                 printer.write("\n")
                 continue
 
-            if self.__compile_source_file(source_file, printer):
+            if self._compile_source_file(source_file, printer):
                 project.update(source_file)
             else:
                 source_files_to_skip.update(

vunit/sim_if/activehdl.py

@@ -68,6 +68,13 @@ def supports_vhdl_package_generics(cls):
 
         return False
 
+    @staticmethod
+    def supports_coverage():
+        """
+        Returns True when the simulator supports coverage
+        """
+        return True
+
     def __init__(self, prefix, output_path, gui=False):
         SimulatorInterface.__init__(self, output_path, gui)
         self._library_cfg = str(Path(output_path) / "library.cfg")

vunit/sim_if/factory.py

@@ -39,7 +39,7 @@ def _extract_compile_options(self):
         """
         Return all supported compile options
         """
-        result = dict()
+        result = dict((opt.name, opt) for opt in [BooleanOption("enable_coverage")])
         for sim_class in self.supported_simulators():
             for opt in sim_class.compile_options:
                 assert hasattr(opt, "name")

vunit/sim_if/ghdl.py

@@ -14,6 +14,7 @@
 import subprocess
 import shlex
 import re
+import shutil
 from json import dump
 from sys import stdout  # To avoid output catched in non-verbose mode
 from warnings import warn
@@ -25,7 +26,7 @@
 LOGGER = logging.getLogger(__name__)
 
 
-class GHDLInterface(SimulatorInterface):
+class GHDLInterface(SimulatorInterface):  # pylint: disable=too-many-instance-attributes
     """
     Interface for GHDL simulator
     """
@@ -108,6 +109,7 @@ def __init__(  # pylint: disable=too-many-arguments
         self._gtkwave_args = gtkwave_args
         self._backend = backend
         self._vhdl_standard = None
+        self._coverage_test_dirs = set()
 
     def has_valid_exit_code(self):
         """
@@ -164,12 +166,19 @@ def determine_version(cls, prefix):
     @classmethod
     def supports_vhpi(cls):
         """
-        Return if the simulator supports VHPI
+        Returns True when the simulator supports VHPI
         """
         return (cls.determine_backend(cls.find_prefix_from_path()) != "mcode") or (
             cls.determine_version(cls.find_prefix_from_path()) > 0.36
         )
 
+    @classmethod
+    def supports_coverage(cls):
+        """
+        Returns True when the simulator supports coverage
+        """
+        return cls.determine_backend(cls.find_prefix_from_path()) == "gcc"
+
     def _has_output_flag(self):
         """
         Returns if backend supports output flag
@@ -254,10 +263,18 @@ def compile_vhdl_file_command(self, source_file):
             a_flags += flags
 
         cmd += a_flags
+
+        if source_file.compile_options.get("enable_coverage", False):
+            # Add gcc compilation flags for coverage
+            #   -ftest-coverages creates .gcno notes files needed by gcov
+            #   -fprofile-arcs creates branch profiling in .gcda database files
+            cmd += ["-fprofile-arcs", "-ftest-coverage"]
         cmd += [source_file.name]
         return cmd
 
-    def _get_command(self, config, output_path, elaborate_only, ghdl_e, wave_file):
+    def _get_command(  # pylint: disable=too-many-branches
+        self, config, output_path, elaborate_only, ghdl_e, wave_file
+    ):
         """
         Return GHDL simulation command
         """
@@ -282,6 +299,9 @@ def _get_command(self, config, output_path, elaborate_only, ghdl_e, wave_file):
         if self._has_output_flag():
             cmd += ["-o", bin_path]
         cmd += config.sim_options.get("ghdl.elab_flags", [])
+        if config.sim_options.get("enable_coverage", False):
+            # Enable coverage in linker
+            cmd += ["-Wl,-lgcov"]
         cmd += [config.entity_name, config.architecture_name]
 
         sim = config.sim_options.get("ghdl.sim_flags", [])
@@ -347,8 +367,16 @@ def simulate(  # pylint: disable=too-many-locals
         )
 
         status = True
+
+        gcov_env = environ.copy()
+        if config.sim_options.get("enable_coverage", False):
+            # Set environment variable to put the coverage output in the test_output folder
+            coverage_dir = str(Path(output_path) / "coverage")
+            gcov_env["GCOV_PREFIX"] = coverage_dir
+            self._coverage_test_dirs.add(coverage_dir)
+
         try:
-            proc = Process(cmd)
+            proc = Process(cmd, env=gcov_env)
             proc.consume_output()
         except Process.NonZeroExitCode:
             status = False
@@ -364,3 +392,54 @@ def simulate(  # pylint: disable=too-many-locals
             subprocess.call(cmd)
 
         return status
+
+    def _compile_source_file(self, source_file, printer):
+        """
+        Runs parent command for compilation, and moves any .gcno files to the compilation output
+        """
+        compilation_ok = super()._compile_source_file(source_file, printer)
+
+        if source_file.compile_options.get("enable_coverage", False):
+            # GCOV gcno files are output to where the command is run,
+            # move it back to the compilation folder
+            source_path = Path(source_file.name)
+            gcno_file = Path(source_path.stem + ".gcno")
+            if Path(gcno_file).exists():
+                new_path = Path(source_file.library.directory) / gcno_file
+                gcno_file.rename(new_path)
+
+        return compilation_ok
+
+    def merge_coverage(self, file_name, args=None):
+        """
+        Merge coverage from all test cases
+        """
+        output_dir = file_name
+
+        # Loop over each .gcda output folder and merge them two at a time
+        first_input = True
+        for coverage_dir in self._coverage_test_dirs:
+            if Path(coverage_dir).exists():
+                merge_command = [
+                    "gcov-tool",
+                    "merge",
+                    "-o",
+                    output_dir,
+                    coverage_dir if first_input else output_dir,
+                    coverage_dir,
+                ]
+                subprocess.call(merge_command)
+                first_input = False
+            else:
+                LOGGER.warning("Missing coverage directory: %s", coverage_dir)
+
+        # Find actual output path of the .gcda files (they are deep in hierarchy)
+        dir_path = Path(output_dir)
+        gcda_dirs = {x.parent for x in dir_path.glob("**/*.gcda")}
+        assert len(gcda_dirs) == 1, "Expected exactly one folder with gcda files"
+        gcda_dir = gcda_dirs.pop()
+
+        # Add compile-time .gcno files as well, they are needed for the report
+        for library in self._project.get_libraries():
+            for gcno_file in Path(library.directory).glob("*.gcno"):
+                shutil.copy(gcno_file, gcda_dir)

vunit/sim_if/modelsim.py

@@ -81,6 +81,13 @@ def supports_vhdl_package_generics(cls):
         """
         return True
 
+    @staticmethod
+    def supports_coverage():
+        """
+        Returns True when the simulator supports coverage
+        """
+        return True
+
     def __init__(self, prefix, output_path, persistent=False, gui=False):
         SimulatorInterface.__init__(self, output_path, gui)
         VsimSimulatorMixin.__init__(

vunit/sim_if/rivierapro.py

@@ -95,6 +95,13 @@ def supports_vhdl_package_generics(cls):
         """
         return True
 
+    @staticmethod
+    def supports_coverage():
+        """
+        Returns True when the simulator supports coverage
+        """
+        return True
+
     def __init__(self, prefix, output_path, persistent=False, gui=False):
         SimulatorInterface.__init__(self, output_path, gui)
         VsimSimulatorMixin.__init__(

vunit/ui/__init__.py

@@ -1099,3 +1099,13 @@ def get_simulator_name(self):
         if self._simulator_class is None:
             return None
         return self._simulator_class.name
+
+    def simulator_supports_coverage(self):
+        """
+        Returns True when the simulator supports coverage
+
+        Will return None if no simulator was found.
+        """
+        if self._simulator_class is None:
+            return None
+        return self._simulator_class.supports_coverage