feat(project CreateBom): write control file with attachment details

This can be used to check meta data of attachments before feeding the
list into "bom downloadAttachments".

Author: gernot-h

capycli/main/options.py

@@ -219,6 +219,13 @@ def register_options(self) -> None:
             help="create an mapping overview JSON file",
         )
 
+        self.parser.add_argument(
+            "-ct",
+            "--controlfile",
+            dest="controlfile",
+            help="control file for \"bom DownloadAttachments\" and \"project CreateReadme\"",
+        )
+
         self.parser.add_argument(
             "-mr",
             "--mapresult",

capycli/project/create_bom.py

@@ -8,7 +8,8 @@
 
 import logging
 import sys
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Tuple
+import json
 
 from cyclonedx.model import ExternalReferenceType, HashAlgorithm
 from cyclonedx.model.bom import Bom
@@ -21,6 +22,7 @@
 from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport, SbomCreator
 from capycli.common.print import print_red, print_text, print_yellow
 from capycli.common.purl_utils import PurlUtils
+from capycli.common.script_support import ScriptSupport
 from capycli.main.result_codes import ResultCode
 
 LOG = get_logger(__name__)
@@ -45,8 +47,9 @@ def get_clearing_state(self, proj: Dict[str, Any], href: str) -> str:
 
         return ""
 
-    def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
+    def create_project_bom(self, project: Dict[str, Any], create_controlfile: bool) -> Tuple[List, List]:
         bom: List[Component] = []
+        details: List[Dist] = []
         if not self.client:
             print_red("  No client!")
             sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
@@ -100,7 +103,6 @@ def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
                 if "repository" in release_details and "url" in release_details["repository"]:
                     CycloneDxSupport.set_ext_ref(rel_item, ExternalReferenceType.VCS, comment="",
                                                  value=release_details["repository"]["url"])
-
                 attachments = self.get_release_attachments(release_details)
                 for attachment in attachments:
                     at_type = attachment["attachmentType"]
@@ -111,8 +113,23 @@ def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
                         ext_ref_type = ExternalReferenceType.DISTRIBUTION
                     else:
                         ext_ref_type = ExternalReferenceType.OTHER
-                        comment += (", sw360Id: "
-                                    + self.client.get_id_from_href(attachment["_links"]["self"]["href"]))
+                        if create_controlfile:
+                            at_data = self.client.get_attachment_by_url(attachment["_links"]["self"]["href"])
+
+                            at_details = {
+                                "ComponentName": " ".join((release["name"], release["version"])),
+                                "Sw360Id": sw360_id,
+                                "Sw360AttachmentId": self.client.get_id_from_href(attachment["_links"]["self"]["href"])}
+                            for key in ("createdBy", "createdTeam", "createdOn", "createdComment", "checkStatus",
+                                        "checkedBy", "checkedTeam", "checkedOn", "checkedComment"):
+                                if key in at_data and at_data[key]:
+                                    at_details[key[0].upper() + key[1:]] = at_data[key]
+
+                            if at_type == "COMPONENT_LICENSE_INFO_XML":
+                                at_details["CliFile"] = attachment["filename"]
+                            elif at_type == "CLEARING_REPORT":
+                                at_details["ReportFile"] = attachment["filename"]
+                            details.append(at_details)
                     CycloneDxSupport.set_ext_ref(rel_item, ext_ref_type,
                                                  comment, attachment["filename"],
                                                  HashAlgorithm.SHA_1, attachment.get("sha1"))
@@ -136,9 +153,9 @@ def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
 
         # sub-projects are not handled at the moment
 
-        return bom
+        return bom, details
 
-    def create_project_cdx_bom(self, project_id: str) -> Bom:
+    def create_project_cdx_bom(self, project_id: str, create_controlfile:bool) -> Tuple[Bom, Dict]:
         if not self.client:
             print_red("  No client!")
             sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
@@ -154,14 +171,19 @@ def create_project_cdx_bom(self, project_id: str) -> Bom:
 
         print_text("  Project name: " + project["name"] + ", " + project["version"])
 
-        cdx_components = self.create_project_bom(project)
+        cdx_components, control_components = self.create_project_bom(project, create_controlfile)
 
         creator = SbomCreator()
         sbom = creator.create(cdx_components, addlicense=True, addprofile=True, addtools=True,
                               name=project.get("name", ""), version=project.get("version", ""),
                               description=project.get("description", ""), addprojectdependencies=True)
 
-        return sbom
+        controlfile = {
+            "ProjectName": ScriptSupport.get_full_name_from_dict(project, "name", "version"),
+            "Components": control_components
+        }
+
+        return sbom, controlfile
 
     def show_command_help(self) -> None:
         print("\nusage: CaPyCli project createbom [options]")
@@ -174,6 +196,7 @@ def show_command_help(self) -> None:
   -name            name of the project, component or release
   -version         version of the project, component or release
   -o OUTPUTFILE    output file to write to
+  -ct CONTROLFILE  write control file for "bom DownloadAttachments" and "project CreateReadme"
         """)
 
         print()
@@ -220,7 +243,11 @@ def run(self, args: Any) -> None:
             sys.exit(ResultCode.RESULT_COMMAND_ERROR)
 
         if pid:
-            bom = self.create_project_cdx_bom(pid)
+            bom, controlfile = self.create_project_cdx_bom(pid, args.controlfile)
             CaPyCliBom.write_sbom(bom, args.outputfile)
+
+            if args.controlfile:
+                with open(args.controlfile, "w") as outfile:
+                    json.dump(controlfile, outfile, indent=2)
         else:
             print_yellow("  No matching project found")

tests/fixtures/sbom_for_download.json

@@ -91,7 +91,7 @@
         },
 	{
           "url": "CLIXML_certifi-2022.12.7.xml",
-          "comment": "component license information (local copy), sw360Id: 794446",
+          "comment": "component license information (local copy)",
           "type": "other",
           "hashes": [
             {
@@ -102,7 +102,7 @@
         },
         {
           "url": "certifi-2022.12.7_clearing_report.docx",
-          "comment": "clearing report (local copy), sw360Id: 63b368",
+          "comment": "clearing report (local copy)",
           "type": "other",
           "hashes": [
             {

tests/test_create_bom.py

@@ -103,6 +103,7 @@ def test_project_not_found(self) -> None:
         args.verbose = True
         args.id = "34ef5c5452014c52aa9ce4bc180624d8"
         args.outputfile = self.OUTPUTFILE
+        args.controlfile = None
 
         self.add_login_response()
 
@@ -150,27 +151,21 @@ def test_create_bom_multiple_purls(self, capsys: Any) -> None:
             adding_headers={"Authorization": "Token " + self.MYTOKEN},
         )
 
-        cdx_components = sut.create_project_bom(self.get_project_for_test())
+        cdx_components, _ = sut.create_project_bom(self.get_project_for_test(),
+                                                   create_controlfile=False)
         captured = capsys.readouterr()
 
         assert "Multiple purls added" in captured.out
         assert cdx_components[0].purl is not None
         if cdx_components[0].purl:
             assert cdx_components[0].purl.to_string() == "pkg:deb/debian/cli-support%401.3-1%20pkg:pypi/cli-support@1.3"
 
-    @responses.activate
-    def test_project_by_id(self) -> None:
-        sut = CreateBom()
-
-        self.add_login_response()
-        sut.login(token=TestBasePytest.MYTOKEN, url=TestBasePytest.MYURL)
-
+    def add_project_releases_responses(self):
         # the project
-        project = self.get_project_for_test()
         responses.add(
             responses.GET,
             url=self.MYURL + "resource/api/projects/p001",
-            json=project,
+            json=self.get_project_for_test(),
             status=200,
             content_type="application/json",
             adding_headers={"Authorization": "Token " + self.MYTOKEN},
@@ -197,7 +192,7 @@ def test_project_by_id(self) -> None:
             "attachmentType": "SOURCE_SELF",
             "_links": {
                 "self": {
-                    "href": "https://my.server.com/resource/api/attachments/r002a002"
+                    "href": "https://my.server.com/resource/api/attachments/r002a003"
                 }
             }
         })
@@ -207,7 +202,7 @@ def test_project_by_id(self) -> None:
             "attachmentType": "CLEARING_REPORT",
             "_links": {
                 "self": {
-                    "href": "https://my.server.com/resource/api/attachments/r002a003"
+                    "href": "https://my.server.com/resource/api/attachments/r002a004"
                 }
             }
         })
@@ -220,8 +215,19 @@ def test_project_by_id(self) -> None:
             content_type="application/json",
             adding_headers={"Authorization": "Token " + self.MYTOKEN},
         )
+        return release
 
-        cdx_bom = sut.create_project_cdx_bom("p001")
+    @responses.activate
+    def test_project_by_id(self) -> None:
+        sut = CreateBom()
+
+        self.add_login_response()
+        sut.login(token=TestBasePytest.MYTOKEN, url=TestBasePytest.MYURL)
+
+        release = self.add_project_releases_responses()
+        project = self.get_project_for_test()
+
+        cdx_bom, _ = sut.create_project_cdx_bom("p001", create_controlfile=False)
         cx_comp = cdx_bom.components[0]
         assert cx_comp.purl.to_string() == release["externalIds"]["package-url"]
 
@@ -242,15 +248,15 @@ def test_project_by_id(self) -> None:
         assert len(ext_refs) == 1
         assert str(ext_refs[0].url) == release["_embedded"]["sw360:attachments"][1]["filename"]
         assert ext_refs[0].type == ExternalReferenceType.OTHER
-        assert ext_refs[0].comment, CaPyCliBom.CLI_FILE_COMMENT + " == sw360Id: r002a002"
+        assert ext_refs[0].comment == CaPyCliBom.CLI_FILE_COMMENT
         assert ext_refs[0].hashes[0].alg == "SHA-1"
         assert ext_refs[0].hashes[0].content == release["_embedded"]["sw360:attachments"][1]["sha1"]
 
         ext_refs = [e for e in cx_comp.external_references
                     if e.comment and e.comment.startswith(CaPyCliBom.CRT_FILE_COMMENT)]
         assert len(ext_refs) == 1
         assert str(ext_refs[0].url) == release["_embedded"]["sw360:attachments"][3]["filename"]
-        assert ext_refs[0].comment, CaPyCliBom.CRT_FILE_COMMENT + " == sw360Id: r002a003"
+        assert ext_refs[0].comment == CaPyCliBom.CRT_FILE_COMMENT
         assert ext_refs[0].type == ExternalReferenceType.OTHER
         assert ext_refs[0].hashes[0].alg == "SHA-1"
         assert ext_refs[0].hashes[0].content == release["_embedded"]["sw360:attachments"][3]["sha1"]
@@ -265,6 +271,71 @@ def test_project_by_id(self) -> None:
             assert cdx_bom.metadata.component.version == project["version"]
             assert cdx_bom.metadata.component.description == project["description"]
 
+    @responses.activate
+    def test_project_by_id_controlfile(self):
+        sut = CreateBom()
+        self.add_login_response()
+        sut.login(token=TestBasePytest.MYTOKEN, url=TestBasePytest.MYURL)
+
+        self.add_project_releases_responses()
+
+        # attachment info
+        responses.add(
+            method=responses.GET,
+            url=self.MYURL + "resource/api/attachments/r001a001",
+            body="""
+                {
+                    "filename": "CLIXML_wheel-0.38.4.xml",
+                    "sha1": "ccd9f1ed2f59c46ff3f0139c05bfd76f83fd9851",
+                    "attachmentType": "COMPONENT_LICENSE_INFO_XML"
+                }""",
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+        responses.add(
+            method=responses.GET,
+            url=self.MYURL + "resource/api/attachments/r002a002",
+            body="""
+                {
+                    "filename": "CLIXML_clipython-1.3.0.xml",
+                    "sha1": "dd4c38387c6811dba67d837af7742d84e61e20de",
+                    "attachmentType": "COMPONENT_LICENSE_INFO_XML",
+                    "checkedBy": "user2@siemens.com",
+                    "checkStatus": "ACCEPTED",
+                    "createdBy": "user1@siemens.com"
+                }""",
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+        responses.add(
+            method=responses.GET,
+            url=self.MYURL + "resource/api/attachments/r002a004",
+            body="""
+                {
+                    "filename": "clipython-1.3.0.docx",
+                    "sha1": "f0d8f2ddd017bdeaecbaec72ff76a6c0a045ec66",
+                    "attachmentType": "CLEARING_REPORT"
+
+                }""",
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+
+        _, controlfile = sut.create_project_cdx_bom("p001", create_controlfile=True)
+        assert controlfile['ProjectName'] == 'CaPyCLI, 1.9.0'
+        assert controlfile['Components'][0]['ComponentName'] == 'cli-support 1.3'
+        assert controlfile['Components'][0]['Sw360Id'] == 'r002'
+        assert controlfile['Components'][0]['Sw360AttachmentId'] == 'r002a002'
+        assert controlfile['Components'][0]['CliFile'] == 'CLIXML_clipython-1.3.0.xml'
+        assert controlfile['Components'][0]['CheckedBy'] == 'user2@siemens.com'
+        assert controlfile['Components'][0]['CheckStatus'] == 'ACCEPTED'
+        assert controlfile['Components'][0]['CreatedBy'] == 'user1@siemens.com'
+
+        assert controlfile['Components'][1]['ReportFile'] == 'clipython-1.3.0.docx'
+
     @responses.activate
     def test_project_show_by_name(self) -> None:
         sut = CreateBom()
@@ -280,6 +351,7 @@ def test_project_show_by_name(self) -> None:
         args.name = "CaPyCLI"
         args.version = "1.9.0"
         args.outputfile = self.OUTPUTFILE
+        args.controlfile = None
 
         self.add_login_response()
 
@@ -363,6 +435,65 @@ def test_project_show_by_name(self) -> None:
 
         self.delete_file(self.OUTPUTFILE)
 
+    @responses.activate
+    def test_create_project_bom_release_error(self):
+        sut = CreateBom()
+
+        self.add_login_response()
+        sut.login(token=TestBasePytest.MYTOKEN, url=TestBasePytest.MYURL)
+
+        responses.add(
+            responses.GET,
+            url=self.MYURL + "resource/api/releases/r001",
+            status=404,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+        responses.add(
+            responses.GET,
+            url=self.MYURL + "resource/api/releases/r002",
+            json=self.get_release_cli_for_test(),
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+        with pytest.raises(SystemExit):
+            bom, _ = sut.create_project_bom(self.get_project_for_test(), create_controlfile=False)
+
+    @responses.activate
+    def test_create_project_bom_controlfile_attachment_error(self):
+        sut = CreateBom()
+
+        self.add_login_response()
+        sut.login(token=TestBasePytest.MYTOKEN, url=TestBasePytest.MYURL)
+
+        responses.add(
+            responses.GET,
+            url=self.MYURL + "resource/api/releases/r001",
+            json=self.get_release_wheel_for_test(),
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+        responses.add(
+            responses.GET,
+            url=self.MYURL + "resource/api/releases/r002",
+            json=self.get_release_cli_for_test(),
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+        responses.add(
+            method=responses.GET,
+            url=self.MYURL + "resource/api/attachments/r002a002",
+            status=404,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+
+        with pytest.raises(SystemExit):
+            bom, _ = sut.create_project_bom(self.get_project_for_test(), create_controlfile=True)
+
 
 if __name__ == "__main__":
     APP = TestCreateBom()