diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7da..135e6189c6f 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -87,6 +87,7 @@ The following organizations or individuals have contributed to ScanCode: - Shrijal Acharya @OctoPie23 - Shivam Chauhan @chashiv - Shivam Sandbhor @sbs2001 +- Shuvagata Dey @0Cymantek0 - Steven Esser @majurg - Sushant Gupta @susg - Theodore Aptekarev @piiq diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0bc400407d3..6c10ddf8678 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,12 @@ Next release ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 +- Fix the ``--cyclonedx`` JSON output to use strings for all + ``metadata.properties[].value`` fields as required by the CycloneDX + specification. Lists of messages are joined with newlines, mappings are + serialized to JSON and empty or null values are dropped. + https://github.com/aboutcode-org/scancode-toolkit/issues/5349 + - Improve copyright detection for statements with parens or trailing "authors" - Add experimental option for using cached results during scan time. When the diff --git a/src/formattedcode/output_cyclonedx.py b/src/formattedcode/output_cyclonedx.py index 3ffeb98e5e6..b5352a688ef 100644 --- a/src/formattedcode/output_cyclonedx.py +++ b/src/formattedcode/output_cyclonedx.py @@ -566,6 +566,28 @@ def get_ts(): return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') +def get_property_string_value(value): + """ + Return a CycloneDX property value as a string, or None when there is no + value to report. CycloneDX property values are always strings: lists of + messages are joined with newlines, mappings are serialized to JSON and + anything else is stringified. Falsy values such as None, empty strings + and empty collections return None. Numbers and booleans, including 0 and + False, are stringified. + """ + if isinstance(value, (int, float)): + return str(value) + if not value: + return None + if isinstance(value, str): + return value + if isinstance(value, (list, tuple)): + return '\n'.join(str(item) for item in value) + if isinstance(value, dict): + return json.dumps(value) + return str(value) + + @attr.s class CycloneDxMetadata: """ @@ -615,7 +637,12 @@ def from_headers(cls, headers): message=scancode_header.get('message'), ) props.update(scancode_header.get('extra_data', {})) - properties = [CycloneDxProperty(k, v) for k, v in props.items()] + properties = [] + for name, value in props.items(): + value = get_property_string_value(value) + if value is None: + continue + properties.append(CycloneDxProperty(name, value)) if TRACE: logger_debug('CycloneDxMetadata: properties') diff --git a/tests/formattedcode/test_output_cyclonedx.py b/tests/formattedcode/test_output_cyclonedx.py index c2b220d2b48..205bcda003f 100644 --- a/tests/formattedcode/test_output_cyclonedx.py +++ b/tests/formattedcode/test_output_cyclonedx.py @@ -22,6 +22,7 @@ from formattedcode.output_cyclonedx import CycloneDxLicenseExpression from formattedcode.output_cyclonedx import CycloneDxMetadata from formattedcode.output_cyclonedx import get_author_from_parties +from formattedcode.output_cyclonedx import get_property_string_value from scancode.cli_test_utils import run_scan_click from scancode_config import REGEN_TEST_FIXTURES @@ -196,7 +197,7 @@ def test_CycloneDxMetadata_from_headers(): 'tool_version': '31.1.1', 'notice': 'some notice', 'message': 'some message', - 'errors': ['some error'], + 'errors': ['some error', 'another error'], 'warnings': ['some warning'], 'extra_data': {'spdx_version': '3.1.2'} }] @@ -205,8 +206,8 @@ def test_CycloneDxMetadata_from_headers(): expected = { 'properties': [ {'name': 'notice', 'value': 'some notice'}, - {'name': 'errors', 'value': ['some error']}, - {'name': 'warnings', 'value': ['some warning']}, + {'name': 'errors', 'value': 'some error\nanother error'}, + {'name': 'warnings', 'value': 'some warning'}, {'name': 'message', 'value': 'some message'}, {'name': 'spdx_version', 'value': '3.1.2'}, ], @@ -217,6 +218,69 @@ def test_CycloneDxMetadata_from_headers(): assert m == expected +def test_CycloneDxMetadata_from_headers_property_values_are_always_strings(): + headers = [{ + 'tool_name': 'scancode-toolkit', + 'tool_version': '31.1.1', + 'notice': 'some notice', + 'errors': ['some error', 'another error'], + 'warnings': ['some warning'], + 'message': None, + 'extra_data': { + 'system_environment': {'operating_system': 'win', 'cpu_architecture': '64'}, + 'spdx_license_list_version': '3.28', + }, + }] + properties = CycloneDxMetadata.from_headers(headers).to_dict()['properties'] + # the CycloneDX schema types property values as strings in all versions + for prop in properties: + assert isinstance(prop['value'], str), prop + assert [p['name'] for p in properties] == [ + 'notice', + 'errors', + 'warnings', + 'system_environment', + 'spdx_license_list_version', + ] + assert properties[1]['value'] == 'some error\nanother error' + assert json.loads(properties[3]['value']) == { + 'operating_system': 'win', + 'cpu_architecture': '64', + } + + +def test_CycloneDxMetadata_from_headers_drops_empty_property_values(): + headers = [{ + 'tool_name': 'scancode-toolkit', + 'tool_version': '31.1.1', + 'notice': 'some notice', + 'errors': [], + 'warnings': [], + 'message': None, + 'extra_data': {'empty_mapping': {}, 'OUTDATED': 'a newer version is available'}, + }] + properties = CycloneDxMetadata.from_headers(headers).to_dict()['properties'] + assert properties == [ + {'name': 'notice', 'value': 'some notice'}, + {'name': 'OUTDATED', 'value': 'a newer version is available'}, + ] + + +def test_get_property_string_value(): + assert get_property_string_value('some notice') == 'some notice' + assert get_property_string_value(['error 1', 'error 2']) == 'error 1\nerror 2' + assert get_property_string_value(['error 1', 2]) == 'error 1\n2' + assert get_property_string_value({'operating_system': 'win'}) == ( + '{"operating_system": "win"}') + assert get_property_string_value(33) == '33' + assert get_property_string_value(0) == '0' + assert get_property_string_value(False) == 'False' + assert get_property_string_value(None) is None + assert get_property_string_value('') is None + assert get_property_string_value([]) is None + assert get_property_string_value({}) is None + + def test_cyclonedx_plugin_does_not_fail_without_packages(): test_dir = test_env.get_test_loc('cyclonedx/simple') result_file = test_env.get_temp_file('cyclonedx.json') @@ -241,6 +305,18 @@ def test_cyclonedx_plugin_json_simple_package_npm(): check_cyclone_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES) +def test_cyclonedx_json_metadata_property_values_are_strings(): + test_dir = test_env.get_test_loc('cyclonedx/simple') + result_file = test_env.get_temp_file('cyclonedx.json') + run_scan_click(['--package', test_dir, '--cyclonedx', result_file]) + with io.open(result_file, encoding='utf-8') as f: + bom = json.load(f) + properties = bom['metadata']['properties'] + assert properties + for prop in properties: + assert isinstance(prop['value'], str), prop + + def test_cyclonedx_plugin_xml_components_and_dependencies_are_serialized_correctly(): test_dir = test_env.get_test_loc('cyclonedx/simple') result_file = test_env.get_temp_file('cyclonedx.xml')