Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/packagedcode/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@
pypi.PypiEggHandler,
# pypi.PypiSdistArchiveHandler,
pypi.PypiWheelHandler,
pypi.PipWheelCacheOriginJsonHandler,
pypi.PyprojectTomlHandler,
pypi.PoetryPyprojectTomlHandler,
pypi.PoetryLockHandler,
Expand Down
114 changes: 114 additions & 0 deletions src/packagedcode/pypi.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from fnmatch import fnmatchcase
from pathlib import Path
from typing import NamedTuple
from urllib.parse import urlsplit

import dparse2
# NOTE: we always want to use the external library rather than the built-in for now
Expand All @@ -37,6 +38,11 @@
from packvers import markers
from packvers.requirements import Requirement
from packvers.utils import canonicalize_name
from packvers.utils import InvalidSdistFilename
from packvers.utils import InvalidWheelFilename
from packvers.utils import parse_sdist_filename
from packvers.utils import parse_wheel_filename
from packvers.version import InvalidVersion

from packagedcode import models
from packagedcode.utils import build_description
Expand Down Expand Up @@ -1445,6 +1451,114 @@ def parse(cls, location, package_only=False):
yield models.PackageData.from_data(package_data, package_only)


def parse_distribution_filename(filename):
"""
Return a (name, version) tuple parsed from a wheel or sdist file name
such as ``pefile-2022.5.30-py3-none-any.whl`` or
``pefile-2022.5.30.tar.gz``. Return None if the file name cannot be
parsed as a distribution file name.
"""
if not filename:
return None
try:
if filename.endswith('.whl'):
name, version, _build, _tags = parse_wheel_filename(filename)
else:
name, version = parse_sdist_filename(filename)
except (InvalidWheelFilename, InvalidSdistFilename, InvalidVersion):
return None
return str(name), str(version)


class PipWheelCacheOriginJsonHandler(models.DatafileHandler):
datasource_id = 'pypi_pip_cache_origin_json'
path_patterns = ('*origin.json',)
default_package_type = 'pypi'
default_primary_language = 'Python'
description = 'pip wheel cache origin.json'
documentation_url = 'https://pip.pypa.io/en/stable/topics/caching/'

@classmethod
def parse(cls, location, package_only=False):
try:
with open(location) as f:
data = json.load(f)
except (OSError, ValueError):
# unreadable or invalid JSON: not a pip cache origin file
return

if not isinstance(data, dict):
return

url = data.get('url')
if not isinstance(url, str) or not url:
url = None

archive_info = data.get('archive_info') or {}
if not isinstance(archive_info, dict):
archive_info = {}

sha256 = None
hashes = archive_info.get('hashes') or {}
if isinstance(hashes, dict):
sha256 = hashes.get('sha256')
if not isinstance(sha256, str):
sha256 = None
if not sha256:
# the single hash is recorded as e.g. "sha256=<hex digest>"
hash_value = archive_info.get('hash') or ''
if isinstance(hash_value, str):
for prefix in ('sha256:', 'sha256='):
if hash_value.startswith(prefix):
sha256 = hash_value[len(prefix):]
break

name, version = cls.get_name_version(location, url)
if not name:
return

urls = get_pypi_urls(name, version)
if url:
# prefer the exact URL recorded in the cache origin file
urls['repository_download_url'] = url

package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
name=name,
version=version,
download_url=url,
sha256=sha256,
**urls,
)
yield models.PackageData.from_data(package_data, package_only)

@classmethod
def get_name_version(cls, location, url):
"""
Return a (name, version) tuple for the package described by a pip
cache ``origin.json`` file, parsed from the download ``url`` file name
or, if there is no usable URL, from the name of a sibling wheel file
in the same cache directory. Return (None, None) if no name and
version can be determined.
"""
candidates = []
if url:
candidates.append(posixpath.basename(urlsplit(url).path))
# a pip wheel cache directory holds the built wheel next to origin.json
location_path = Path(location)
if location_path.parent.is_dir():
candidates.extend(
sorted(p.name for p in location_path.parent.glob('*.whl'))
)
for filename in candidates:
name_version = parse_distribution_filename(filename)
if name_version:
return name_version
return None, None


class PythonSetupPyHandler(BaseExtractedPythonLayout):
datasource_id = 'pypi_setup_py'
path_patterns = ('*setup.py',)
Expand Down
7 changes: 7 additions & 0 deletions tests/packagedcode/data/plugin/plugins_list_linux.txt
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,13 @@ Package type: pypi
description: Python poetry pyproject.toml
path_patterns: '*pip-inspect.deplock'
--------------------------------------------
Package type: pypi
datasource_id: pypi_pip_cache_origin_json
documentation URL: https://pip.pypa.io/en/stable/topics/caching/
primary language: Python
description: pip wheel cache origin.json
path_patterns: '*origin.json'
--------------------------------------------
Package type: pypi
datasource_id: pypi_poetry_lock
documentation URL: https://python-poetry.org/docs/basic-usage/#installing-with-poetrylock
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
this is not JSON at all
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"archive_info":{},"url":"https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz"}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"archive_info":{"hash":"sha256=a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b","hashes":{"sha256":"a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b"}}}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
[
{
"type": "pypi",
"namespace": null,
"name": "pefile",
"version": "2022.5.30",
"qualifiers": {},
"subpath": null,
"primary_language": "Python",
"description": null,
"release_date": null,
"parties": [],
"keywords": [],
"homepage_url": null,
"download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz",
"size": null,
"sha1": null,
"md5": null,
"sha256": "a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b",
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"copyright": null,
"holder": null,
"declared_license_expression": null,
"declared_license_expression_spdx": null,
"license_detections": [],
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
"extracted_license_statement": null,
"notice_text": null,
"source_packages": [],
"file_references": [],
"is_private": false,
"is_virtual": false,
"extra_data": {},
"dependencies": [],
"repository_homepage_url": "https://pypi.org/project/pefile",
"repository_download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz",
"api_data_url": "https://pypi.org/pypi/pefile/2022.5.30/json",
"datasource_id": "pypi_pip_cache_origin_json",
"purl": "pkg:pypi/pefile@2022.5.30"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
{
"packages": [
{
"type": "pypi",
"namespace": null,
"name": "pefile",
"version": "2022.5.30",
"qualifiers": {},
"subpath": null,
"primary_language": "Python",
"description": null,
"release_date": null,
"parties": [],
"keywords": [],
"homepage_url": null,
"download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz",
"size": null,
"sha1": null,
"md5": null,
"sha256": "a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b",
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"copyright": null,
"holder": null,
"declared_license_expression": null,
"declared_license_expression_spdx": null,
"license_detections": [],
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
"extracted_license_statement": null,
"notice_text": null,
"source_packages": [],
"is_private": false,
"is_virtual": false,
"extra_data": {},
"repository_homepage_url": "https://pypi.org/project/pefile",
"repository_download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz",
"api_data_url": "https://pypi.org/pypi/pefile/2022.5.30/json",
"package_uid": "pkg:pypi/pefile@2022.5.30?uuid=fixed-uid-done-for-testing-5642512d1758",
"datafile_paths": [
"origin.json"
],
"datasource_ids": [
"pypi_pip_cache_origin_json"
],
"purl": "pkg:pypi/pefile@2022.5.30"
}
],
"dependencies": [],
"files": [
{
"path": "origin.json",
"type": "file",
"package_data": [
{
"type": "pypi",
"namespace": null,
"name": "pefile",
"version": "2022.5.30",
"qualifiers": {},
"subpath": null,
"primary_language": "Python",
"description": null,
"release_date": null,
"parties": [],
"keywords": [],
"homepage_url": null,
"download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz",
"size": null,
"sha1": null,
"md5": null,
"sha256": "a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b",
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"copyright": null,
"holder": null,
"declared_license_expression": null,
"declared_license_expression_spdx": null,
"license_detections": [],
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
"extracted_license_statement": null,
"notice_text": null,
"source_packages": [],
"file_references": [],
"is_private": false,
"is_virtual": false,
"extra_data": {},
"dependencies": [],
"repository_homepage_url": "https://pypi.org/project/pefile",
"repository_download_url": "https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz",
"api_data_url": "https://pypi.org/pypi/pefile/2022.5.30/json",
"datasource_id": "pypi_pip_cache_origin_json",
"purl": "pkg:pypi/pefile@2022.5.30"
}
],
"for_packages": [
"pkg:pypi/pefile@2022.5.30?uuid=fixed-uid-done-for-testing-5642512d1758"
],
"scan_errors": []
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"archive_info":{"hash":"sha256=a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b","hashes":{"sha256":"a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b"}},"url":"https://files.pythonhosted.org/packages/48/30/4559d06bad5bb627733dac1ef28c34f5e35f1461247ba63e5f6366901277/pefile-2022.5.30.tar.gz"}
63 changes: 63 additions & 0 deletions tests/packagedcode/test_pypi.py
Original file line number Diff line number Diff line change
Expand Up @@ -806,3 +806,66 @@ def test_parse_setup_py(test_loc):
def test_parse_more_setup_py(test_loc):
check_setup_py_parsing(test_loc)



class TestPipWheelCacheOriginJsonHandler(PackageTester):
test_data_dir = os.path.join(os.path.dirname(__file__), 'data')

def test_is_pip_wheel_cache_origin_json(self):
test_file = self.get_test_loc('pypi/pip-cache-origin-json/pefile/origin.json')
assert pypi.PipWheelCacheOriginJsonHandler.is_datafile(test_file)

def test_parse_pip_wheel_cache_origin_json(self):
test_file = self.get_test_loc('pypi/pip-cache-origin-json/pefile/origin.json')
package = pypi.PipWheelCacheOriginJsonHandler.parse(test_file)
expected_loc = self.get_test_loc(
'pypi/pip-cache-origin-json/pefile-origin.json-expected.json',
must_exist=False,
)
self.check_packages_data(
package, expected_loc, must_exist=False, regen=REGEN_TEST_FIXTURES,
)

def test_parse_origin_json_missing_hash_does_not_fail(self):
test_file = self.get_test_loc('pypi/pip-cache-origin-json/missing-hash/origin.json')
packages = list(pypi.PipWheelCacheOriginJsonHandler.parse(test_file))
assert len(packages) == 1
package = packages[0].to_dict()
assert package['name'] == 'requests'
assert package['version'] == '2.32.3'
assert package['download_url'] == (
'https://files.pythonhosted.org/packages/63/70/'
'2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz'
)
assert package['sha256'] is None

def test_parse_origin_json_missing_url_uses_sibling_wheel(self):
test_file = self.get_test_loc('pypi/pip-cache-origin-json/missing-url/origin.json')
packages = list(pypi.PipWheelCacheOriginJsonHandler.parse(test_file))
assert len(packages) == 1
package = packages[0].to_dict()
assert package['name'] == 'pefile'
assert package['version'] == '2022.5.30'
assert package['download_url'] is None
assert package['sha256'] == (
'a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b'
)

def test_parse_origin_json_invalid_content_does_not_fail(self):
test_file = self.get_test_loc('pypi/pip-cache-origin-json/invalid/origin.json')
packages = list(pypi.PipWheelCacheOriginJsonHandler.parse(test_file))
assert packages == []

def test_package_scan_pip_wheel_cache_origin_json_end_to_end(self):
test_dir = self.get_test_loc('pypi/pip-cache-origin-json/pefile/')
result_file = self.get_temp_file('json')
expected_file = self.get_test_loc(
'pypi/pip-cache-origin-json/pefile-package-expected.json',
must_exist=False,
)
run_scan_click(
['--package', '--strip-root', '--processes', '-1', test_dir, '--json-pp', result_file]
)
check_json_scan(
expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES,
)
Loading