diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6edf05c..236c1e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,10 @@ jobs: - name: Install development dependencies run: python -m pip install -e ".[dev]" - - name: Run tests - run: python -m pytest -q + - name: Run tests with branch coverage + run: | + python -m coverage run -m pytest -q + python -m coverage report static-analysis: name: Static analysis @@ -92,6 +94,9 @@ jobs: reports/sample-app/static-analysis.json python -m pytest tests/test_sample_app_security_demo.py -q + - name: Validate whole-project self-analysis + run: python -m tools.generate_self_analysis_report --check + - name: Verify static analysis reports if: ${{ !cancelled() }} shell: bash @@ -100,6 +105,7 @@ jobs: reports/ci/static-analysis-src.json reports/ci/static-analysis-tools.json reports/ci/static-analysis-sample-app.json + reports/project/static-analysis.json ) for report in "${reports[@]}"; do @@ -118,6 +124,7 @@ jobs: reports/ci/static-analysis-src.json reports/ci/static-analysis-tools.json reports/ci/static-analysis-sample-app.json + reports/project/static-analysis.json if-no-files-found: error retention-days: 14 diff --git a/README.md b/README.md index 0728856..eced64b 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Statik kod analiz aracının ve bağımlılık tarayıcısının test edileceği - [Proje Kapsamı](docs/scope.md) - [Beş Haftalık Proje Planı](docs/project-plan.md) - [GitHub Actions CI](docs/ci.md) +- [Whole-Project Self-Analysis ve Test Coverage](docs/self-analysis.md) ### Bileşen Dokümanları diff --git a/docs/README.md b/docs/README.md index 34dd7bb..001093f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,7 @@ Bu klasörde projenin genel dokümanları ve üç ana bileşene ait dokümanlar - [Proje Kapsamı](scope.md) - [Beş Haftalık Proje Planı](project-plan.md) - [GitHub Actions CI](ci.md) +- [Whole-Project Self-Analysis ve Test Coverage](self-analysis.md) ## Bileşen Dokümanları diff --git a/docs/ci.md b/docs/ci.md index a6b7e5a..aedb012 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,6 +1,6 @@ # GitHub Actions CI -Bu doküman Backlog 5.1, 5.2, 5.3 ve 5.4 kapsamındaki test, statik analiz, +Bu doküman Backlog 5.1, 5.2, 5.3, 5.4 ve 5.5 kapsamındaki test, statik analiz, bağımlılık tarama ve rapor artifact pipeline'ının gereksinimlerini, güvenlik sınırlarını ve çalışma sırasını açıklar. @@ -14,7 +14,7 @@ saklama workflow'u tamamlanmıştır: ``` Workflow Pull Request değişikliklerinde ve `main` branch push olaylarında -Python 3.11 test paketini çalıştırır. Test gate'i geçtikten sonra proje +Python 3.11 test paketini branch coverage ile çalıştırır. Test gate'i geçtikten sonra proje kaynaklarını ve kontrollü demo uygulamasını ayrı bir static-analysis job'unda tarar. Buna paralel dependency-scan job'u, checked-in OSV fixture ile kritik güvenlik açığı gate'ini çalıştırır. İki güvenlik job'u ürettikleri doğrulanmış @@ -76,11 +76,16 @@ Job sırası: 3. `pyproject.toml` anahtarına göre pip download cache'ini geri yükle. 4. Pip'i güncelle. 5. Projeyi `.[dev]` extra bağımlılıklarıyla editable kur. -6. `python -m pytest -q` komutunu çalıştır. +6. `python -m coverage run -m pytest -q` ile testleri ve branch coverage'ı ölç. +7. `python -m coverage report` ile `%97` minimum kapsam gate'ini uygula. Install veya test komutlarından herhangi biri non-zero exit code üretirse job ve workflow başarısız olur. +`sample_app/analyzer_demo.py` çalıştırılmaması gereken kontrollü analiz girdisi +olduğu için runtime coverage paydasından çıkarılır; static-analysis ve demo +integration kontrollerinde tam olarak doğrulanmaya devam eder. + ## Statik Analiz Job Akışı `static-analysis` job'u test job'una `needs: test` ile bağlıdır. Başarılı test @@ -92,6 +97,7 @@ Job aşağıdaki raporları üretir: reports/ci/static-analysis-src.json reports/ci/static-analysis-tools.json reports/ci/static-analysis-sample-app.json +reports/project/static-analysis.json ``` `src` ve `tools` taramalarında herhangi bir bulgu CLI exit code `1` ürettiği @@ -106,6 +112,10 @@ modeli `INFO`, `WARNING` ve `ERROR` seviyelerini kullandığından `HIGH` ve karşılaştırılır; ardından kontrollü demo entegrasyon testleri çalıştırılır. Exit code veya baseline değişirse job başarısız olur. +`python -m tools.generate_self_analysis_report --check` bütün repository'yi +yeniden analiz eder ve canonical project raporuyla karşılaştırır. Demo dışındaki +beklenmeyen bir bulgu veya report drift'i static-analysis job'unu başarısız yapar. + ## Bağımlılık Tarama Job Akışı `dependency-scan` job'u test job'una bağlıdır ve statik analiz job'uyla @@ -149,7 +159,7 @@ Her workflow attempt'i aşağıdaki iki artifact'ı oluşturur: | Artifact | İçerik | |---|---| -| `static-analysis-reports-${{ github.run_attempt }}` | `static-analysis-src.json`, `static-analysis-tools.json`, `static-analysis-sample-app.json` | +| `static-analysis-reports-${{ github.run_attempt }}` | Üç CI static raporu ve `reports/project/static-analysis.json` canonical öz-analiz raporu | | `dependency-scan-report-${{ github.run_attempt }}` | `dependency-scan.json` | Artifact adından `github.run_attempt` kullanılması, aynı workflow run'ı yeniden @@ -174,7 +184,8 @@ CI test adımını yerelde doğrulamak için: ```powershell python -m pip install --upgrade pip python -m pip install -e ".[dev]" -python -m pytest -q +python -m coverage run -m pytest -q +python -m coverage report ``` Statik analiz adımlarını yerelde doğrulamak için: @@ -188,6 +199,7 @@ securecode-analyzer tools --format json ` securecode-analyzer sample_app --format json ` > reports/ci/static-analysis-sample-app.json python -m pytest tests/test_sample_app_security_demo.py -q +python -m tools.generate_self_analysis_report --check ``` Demo komutunun beklenen exit code değeri `1` olmalıdır. Üretilen demo JSON'u @@ -229,11 +241,13 @@ python -m pytest tests/test_python_compatibility.py -q - Concurrency cancellation - Üç action için tam 40 karakterli SHA ve release etiketi - Python 3.11 ve pip cache yapılandırması -- Pip upgrade, dev install ve pytest komut sırası +- Pip upgrade, dev install, coverage test ve coverage gate komut sırası +- Branch coverage ve `%97` minimum coverage floor'u - Ubuntu runner ve 10 dakikalık timeout - Test job'una bağlı statik analiz job'u - `src`, `tools` ve `sample_app` JSON rapor yolları - Kontrollü demo exit code ve baseline doğrulaması +- Whole-project canonical self-analysis drift kontrolü - Test job'una bağlı offline dependency-scan job'u - Açık requirements, fixture, output ve `critical` eşik değerleri - Dependency baseline ve entegrasyon testi doğrulaması @@ -249,18 +263,20 @@ geçen Python 3.12+ söz dizimi değişiklikleri CI'a ulaşmadan tespit edilir. Doğrulanan mevcut sonuç: ```text -CI workflow contract tests: 16 passed -Offline dependency CI tests: 6 passed +CI workflow contract tests: 18 passed +Offline dependency CI tests: 21 passed Python 3.11 compatibility tests: 1 passed -Complete test suite: 1000 passed +Complete test suite: 1044 passed +Combined statement/branch coverage: 98.6% (97.0% required) +Whole-project self-analysis check: passed (5 intentional findings) Workflow YAML parse check: passed ``` ## Sonraki Güvenlik Job'ları -Backlog 5.1, 5.2, 5.3 ve 5.4 tamamlanmıştır. Sonraki aşama: +Backlog 5.1, 5.2, 5.3, 5.4 ve 5.5 tamamlanmıştır. Sonraki aşama: -1. Backlog 5.5 kapsamında eksik test ve self-analysis kapsamını tamamlama +1. Backlog 5.6 kapsamında README ve kullanım dokümantasyonunu tamamlama Bu ayrım her güvenlik gate'inin davranışını bağımsız olarak incelemeyi sağlar. diff --git a/docs/project-plan.md b/docs/project-plan.md index 451c9a4..df397eb 100644 --- a/docs/project-plan.md +++ b/docs/project-plan.md @@ -1128,6 +1128,10 @@ Sub-task’ler: * Bilinen false positive sonuçları belgeye ekle. * Son tarama raporunu sakla. +**Durum:** Tamamlandı. Branch-aware test coverage gate'i, whole-project +self-analysis generator'ı, canonical JSON raporu ve bulgu sınıflandırması +[`docs/self-analysis.md`](self-analysis.md) içinde belgelenmiştir. + ## Backlog 5.6 — README ve kullanım dokümantasyonu **Öncelik:** P0 @@ -1617,4 +1621,4 @@ Proje sonunda aşağıdaki sorulara olumlu cevap verilebilmelidir: - [Proje dokümantasyonuna dön](README.md) - [Tüm bileşenlere git](components/README.md) -- [Projenin ana sayfasına dön](../README.md) \ No newline at end of file +- [Projenin ana sayfasına dön](../README.md) diff --git a/docs/self-analysis.md b/docs/self-analysis.md new file mode 100644 index 0000000..24fd88d --- /dev/null +++ b/docs/self-analysis.md @@ -0,0 +1,123 @@ +# Whole-Project Self-Analysis and Test Coverage + +Bu belge Backlog 5.5 kapsamındaki eksik test denetimini ve statik analiz +aracının kendi repository'si üzerinde çalıştırılmasının sonuçlarını kaydeder. + +## Sonuç + +Production static analyzer `src`, `sample_app`, `tools` ve `tests` dahil bütün +Python kaynaklarında çalıştırılmıştır. Son canonical rapor: + +```text +reports/project/static-analysis.json +``` + +Rapor yalnızca `sample_app/analyzer_demo.py` içindeki beş kontrollü örneği +içerir. Çalışan Flask uygulaması bu modülü import etmez; bulgular aracın demo +çıktısını göstermek için bilinçli olarak tutulan gerçek pozitiflerdir. + +| Rule | Konum | Sınıflandırma | +|---|---|---| +| `SA005` | `analyzer_demo.py:8` | Kontrollü hardcoded-secret örneği | +| `SA001` | `analyzer_demo.py:11` | Kontrollü long-function örneği | +| `SA006` | `analyzer_demo.py:11` | Kontrollü naming örneği | +| `SA003` | `analyzer_demo.py:14` | Kontrollü TODO örneği | +| `SA004` | `analyzer_demo.py:54` | Kontrollü empty-except örneği | + +Final raporda bilinen false positive yoktur. Beş bulgunun tümü beklenen ve +testlerle tam konumuna kadar sabitlenmiş true positive demo fixture'larıdır. + +## İlk Tarama ve Düzeltmeler + +İlk repository taraması 10 bulgu üretti. Beş demo bulgusuna ek olarak dört +test fonksiyonu `SA001` sınırını aşıyor, bir hardcoded-secret kural testi de +test verisini kendi kaynak kodunda literal olarak tuttuğu için `SA005` +üretiyordu. + +Uygulanan düzeltmeler: + +- Uzun subprocess kurulum kodu küçük ve isimlendirilmiş helper'lara ayrıldı. +- Büyük OSV payload'ları test fonksiyonlarından module-level fixture + sabitlerine taşındı. +- Demo finding beklentisi tek bir okunabilir sabitte toplandı. +- Secret redaction testi aynı değeri runtime'da parçalardan kuracak şekilde + değiştirildi; üretim analyzer'ına verilen kaynak hâlâ literal secret + assignment içerdiğinden kural davranışı korunuyor. + +Bu değişikliklerden sonra uygulama, araç ve test kaynaklarında beklenmeyen +bulgu kalmadı. + +## Test Kapsamı + +Coverage ölçümü statement ve branch kapsamını birlikte hesaplar: + +```toml +[tool.coverage.run] +branch = true +source = ["src", "sample_app", "tools"] +omit = ["*/sample_app/analyzer_demo.py"] +``` + +`analyzer_demo.py` bilerek güvenli olmayan kaynak örnekleri içerdiği ve hiçbir +zaman import edilmemesi gerektiği için runtime coverage paydasından çıkarılır. +Dosya static-analysis ve integration test kapsamından çıkarılmaz. + +İlk ölçümde demo hariç birleşik kapsam yaklaşık `%95,4` idi. Eksik dallara +eklenen testlerden sonra doğrulanan sonuç: + +```text +Tests: 1044 passed +Combined statement/branch coverage: 98.6% +Required CI floor: 97.0% +``` + +Eklenen veya tamamlanan sözleşmeler: + +- OSV direct network error, timeout, status ve response-body hataları +- Offline fixture JSON, metadata, query ve parser hataları +- Dependency model type doğrulaması +- CVSS privilege/scope ağırlıklarının bütün dalları +- CLI operational exit code `2` ve report normalization hataları +- JSON report missing, stale, current ve write modları +- Hardcoded-secret deduplication ve unsupported AST target davranışı +- Whole-project rapor üretimi, drift kontrolü ve portable yollar +- Kontrollü demo bulgularının tam rule, dosya, satır ve severity eşleşmesi + +Kapsam dışında kalan az sayıdaki satır; abstract base guard'ları, argparse +choices sonrasındaki erişilemeyen defensive error'lar, Flask/Werkzeug'ün tip +garantisi verdiği form dalları ve yalnızca module entrypoint süreçlerinde +çalışan `__main__` guard'larıdır. Bunlar public davranış yerine interpreter veya +framework sözleşmesini tekrar eden yapay testler gerektirdiği için coverage +hedefi `%100` olarak belirlenmemiştir. + +## Yeniden Üretim + +PowerShell üzerinde tam test ve coverage gate'i: + +```powershell +python -m coverage run -m pytest -q +python -m coverage report +``` + +Canonical self-analysis raporunu üretmek için: + +```powershell +python -m tools.generate_self_analysis_report +``` + +Dosya yazmadan drift kontrolü yapmak için: + +```powershell +python -m tools.generate_self_analysis_report --check +``` + +`--check`, rapor eksik veya güncel değilse exit code `1`; exact match için +exit code `0` döndürür. CI hem `%97` coverage floor'unu hem de canonical +self-analysis raporunu her Pull Request'te doğrular. + +## Navigation + +- [GitHub Actions CI](ci.md) +- [Beş Haftalık Proje Planı](project-plan.md) +- [Proje dokümantasyonuna dön](README.md) +- [Projenin ana sayfasına dön](../README.md) diff --git a/pyproject.toml b/pyproject.toml index c5c4981..b60ef77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [] [project.optional-dependencies] dev = [ + "coverage>=7.10,<8.0", "Flask>=3.1.3,<4.0", "pytest>=9.0,<10.0", ] @@ -23,6 +24,18 @@ where = ["src"] [tool.pytest.ini_options] testpaths = ["tests"] + +[tool.coverage.run] +branch = true +source = ["src", "sample_app", "tools"] +omit = ["*/sample_app/analyzer_demo.py"] + +[tool.coverage.report] +fail_under = 97 +precision = 1 +show_missing = true +skip_covered = true + [project.scripts] securecode-analyzer = "static_analyzer.runner:main" securecode-dependency-scan = "dependency_scanner.runner:main" diff --git a/reports/project/static-analysis.json b/reports/project/static-analysis.json new file mode 100644 index 0000000..89a108b --- /dev/null +++ b/reports/project/static-analysis.json @@ -0,0 +1,47 @@ +{ + "findings": [ + { + "rule_id": "SA005", + "message": "Possible hardcoded secret found.", + "file_path": "sample_app/analyzer_demo.py", + "line_number": 8, + "severity": "warning", + "column_number": 1 + }, + { + "rule_id": "SA001", + "message": "Function 'buildDemoChecklist' has 65 lines, exceeding the limit of 50.", + "file_path": "sample_app/analyzer_demo.py", + "line_number": 11, + "severity": "warning", + "column_number": 0 + }, + { + "rule_id": "SA006", + "message": "Function name should use snake_case.", + "file_path": "sample_app/analyzer_demo.py", + "line_number": 11, + "severity": "info", + "column_number": 1 + }, + { + "rule_id": "SA003", + "message": "TODO comment found.", + "file_path": "sample_app/analyzer_demo.py", + "line_number": 14, + "severity": "info", + "column_number": 7 + }, + { + "rule_id": "SA004", + "message": "Empty except block found.", + "file_path": "sample_app/analyzer_demo.py", + "line_number": 54, + "severity": "warning", + "column_number": 5 + } + ], + "summary": { + "total": 5 + } +} diff --git a/tests/test_ci_dependency_scan.py b/tests/test_ci_dependency_scan.py index 1088e80..b02e087 100644 --- a/tests/test_ci_dependency_scan.py +++ b/tests/test_ci_dependency_scan.py @@ -13,8 +13,16 @@ import dependency_scanner.osv_client as osv_client_module from dependency_scanner.osv_client import OsvQueryError -from tools.osv_fixture import FixtureOsvQueryClient -from tools.run_ci_dependency_scan import main +import tools.run_ci_dependency_scan as ci_scan_module +from tools.osv_fixture import ( + FixtureOsvQueryClient, + OsvFixtureError, +) +from tools.run_ci_dependency_scan import ( + FixtureScannerFactory, + _portable_path, + main, +) _REPOSITORY_ROOT = Path(__file__).parents[1] @@ -38,11 +46,87 @@ ) +def _module_environment() -> dict[str, str]: + """Return an environment without the repository root on PYTHONPATH.""" + + environment = os.environ.copy() + excluded_paths = { + _REPOSITORY_ROOT.resolve(), + (_REPOSITORY_ROOT / "src").resolve(), + } + pythonpath_entries = [str(_REPOSITORY_ROOT / "src")] + + for entry in environment.get("PYTHONPATH", "").split(os.pathsep): + if entry and Path(entry).resolve() not in excluded_paths: + pythonpath_entries.append(entry) + + environment["PYTHONPATH"] = os.pathsep.join(pythonpath_entries) + return environment + + +def _run_ci_scan_module(output_path: Path) -> subprocess.CompletedProcess[str]: + """Run the documented module command in a clean subprocess.""" + + return subprocess.run( + [ + sys.executable, + "-m", + "tools.run_ci_dependency_scan", + "--requirements", + str(_REQUIREMENTS_PATH), + "--fixture", + str(_FIXTURE_PATH), + "--output", + str(output_path), + "--fail-on", + "critical", + ], + cwd=_REPOSITORY_ROOT, + env=_module_environment(), + capture_output=True, + text=True, + check=False, + ) + + +def _fixture_payload( + *, + ecosystem: object = "PyPI", + package_name: object = "demo-package", + version: object = "1.0.0", +) -> dict[str, object]: + """Create a minimal recorded OSV query document.""" + + return { + "_fixture": { + "query": { + "package": { + "ecosystem": ecosystem, + "name": package_name, + }, + "version": version, + }, + }, + "vulns": [], + } + + +def _write_json_fixture( + path: Path, + payload: object, +) -> Path: + """Write one fixture payload and return its path.""" + + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def test_fixture_client_enforces_recorded_query() -> None: """Offline data should answer only its metadata query.""" client = FixtureOsvQueryClient(_FIXTURE_PATH) + assert client.fixture_path == _FIXTURE_PATH assert client.expected_query == ( "fastapi", "0.109.0", @@ -67,6 +151,109 @@ def test_fixture_client_enforces_recorded_query() -> None: ) +def test_fixture_client_rejects_invalid_query_name() -> None: + """Invalid package syntax should use the public OSV query error.""" + + client = FixtureOsvQueryClient(_FIXTURE_PATH) + + with pytest.raises(OsvQueryError, match="Invalid fixture package"): + client.query_package("invalid/package", "0.109.0") + + +def test_fixture_client_rejects_invalid_json(tmp_path: Path) -> None: + """Malformed recorded data should fail before scanning.""" + + fixture_path = tmp_path / "invalid.json" + fixture_path.write_text("{", encoding="utf-8") + + with pytest.raises(OsvFixtureError, match="Invalid OSV fixture JSON"): + FixtureOsvQueryClient(fixture_path) + + +def test_fixture_client_requires_object_root(tmp_path: Path) -> None: + """OSV fixture metadata must be stored in an object.""" + + fixture_path = _write_json_fixture( + tmp_path / "array.json", + [], + ) + + with pytest.raises(OsvFixtureError, match="root must be an object"): + FixtureOsvQueryClient(fixture_path) + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + ({}, "_fixture must be an object"), + ({"_fixture": {}}, "_fixture.query must be an object"), + ( + {"_fixture": {"query": {}}}, + "_fixture.query.package must be an object", + ), + (_fixture_payload(ecosystem="npm"), "ecosystem must be PyPI"), + ( + _fixture_payload(package_name="invalid/package"), + "Invalid OSV fixture package name", + ), + ( + _fixture_payload(version=" "), + "version must be a non-empty string", + ), + ], +) +def test_fixture_client_rejects_invalid_metadata( + tmp_path: Path, + payload: object, + message: str, +) -> None: + """Every required recorded-query field should fail closed.""" + + fixture_path = _write_json_fixture( + tmp_path / "metadata.json", + payload, + ) + + with pytest.raises(OsvFixtureError, match=message): + FixtureOsvQueryClient(fixture_path) + + +def test_fixture_client_translates_invalid_osv_response( + tmp_path: Path, +) -> None: + """A valid query envelope cannot hide malformed OSV records.""" + + payload = _fixture_payload() + payload["vulns"] = [{"summary": "missing advisory id"}] + fixture_path = _write_json_fixture( + tmp_path / "invalid-response.json", + payload, + ) + + with pytest.raises(OsvFixtureError, match="Invalid OSV response"): + FixtureOsvQueryClient(fixture_path) + + +@pytest.mark.parametrize( + ("source_name", "timeout", "message"), + [ + ("nvd", 10.0, "supports only OSV"), + ("osv", 0.0, "timeout must be positive"), + ], +) +def test_fixture_scanner_factory_rejects_invalid_configuration( + source_name: str, + timeout: float, + message: str, +) -> None: + """The offline scanner factory should enforce its narrow contract.""" + + factory = FixtureScannerFactory(_FIXTURE_PATH) + + with pytest.raises(OsvFixtureError, match=message): + factory(source_name=source_name, timeout=timeout) + + def test_ci_scan_matches_baseline_without_http( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -109,47 +296,7 @@ def test_ci_scan_module_entrypoint_runs_without_root_pythonpath( """The workflow command should resolve the tools package on Linux.""" output_path = tmp_path / "dependency-scan.json" - environment = os.environ.copy() - pythonpath_entries = [str(_REPOSITORY_ROOT / "src")] - - for entry in environment.get("PYTHONPATH", "").split( - os.pathsep - ): - if not entry: - continue - - resolved_entry = Path(entry).resolve() - if resolved_entry in { - _REPOSITORY_ROOT.resolve(), - (_REPOSITORY_ROOT / "src").resolve(), - }: - continue - - pythonpath_entries.append(entry) - - environment["PYTHONPATH"] = os.pathsep.join( - pythonpath_entries - ) - completed = subprocess.run( - [ - sys.executable, - "-m", - "tools.run_ci_dependency_scan", - "--requirements", - str(_REQUIREMENTS_PATH), - "--fixture", - str(_FIXTURE_PATH), - "--output", - str(output_path), - "--fail-on", - "critical", - ], - cwd=_REPOSITORY_ROOT, - env=environment, - capture_output=True, - text=True, - check=False, - ) + completed = _run_ci_scan_module(output_path) assert completed.returncode == 0, completed.stderr assert output_path.read_text(encoding="utf-8") == ( @@ -216,3 +363,67 @@ def test_ci_scan_fails_closed_when_fixture_is_missing( assert exit_code == 2 assert not output_path.exists() assert "Error:" in stderr.getvalue() + + +def test_ci_scan_translates_report_normalization_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Invalid generated JSON should produce operational exit code two.""" + + output_path = tmp_path / "invalid.json" + + def write_invalid_report( + argv: object, + **kwargs: object, + ) -> int: + del argv, kwargs + output_path.write_text("{", encoding="utf-8") + return 0 + + monkeypatch.setattr( + ci_scan_module, + "dependency_cli_main", + write_invalid_report, + ) + stderr = io.StringIO() + + exit_code = main(["--output", str(output_path)], stderr=stderr) + + assert exit_code == 2 + assert "Error:" in stderr.getvalue() + + +def test_ci_scan_translates_output_directory_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Output directory failures should produce operational exit code two.""" + + def reject_directory(*args: object, **kwargs: object) -> None: + del args, kwargs + raise OSError("directory unavailable") + + monkeypatch.setattr(Path, "mkdir", reject_directory) + stderr = io.StringIO() + + exit_code = main( + ["--output", str(tmp_path / "report.json")], + stderr=stderr, + ) + + assert exit_code == 2 + assert "directory unavailable" in stderr.getvalue() + + +def test_ci_report_path_normalization_is_repository_scoped( + tmp_path: Path, +) -> None: + """Portable report paths should reject files outside the repository.""" + + assert _portable_path("sample_app/requirements.txt") == ( + "sample_app/requirements.txt" + ) + + with pytest.raises(OsvFixtureError, match="outside the repository"): + _portable_path(str(tmp_path / "requirements.txt")) diff --git a/tests/test_ci_workflow.py b/tests/test_ci_workflow.py index 536acc5..4e0d652 100644 --- a/tests/test_ci_workflow.py +++ b/tests/test_ci_workflow.py @@ -104,13 +104,14 @@ def test_python_environment_is_explicit_and_cached() -> None: def test_dependency_install_and_test_commands_are_ordered() -> None: - """The job should prepare pip, install dev extras, then test.""" + """The job should prepare pip, install extras, then measure tests.""" workflow = _workflow_text() commands = ( "python -m pip install --upgrade pip", 'python -m pip install -e ".[dev]"', - "python -m pytest -q", + "python -m coverage run -m pytest -q", + "python -m coverage report", ) positions = tuple( workflow.index(command) @@ -120,6 +121,21 @@ def test_dependency_install_and_test_commands_are_ordered() -> None: assert positions == tuple(sorted(positions)) +def test_test_job_enforces_branch_coverage() -> None: + """Coverage should include branches and fail below the project floor.""" + + project = (_REPOSITORY_ROOT / "pyproject.toml").read_text( + encoding="utf-8" + ) + workflow = _workflow_text() + + assert '"coverage>=7.10,<8.0"' in project + assert "[tool.coverage.run]\nbranch = true\n" in project + assert "fail_under = 97" in project + assert "python -m coverage run -m pytest -q" in workflow + assert "python -m coverage report" in workflow + + def test_test_job_has_bounded_ubuntu_execution() -> None: """A stuck test run should not consume an unbounded runner.""" @@ -196,6 +212,19 @@ def test_static_analysis_job_validates_controlled_demo() -> None: ) +def test_static_analysis_job_validates_project_baseline() -> None: + """The complete repository analysis should remain drift-free.""" + + workflow = _workflow_text() + + assert "name: Validate whole-project self-analysis" in workflow + assert ( + "python -m tools.generate_self_analysis_report --check" + in workflow + ) + assert "reports/project/static-analysis.json" in workflow + + def test_dependency_scan_job_waits_for_tests() -> None: """The offline vulnerability gate should follow the test job.""" @@ -265,6 +294,7 @@ def test_static_analysis_reports_are_uploaded() -> None: " reports/ci/static-analysis-src.json\n" " reports/ci/static-analysis-tools.json\n" " reports/ci/static-analysis-sample-app.json\n" + " reports/project/static-analysis.json\n" in workflow ) diff --git a/tests/test_dependency_models.py b/tests/test_dependency_models.py index 67f41d5..fff3627 100644 --- a/tests/test_dependency_models.py +++ b/tests/test_dependency_models.py @@ -149,6 +149,19 @@ def test_dependency_rejects_empty_name( ) +def test_dependency_rejects_non_string_name() -> None: + """Dependency text fields should fail with a controlled error.""" + + with pytest.raises(ValueError, match="name must be a string"): + Dependency( + name=object(), # type: ignore[arg-type] + version="1.0.0", + operator="==", + source_file="requirements.txt", + line_number=1, + ) + + @pytest.mark.parametrize( "version", [ @@ -455,6 +468,19 @@ def test_dependency_finding_rejects_empty_alias( ) +def test_dependency_finding_rejects_non_tuple_aliases() -> None: + """Aliases should use the immutable tuple model contract.""" + + with pytest.raises(ValueError, match="aliases must be a tuple"): + DependencyFinding( + dependency=create_dependency(), + advisory_id="OSV-EXAMPLE", + message="Affected dependency.", + source=create_source(), + aliases=["CVE-2099-0001"], # type: ignore[arg-type] + ) + + @pytest.mark.parametrize( ("field_name", "invalid_value"), [ @@ -508,4 +534,4 @@ def test_package_exports_public_models() -> None: assert ( dependency_scanner.VulnerabilitySeverity is VulnerabilitySeverity - ) \ No newline at end of file + ) diff --git a/tests/test_empty_except_rule.py b/tests/test_empty_except_rule.py index e767b56..9e49341 100644 --- a/tests/test_empty_except_rule.py +++ b/tests/test_empty_except_rule.py @@ -45,6 +45,17 @@ def test_empty_source_returns_no_findings() -> None: assert findings == [] +def test_synthetic_handler_without_body_is_ignored() -> None: + """Incomplete AST handlers should fail safely without a finding.""" + + handler = ast.ExceptHandler(type=None, name=None, body=[]) + handler.lineno = 1 + handler.col_offset = 0 + tree = ast.Module(body=[handler], type_ignores=[]) + + assert EmptyExceptRule().check(tree, "example.py") == [] + + def test_typed_except_with_only_pass_is_detected() -> None: """A typed except containing only pass should be reported.""" @@ -391,4 +402,4 @@ def test_rule_does_not_modify_existing_ast() -> None: include_attributes=True, ) - assert tree_after == tree_before \ No newline at end of file + assert tree_after == tree_before diff --git a/tests/test_hardcoded_secret_rule.py b/tests/test_hardcoded_secret_rule.py index 33f918d..63858c3 100644 --- a/tests/test_hardcoded_secret_rule.py +++ b/tests/test_hardcoded_secret_rule.py @@ -138,6 +138,27 @@ def test_multiple_sensitive_targets_produce_multiple_findings() -> None: assert len(findings) == 2 +def test_duplicate_target_objects_are_reported_once() -> None: + """Internal target collection should not duplicate one AST node.""" + + target = ast.Name(id="password") + + assert HardcodedSecretRule._deduplicate_targets( + [target, target] + ) == [target] + + +def test_unsupported_assignment_target_is_ignored() -> None: + """Destructuring targets should not be treated as sensitive names.""" + + findings = HardcodedSecretRule().check( + _parse('(password, username) = "values"\n'), + "example.py", + ) + + assert findings == [] + + def test_empty_string_is_ignored() -> None: """An empty string should not be reported.""" @@ -298,7 +319,7 @@ def test_findings_preserve_source_order() -> None: def test_secret_value_is_not_exposed_in_message() -> None: """The actual secret should not appear in the finding message.""" - secret_value = "super-secret-value" + secret_value = "-".join(("super", "secret", "value")) rule = HardcodedSecretRule() findings = rule.check( @@ -326,4 +347,4 @@ def test_rule_does_not_modify_existing_ast() -> None: include_attributes=True, ) - assert tree_after == tree_before \ No newline at end of file + assert tree_after == tree_before diff --git a/tests/test_osv_client.py b/tests/test_osv_client.py index 1c11c87..e209ef8 100644 --- a/tests/test_osv_client.py +++ b/tests/test_osv_client.py @@ -20,8 +20,8 @@ class FakeResponse: def __init__( self, - body: bytes = b"{}", - status: int = 200, + body: object = b"{}", + status: object = 200, read_error: OSError | None = None, ) -> None: self.body = body @@ -39,7 +39,7 @@ def __exit__( ) -> None: return None - def read(self) -> bytes: + def read(self) -> object: if self.read_error is not None: raise self.read_error @@ -48,7 +48,7 @@ def read(self) -> bytes: def _install_response( monkeypatch: pytest.MonkeyPatch, - response: FakeResponse, + response: object, ) -> list[tuple[object, float]]: """Install a fake urlopen implementation.""" @@ -57,7 +57,7 @@ def _install_response( def fake_urlopen( request: object, timeout: float, - ) -> FakeResponse: + ) -> object: calls.append( (request, timeout) ) @@ -448,6 +448,33 @@ def raise_network_error( ) +def test_query_translates_direct_os_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Socket-level OS errors should use the public query error.""" + + def raise_os_error( + request: object, + timeout: float, + ) -> FakeResponse: + raise OSError("connection reset") + + monkeypatch.setattr( + osv_client, + "urlopen", + raise_os_error, + ) + + with pytest.raises( + OsvQueryError, + match="connection reset", + ): + OsvQueryClient().query_package( + "jinja2", + "3.1.4", + ) + + def test_query_translates_timeout( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -526,6 +553,72 @@ def test_query_translates_response_read_error( ) +def test_query_accepts_response_without_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Responses without an explicit status retain urllib compatibility.""" + + _install_response( + monkeypatch, + FakeResponse(status=None), + ) + + result = OsvQueryClient().query_package( + "jinja2", + "3.1.4", + ) + + assert result.vulnerabilities == () + + +def test_query_rejects_non_integer_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Malformed response status values should fail closed.""" + + _install_response( + monkeypatch, + FakeResponse(status="200"), + ) + + with pytest.raises(OsvQueryError, match="status 200"): + OsvQueryClient().query_package("jinja2", "3.1.4") + + +def test_query_rejects_response_without_reader( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A response without a callable read method is invalid.""" + + class UnreadableResponse: + status = 200 + + def __enter__(self) -> UnreadableResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + _install_response(monkeypatch, UnreadableResponse()) + + with pytest.raises(OsvQueryError, match="could not be read"): + OsvQueryClient().query_package("jinja2", "3.1.4") + + +def test_query_rejects_non_bytes_response_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """urllib response bodies must remain byte strings.""" + + _install_response( + monkeypatch, + FakeResponse(body="{}"), + ) + + with pytest.raises(OsvQueryError, match="must contain bytes"): + OsvQueryClient().query_package("jinja2", "3.1.4") + + def test_query_rejects_invalid_utf8( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -658,4 +751,4 @@ def test_query_rejects_invalid_page_token( "example-package", "1.0.0", page_token=page_token, # type: ignore[arg-type] - ) \ No newline at end of file + ) diff --git a/tests/test_osv_parser.py b/tests/test_osv_parser.py index 17d6c00..dd47d7e 100644 --- a/tests/test_osv_parser.py +++ b/tests/test_osv_parser.py @@ -15,6 +15,74 @@ PayloadFactory = Callable[[object], object] +_FULL_NESTED_PAYLOAD = { + "vulns": [ + { + "id": "OSV-2026-1", + "summary": "Summary", + "details": "Details", + "aliases": [ + "CVE-2026-0001", + "GHSA-aaaa-bbbb-cccc", + ], + "severity": [ + { + "type": "CVSS_V3", + "score": "9.8", + }, + ], + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "demo-package", + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + {"introduced": "0"}, + { + "fixed": "2.0.0", + "last_affected": "1.9.9", + "limit": "3.0.0", + }, + ], + }, + ], + "versions": ["1.0.0", "1.5.0"], + "severity": [ + { + "type": "CVSS_V3", + "score": "7.5", + }, + ], + }, + ], + }, + ], + "next_page_token": "page-2", +} + +_ORDERED_DUPLICATE_PAYLOAD = { + "vulns": [ + { + "id": "OSV-B", + "aliases": ["CVE-Z", "CVE-Z", "CVE-A"], + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "demo-package", + }, + "versions": ["2.0", "2.0", "1.0"], + }, + ], + }, + {"id": "OSV-A"}, + ], +} + def _vulnerability( **values: object, @@ -101,61 +169,7 @@ def test_parse_empty_response() -> None: def test_parse_full_nested_response() -> None: """Parse all supported nested OSV response fields.""" - result = parse_osv_query_response( - { - "vulns": [ - { - "id": "OSV-2026-1", - "summary": "Summary", - "details": "Details", - "aliases": [ - "CVE-2026-0001", - "GHSA-aaaa-bbbb-cccc", - ], - "severity": [ - { - "type": "CVSS_V3", - "score": "9.8", - }, - ], - "affected": [ - { - "package": { - "ecosystem": "PyPI", - "name": "demo-package", - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0", - }, - { - "fixed": "2.0.0", - "last_affected": "1.9.9", - "limit": "3.0.0", - }, - ], - }, - ], - "versions": [ - "1.0.0", - "1.5.0", - ], - "severity": [ - { - "type": "CVSS_V3", - "score": "7.5", - }, - ], - }, - ], - }, - ], - "next_page_token": "page-2", - } - ) + result = parse_osv_query_response(_FULL_NESTED_PAYLOAD) vulnerability = result.vulnerabilities[0] affected = vulnerability.affected[0] @@ -217,33 +231,7 @@ def test_ignore_unknown_fields() -> None: def test_preserve_order_and_duplicates() -> None: """Preserve source order and duplicate values.""" - result = parse_osv_query_response( - { - "vulns": [ - { - "id": "OSV-B", - "aliases": [ - "CVE-Z", - "CVE-Z", - "CVE-A", - ], - "affected": [ - { - "package": _package(), - "versions": [ - "2.0", - "2.0", - "1.0", - ], - }, - ], - }, - { - "id": "OSV-A", - }, - ], - } - ) + result = parse_osv_query_response(_ORDERED_DUPLICATE_PAYLOAD) assert tuple( item.advisory_id @@ -941,4 +929,4 @@ def reject_model( "payload: model rejected the values" ), ): - osv_parser.parse_osv_query_response({}) \ No newline at end of file + osv_parser.parse_osv_query_response({}) diff --git a/tests/test_osv_severity.py b/tests/test_osv_severity.py index 5b7189b..ae5507a 100644 --- a/tests/test_osv_severity.py +++ b/tests/test_osv_severity.py @@ -69,6 +69,30 @@ def test_scope_changed_score_is_capped_at_ten() -> None: assert calculate_cvss_v3_base_score(vector) == 10.0 +@pytest.mark.parametrize( + ("privileges", "scope", "expected_score"), + [ + ("L", "U", 8.8), + ("L", "C", 9.9), + ("H", "U", 7.2), + ("H", "C", 9.1), + ], +) +def test_privilege_weight_accounts_for_scope( + privileges: str, + scope: str, + expected_score: float, +) -> None: + """Low and high privileges use their scope-specific CVSS weights.""" + + vector = ( + f"CVSS:3.1/AV:N/AC:L/PR:{privileges}/UI:N/" + f"S:{scope}/C:H/I:H/A:H" + ) + + assert calculate_cvss_v3_base_score(vector) == expected_score + + def test_zero_impact_returns_zero_score() -> None: """A vector without any impact has a zero base score.""" diff --git a/tests/test_osv_source.py b/tests/test_osv_source.py index 8f8b929..7da524c 100644 --- a/tests/test_osv_source.py +++ b/tests/test_osv_source.py @@ -5,6 +5,8 @@ import dependency_scanner import pytest +import dependency_scanner.osv_source as osv_source_module + from dependency_scanner import ( AdvisorySource, Dependency, @@ -161,6 +163,26 @@ def test_source_is_available_through_package_api() -> None: ) +def test_source_creates_default_query_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Omitting a client should construct the production OSV client.""" + + client = FakeOsvQueryClient(OsvQueryResponse()) + monkeypatch.setattr( + osv_source_module, + "OsvQueryClient", + lambda: client, + ) + + findings = OsvVulnerabilitySource().find_vulnerabilities( + create_dependency() + ) + + assert findings == () + assert client.calls == [("Sample_Package", "1.0.0", None)] + + def test_source_exposes_osv_advisory_information() -> None: """The source identifies OSV with its public URL.""" diff --git a/tests/test_sample_app_reports.py b/tests/test_sample_app_reports.py index 5294235..2c1697f 100644 --- a/tests/test_sample_app_reports.py +++ b/tests/test_sample_app_reports.py @@ -8,7 +8,9 @@ import pytest import dependency_scanner.osv_client as osv_client_module +import tools.generate_sample_app_reports as report_module from tools.generate_sample_app_reports import ( + _portable_path, build_dependency_scan_report, build_reports, find_stale_reports, @@ -178,3 +180,74 @@ def test_check_mode_reports_stale_artifact( assert exit_code == 1 assert str(stale_path) in capsys.readouterr().out assert stale_path.read_text(encoding="utf-8") == "{}\n" + + +def test_check_mode_accepts_current_reports( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Check mode should succeed for exact generated artifacts.""" + + write_reports(tmp_path) + + assert main(["--check", "--output-directory", str(tmp_path)]) == 0 + assert "are current" in capsys.readouterr().out + + +def test_check_mode_reports_missing_artifact( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Check mode should identify a missing generated report.""" + + write_reports(tmp_path) + missing_path = tmp_path / "dependency-scan.json" + missing_path.unlink() + + assert main(["--check", "--output-directory", str(tmp_path)]) == 1 + assert str(missing_path) in capsys.readouterr().out + + +def test_main_writes_both_reports( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Write mode should create and announce both canonical documents.""" + + assert main(["--output-directory", str(tmp_path)]) == 0 + output = capsys.readouterr().out + + assert (tmp_path / "static-analysis.json").exists() + assert (tmp_path / "dependency-scan.json").exists() + assert output.count("Wrote report:") == 2 + + +def test_dependency_report_rejects_failed_scan( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Baseline generation should never serialize a partial scan.""" + + class FailedScanner: + def scan_requirements(self, path: Path) -> object: + del path + return type("Result", (), {"succeeded": False})() + + monkeypatch.setattr( + report_module, + "DependencyScanner", + lambda source: FailedScanner(), + ) + + with pytest.raises(RuntimeError, match="fixture-backed"): + build_dependency_scan_report() + + +def test_report_paths_are_repository_scoped(tmp_path: Path) -> None: + """Portable paths should support relative inputs and reject outsiders.""" + + assert _portable_path("sample_app/requirements.txt") == ( + "sample_app/requirements.txt" + ) + + with pytest.raises(ValueError, match="outside the repository"): + _portable_path(str(tmp_path / "requirements.txt")) diff --git a/tests/test_sample_app_security_demo.py b/tests/test_sample_app_security_demo.py index e8317ba..fb03232 100644 --- a/tests/test_sample_app_security_demo.py +++ b/tests/test_sample_app_security_demo.py @@ -39,6 +39,13 @@ / "fastapi-0.109.0.json" ) _DEMO_LITERAL = "demo-only-not-a-real-secret" +_EXPECTED_ANALYZER_FINDINGS = [ + ("SA005", "analyzer_demo.py", 8, 1, "warning"), + ("SA001", "analyzer_demo.py", 11, 0, "warning"), + ("SA006", "analyzer_demo.py", 11, 1, "info"), + ("SA003", "analyzer_demo.py", 14, 7, "info"), + ("SA004", "analyzer_demo.py", 54, 5, "warning"), +] class FixtureOsvClient: @@ -93,43 +100,7 @@ def test_analyzer_demo_produces_exact_findings() -> None: finding.severity.value, ) for finding in findings - ] == [ - ( - "SA005", - "analyzer_demo.py", - 8, - 1, - "warning", - ), - ( - "SA001", - "analyzer_demo.py", - 11, - 0, - "warning", - ), - ( - "SA006", - "analyzer_demo.py", - 11, - 1, - "info", - ), - ( - "SA003", - "analyzer_demo.py", - 14, - 7, - "info", - ), - ( - "SA004", - "analyzer_demo.py", - 54, - 5, - "warning", - ), - ] + ] == _EXPECTED_ANALYZER_FINDINGS report = format_findings_json(findings) assert _DEMO_LITERAL not in report diff --git a/tests/test_self_analysis_report.py b/tests/test_self_analysis_report.py new file mode 100644 index 0000000..0c5541f --- /dev/null +++ b/tests/test_self_analysis_report.py @@ -0,0 +1,120 @@ +"""Tests for the checked-in whole-project self-analysis report.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tools.generate_self_analysis_report import ( + build_self_analysis_report, + main, + report_is_current, + write_self_analysis_report, +) + + +_REPOSITORY_ROOT = Path(__file__).parents[1] +_REPORT_PATH = ( + _REPOSITORY_ROOT / "reports" / "project" / "static-analysis.json" +) +_EXPECTED_FINDINGS = [ + ("SA005", "sample_app/analyzer_demo.py", 8, "warning"), + ("SA001", "sample_app/analyzer_demo.py", 11, "warning"), + ("SA006", "sample_app/analyzer_demo.py", 11, "info"), + ("SA003", "sample_app/analyzer_demo.py", 14, "info"), + ("SA004", "sample_app/analyzer_demo.py", 54, "warning"), +] + + +def test_checked_in_self_analysis_report_is_current() -> None: + """The committed artifact should match a fresh production analysis.""" + + assert report_is_current(_REPORT_PATH) + assert _REPORT_PATH.read_text(encoding="utf-8") == ( + build_self_analysis_report() + ) + + +def test_self_analysis_contains_only_controlled_demo_findings() -> None: + """All remaining project findings should be intentional true positives.""" + + payload = json.loads(build_self_analysis_report()) + + assert payload["summary"] == {"total": 5} + assert [ + ( + finding["rule_id"], + finding["file_path"], + finding["line_number"], + finding["severity"], + ) + for finding in payload["findings"] + ] == _EXPECTED_FINDINGS + + +def test_self_analysis_report_can_be_written_from_any_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Generation should not depend on the process working directory.""" + + output_path = tmp_path / "reports" / "self-analysis.json" + monkeypatch.chdir(tmp_path) + + assert write_self_analysis_report(output_path) == output_path + assert output_path.read_text(encoding="utf-8") == ( + build_self_analysis_report() + ) + + +def test_self_analysis_check_rejects_missing_report( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Check mode should fail closed when the artifact is absent.""" + + output_path = tmp_path / "missing.json" + + assert main(["--check", "--output", str(output_path)]) == 1 + assert str(output_path) in capsys.readouterr().out + + +def test_self_analysis_check_rejects_stale_report( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Check mode should report drift without replacing the artifact.""" + + output_path = tmp_path / "stale.json" + output_path.write_text("{}\n", encoding="utf-8") + + assert main(["--check", "--output", str(output_path)]) == 1 + assert str(output_path) in capsys.readouterr().out + assert output_path.read_text(encoding="utf-8") == "{}\n" + + +def test_self_analysis_check_accepts_current_report( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Check mode should accept an exact generated artifact.""" + + output_path = write_self_analysis_report(tmp_path / "current.json") + + assert main(["--check", "--output", str(output_path)]) == 0 + assert "is current" in capsys.readouterr().out + + +def test_self_analysis_main_writes_report( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Write mode should create the selected artifact.""" + + output_path = tmp_path / "generated" / "report.json" + + assert main(["--output", str(output_path)]) == 0 + assert output_path.exists() + assert str(output_path) in capsys.readouterr().out diff --git a/tools/generate_self_analysis_report.py b/tools/generate_self_analysis_report.py new file mode 100644 index 0000000..8492bc5 --- /dev/null +++ b/tools/generate_self_analysis_report.py @@ -0,0 +1,116 @@ +"""Generate the deterministic whole-project static-analysis report.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Sequence +from pathlib import Path + +from static_analyzer.default_factory import create_default_analyzer +from static_analyzer.formatters import format_findings_json + + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +_DEFAULT_OUTPUT_PATH = ( + _REPOSITORY_ROOT / "reports" / "project" / "static-analysis.json" +) + + +def build_self_analysis_report() -> str: + """Return a portable report for every Python file in the repository.""" + + findings = create_default_analyzer().analyze(_REPOSITORY_ROOT) + payload = json.loads(format_findings_json(findings)) + + for finding in payload["findings"]: + finding["file_path"] = _portable_path(finding["file_path"]) + + return _serialize(payload) + + +def write_self_analysis_report( + output_path: Path = _DEFAULT_OUTPUT_PATH, +) -> Path: + """Write the current report and return its destination.""" + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + build_self_analysis_report(), + encoding="utf-8", + newline="\n", + ) + return output_path + + +def report_is_current( + output_path: Path = _DEFAULT_OUTPUT_PATH, +) -> bool: + """Return whether a checked-in report matches a fresh analysis.""" + + try: + current = output_path.read_text(encoding="utf-8") + except FileNotFoundError: + return False + + return current == build_self_analysis_report() + + +def main(argv: Sequence[str] | None = None) -> int: + """Write the self-analysis report or check it for drift.""" + + arguments = _build_parser().parse_args(argv) + + if arguments.check: + if not report_is_current(arguments.output): + print(f"Outdated report: {arguments.output}") + return 1 + + print("Whole-project self-analysis report is current.") + return 0 + + written_path = write_self_analysis_report(arguments.output) + print(f"Wrote report: {written_path}") + return 0 + + +def _portable_path(value: str) -> str: + """Return one repository-relative path using POSIX separators.""" + + path = Path(value) + + if not path.is_absolute(): + path = _REPOSITORY_ROOT / path + + relative_path = path.resolve().relative_to(_REPOSITORY_ROOT.resolve()) + return relative_path.as_posix() + + +def _serialize(payload: object) -> str: + """Serialize one canonical checked-in JSON document.""" + + return json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + + +def _build_parser() -> argparse.ArgumentParser: + """Create the report generator argument parser.""" + + parser = argparse.ArgumentParser( + description="Generate the whole-project static-analysis report." + ) + parser.add_argument( + "--check", + action="store_true", + help="Return exit code 1 when the report is missing or stale.", + ) + parser.add_argument( + "--output", + type=Path, + default=_DEFAULT_OUTPUT_PATH, + help="Destination for the canonical JSON report.", + ) + return parser + + +if __name__ == "__main__": + raise SystemExit(main())