From 72c05803522fbd3b1638a811d7814352f7a8fbf5 Mon Sep 17 00:00:00 2001 From: saagpatel <269905221+saagpatel@users.noreply.github.com> Date: Fri, 6 Nov 2026 02:36:54 -0800 Subject: [PATCH 1/6] feat(portfolio): report missing prior workspace inventory keys --- .../portfolio_truth_metadata.py | 6 +- .../portfolio_truth_publish.py | 97 +++++++++++++++ .../portfolio_truth_validate.py | 26 ++++ tests/test_workspace_inventory_gap.py | 116 ++++++++++++++++++ 4 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 tests/test_workspace_inventory_gap.py diff --git a/src/github_repo_auditor/portfolio_truth_metadata.py b/src/github_repo_auditor/portfolio_truth_metadata.py index 37878dac..babdc863 100644 --- a/src/github_repo_auditor/portfolio_truth_metadata.py +++ b/src/github_repo_auditor/portfolio_truth_metadata.py @@ -81,9 +81,10 @@ def build_source_summary( notion_context_rows: int, notion_context_carried_forward: bool, checkout_collisions: Sequence[dict[str, Any]] = (), + workspace_inventory_gap: dict[str, Any] | None = None, ) -> dict[str, Any]: """Build the exact producer summary from bounded inputs and project facts.""" - return { + summary = { "workspace_root": workspace_root, "project_count": len(projects), "checkout_collisions": checkout_collision_summary(list(checkout_collisions)), @@ -127,6 +128,9 @@ def build_source_summary( projects ), } + if workspace_inventory_gap is not None: + summary["workspace_inventory_gap"] = workspace_inventory_gap + return summary def build_warnings( diff --git a/src/github_repo_auditor/portfolio_truth_publish.py b/src/github_repo_auditor/portfolio_truth_publish.py index 1011c1a6..b95f75f0 100644 --- a/src/github_repo_auditor/portfolio_truth_publish.py +++ b/src/github_repo_auditor/portfolio_truth_publish.py @@ -337,6 +337,91 @@ def _build_project_registry_json(snapshot, *, include_notion: bool) -> str: return json.dumps(registry, indent=2) + "\n" +def _workspace_inventory_gap( + latest_path: Path, + *, + workspace_root: Path, + current_project_keys: set[str], +) -> dict[str, object] | None: + """Describe prior workspace identities absent from this same-root inventory. + + A missing key is only evidence that a previously published project key is + absent from this scan. It does not establish why the project disappeared. + """ + try: + prior = json.loads(latest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + if not isinstance(prior, dict): + return None + + prior_root = prior.get("workspace_root") + if not isinstance(prior_root, str) or not prior_root: + return None + try: + if ( + Path(prior_root).expanduser().resolve() + != workspace_root.expanduser().resolve() + ): + return None + except (OSError, RuntimeError): + return None + + generated_at = prior.get("generated_at") + prior_projects = prior.get("projects") + if not isinstance(generated_at, str) or not generated_at.strip(): + return None + if not isinstance(prior_projects, list): + return None + + prior_keys: set[str] = set() + for project in prior_projects: + if not isinstance(project, dict): + return None + identity = project.get("identity") + if not isinstance(identity, dict): + return None + key = identity.get("project_key") + if not isinstance(key, str) or not key: + return None + if not key.startswith("supp:"): + prior_keys.add(key) + + baseline_generated_at = generated_at + source_summary = prior.get("source_summary") + prior_gap = ( + source_summary.get("workspace_inventory_gap") + if isinstance(source_summary, dict) + else None + ) + if prior_gap is not None: + if not isinstance(prior_gap, dict): + return None + missing = prior_gap.get("missing_project_keys") + baseline = prior_gap.get("baseline_generated_at") + if ( + set(prior_gap) != {"missing_project_keys", "baseline_generated_at"} + or not isinstance(missing, list) + or any(not isinstance(key, str) or not key for key in missing) + or missing != sorted(set(missing)) + or any(key.startswith("supp:") for key in missing) + or set(missing).intersection(prior_keys) + or not isinstance(baseline, str) + or not baseline.strip() + ): + return None + prior_keys.update(key for key in missing if not key.startswith("supp:")) + baseline_generated_at = baseline + + missing_keys = sorted(prior_keys - current_project_keys) + if not missing_keys: + return None + return { + "missing_project_keys": missing_keys, + "baseline_generated_at": baseline_generated_at, + } + + def publish_portfolio_truth( *, workspace_root: Path, @@ -564,6 +649,18 @@ def _publish_portfolio_truth_locked( raise CohortR3Error( f"{exc} R3 diagnostics: {diagnostics_path}", exc.diagnostics ) from exc + current_project_keys = { + project.identity.project_key + for project in build_result.snapshot.projects + if not project.identity.project_key.startswith("supp:") + } + inventory_gap = _workspace_inventory_gap( + latest_path, + workspace_root=workspace_root, + current_project_keys=current_project_keys, + ) + if inventory_gap is not None: + build_result.snapshot.source_summary["workspace_inventory_gap"] = inventory_gap validate_truth_snapshot( build_result.snapshot, security_max_age_hours=security_max_age_hours, diff --git a/src/github_repo_auditor/portfolio_truth_validate.py b/src/github_repo_auditor/portfolio_truth_validate.py index 5e56497e..70b5afa3 100644 --- a/src/github_repo_auditor/portfolio_truth_validate.py +++ b/src/github_repo_auditor/portfolio_truth_validate.py @@ -532,6 +532,29 @@ def _validate_snapshot_metadata( or not isinstance(notion_context_carried_forward, bool) ): raise ValueError("PortfolioTruth source summary has invalid bounded inputs.") + workspace_inventory_gap = summary.get("workspace_inventory_gap") + if workspace_inventory_gap is not None: + if not isinstance(workspace_inventory_gap, dict) or set( + workspace_inventory_gap + ) != {"missing_project_keys", "baseline_generated_at"}: + raise ValueError("PortfolioTruth workspace inventory gap is malformed.") + missing_project_keys = workspace_inventory_gap.get("missing_project_keys") + baseline_generated_at = workspace_inventory_gap.get("baseline_generated_at") + current_project_keys = { + project.identity.project_key + for project in snapshot.projects + if not project.identity.project_key.startswith("supp:") + } + if ( + not isinstance(missing_project_keys, list) + or any(not isinstance(key, str) or not key for key in missing_project_keys) + or missing_project_keys != sorted(set(missing_project_keys)) + or any(key.startswith("supp:") for key in missing_project_keys) + or current_project_keys.intersection(missing_project_keys) + or not isinstance(baseline_generated_at, str) + or not baseline_generated_at.strip() + ): + raise ValueError("PortfolioTruth workspace inventory gap is invalid.") expected_summary = build_source_summary( workspace_root=snapshot.workspace_root, projects=snapshot.projects, @@ -541,6 +564,7 @@ def _validate_snapshot_metadata( notion_context_rows=notion_context_rows, notion_context_carried_forward=notion_context_carried_forward, checkout_collisions=summary["checkout_collisions"]["groups"], + workspace_inventory_gap=workspace_inventory_gap, ) if "security_coverage_clock" in summary: clock = summary["security_coverage_clock"] @@ -2048,6 +2072,7 @@ def canonicalize_prior_security_truth_payload( "notion_context_carried_forward" ], checkout_collisions=[], + workspace_inventory_gap=summary.get("workspace_inventory_gap"), ) expected_legacy_summary = deepcopy(current_summary) expected_legacy_summary.pop("checkout_collisions") @@ -2112,6 +2137,7 @@ def canonicalize_prior_security_truth_payload( "notion_context_carried_forward" ], checkout_collisions=[], + workspace_inventory_gap=summary.get("workspace_inventory_gap"), ) migrated["exclusions"] = build_exclusions(dict(exclusion_counts)) canonicalize_truth_snapshot_payload( diff --git a/tests/test_workspace_inventory_gap.py b/tests/test_workspace_inventory_gap.py new file mode 100644 index 00000000..9f62ac8f --- /dev/null +++ b/tests/test_workspace_inventory_gap.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +import shutil +from datetime import datetime, timezone +from pathlib import Path + +from github_repo_auditor.portfolio_truth_publish import ( + _workspace_inventory_gap, + publish_portfolio_truth, +) + + +def _make_project(workspace: Path, name: str) -> None: + path = workspace / name + path.mkdir(parents=True, exist_ok=True) + (path / "README.md").write_text(f"# {name}\n\nSynthetic inventory fixture.\n") + + +def _publish(workspace: Path, output: Path, at: datetime) -> dict: + output.mkdir(parents=True, exist_ok=True) + result = publish_portfolio_truth( + workspace_root=workspace, + output_dir=output, + registry_output=workspace / "project-registry.md", + portfolio_report_output=workspace / "PORTFOLIO-AUDIT-REPORT.md", + catalog_path=workspace.parent / "missing-portfolio-catalog.yaml", + include_notion=False, + now=at, + ) + return json.loads(result.latest_path.read_text(encoding="utf-8")) + + +def test_workspace_inventory_gap_tracks_disappearance_recovery_and_root_change( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_project(workspace, "Alpha") + _make_project(workspace, "Beta") + _make_project(workspace, "Gamma") + output = tmp_path / "output" + first_at = datetime(2026, 11, 6, 10, tzinfo=timezone.utc) + + first = _publish(workspace, output, first_at) + assert "workspace_inventory_gap" not in first["source_summary"] + beta_key = next( + project["identity"]["project_key"] + for project in first["projects"] + if project["identity"]["display_name"] == "Beta" + ) + gamma_key = next( + project["identity"]["project_key"] + for project in first["projects"] + if project["identity"]["display_name"] == "Gamma" + ) + + shutil.rmtree(workspace / "Beta") + second = _publish(workspace, output, first_at.replace(minute=1)) + assert second["source_summary"]["workspace_inventory_gap"] == { + "missing_project_keys": [beta_key], + "baseline_generated_at": first["generated_at"], + } + + shutil.rmtree(workspace / "Gamma") + repeated = _publish(workspace, output, first_at.replace(minute=2)) + assert repeated["source_summary"]["workspace_inventory_gap"] == { + "missing_project_keys": sorted([beta_key, gamma_key]), + "baseline_generated_at": first["generated_at"], + } + + _make_project(workspace, "Beta") + _make_project(workspace, "Gamma") + recovered = _publish(workspace, output, first_at.replace(minute=3)) + assert "workspace_inventory_gap" not in recovered["source_summary"] + + other_workspace = tmp_path / "other-workspace" + other_workspace.mkdir() + _make_project(other_workspace, "Alpha") + different_root = _publish(other_workspace, output, first_at.replace(minute=4)) + assert "workspace_inventory_gap" not in different_root["source_summary"] + + +def test_workspace_inventory_gap_excludes_supplementary_keys_and_ignores_bad_prior( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + latest = tmp_path / "portfolio-truth-latest.json" + latest.write_text( + json.dumps( + { + "workspace_root": str(workspace), + "generated_at": "2026-11-06T10:00:00+00:00", + "projects": [ + {"identity": {"project_key": "repo:saagar/alpha"}}, + {"identity": {"project_key": "supp:calendar"}}, + ], + } + ), + encoding="utf-8", + ) + assert _workspace_inventory_gap( + latest, workspace_root=workspace, current_project_keys=set() + ) == { + "missing_project_keys": ["repo:saagar/alpha"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + } + + latest.write_bytes(b"\xff\xfe invalid utf-8") + assert ( + _workspace_inventory_gap( + latest, workspace_root=workspace, current_project_keys=set() + ) + is None + ) From 31c32785cc03c37e84dead023fa71ce049ce40ef Mon Sep 17 00:00:00 2001 From: saagpatel <269905221+saagpatel@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:00:53 -0800 Subject: [PATCH 2/6] docs: note explicit workspace inventory gaps --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d50791a..da89b461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] +- Preserve missing workspace project keys as explicit inventory gaps across + portfolio truth generations so consumers can surface incomplete scans. + - Stop retrying ambiguous Notion writes, verify weekly-review replacements before archival, distinguish async GitHub rate limits, and validate web report payloads. - Replace vacuous tier/initiative tests with deterministic checks; pin the From 3fe62365e283922d70170cc378b008c98e84c0f9 Mon Sep 17 00:00:00 2001 From: saagpatel <269905221+saagpatel@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:29:07 -0800 Subject: [PATCH 3/6] Track unresolved workspace inventory gaps --- .../portfolio_truth_metadata.py | 3 + .../portfolio_truth_publish.py | 127 ++++++++-- .../portfolio_truth_validate.py | 52 ++++- tests/test_workspace_inventory_gap.py | 218 ++++++++++++++++-- 4 files changed, 363 insertions(+), 37 deletions(-) diff --git a/src/github_repo_auditor/portfolio_truth_metadata.py b/src/github_repo_auditor/portfolio_truth_metadata.py index babdc863..9970254a 100644 --- a/src/github_repo_auditor/portfolio_truth_metadata.py +++ b/src/github_repo_auditor/portfolio_truth_metadata.py @@ -82,6 +82,7 @@ def build_source_summary( notion_context_carried_forward: bool, checkout_collisions: Sequence[dict[str, Any]] = (), workspace_inventory_gap: dict[str, Any] | None = None, + workspace_inventory_comparison: dict[str, str] | None = None, ) -> dict[str, Any]: """Build the exact producer summary from bounded inputs and project facts.""" summary = { @@ -130,6 +131,8 @@ def build_source_summary( } if workspace_inventory_gap is not None: summary["workspace_inventory_gap"] = workspace_inventory_gap + if workspace_inventory_comparison is not None: + summary["workspace_inventory_comparison"] = workspace_inventory_comparison return summary diff --git a/src/github_repo_auditor/portfolio_truth_publish.py b/src/github_repo_auditor/portfolio_truth_publish.py index b95f75f0..b24c51a9 100644 --- a/src/github_repo_auditor/portfolio_truth_publish.py +++ b/src/github_repo_auditor/portfolio_truth_publish.py @@ -4,6 +4,7 @@ import hashlib import json import os +import stat import tempfile import threading from contextlib import contextmanager, nullcontext @@ -342,50 +343,66 @@ def _workspace_inventory_gap( *, workspace_root: Path, current_project_keys: set[str], -) -> dict[str, object] | None: + current_project_paths: set[str], +) -> tuple[dict[str, object] | None, str | None]: """Describe prior workspace identities absent from this same-root inventory. - A missing key is only evidence that a previously published project key is - absent from this scan. It does not establish why the project disappeared. + Missing keys remain gaps while their recorded paths still exist in the + workspace. A missing path or a path now represented by a different current + key is treated as a confirmed removal or rename. Unreadable paths and paths + outside the workspace retain uncertainty and stay in the gap. The baseline + timestamp marks the start of the continuous unresolved gap chain, so later + missing keys can share that earlier baseline until the chain fully clears. """ try: prior = json.loads(latest_path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None, None except (OSError, UnicodeError, json.JSONDecodeError): - return None + return None, "prior_snapshot_unreadable" if not isinstance(prior, dict): - return None + return None, "prior_snapshot_malformed" prior_root = prior.get("workspace_root") if not isinstance(prior_root, str) or not prior_root: - return None + return None, "prior_workspace_unknown" try: if ( Path(prior_root).expanduser().resolve() != workspace_root.expanduser().resolve() ): - return None + return None, "workspace_root_changed" except (OSError, RuntimeError): - return None + return None, "workspace_root_unavailable" generated_at = prior.get("generated_at") prior_projects = prior.get("projects") if not isinstance(generated_at, str) or not generated_at.strip(): - return None + return None, "prior_generation_time_missing" if not isinstance(prior_projects, list): - return None + return None, "prior_project_inventory_malformed" prior_keys: set[str] = set() + prior_paths: dict[str, str] = {} for project in prior_projects: if not isinstance(project, dict): - return None + return None, "prior_project_inventory_malformed" identity = project.get("identity") if not isinstance(identity, dict): - return None + return None, "prior_project_identity_malformed" key = identity.get("project_key") if not isinstance(key, str) or not key: - return None + return None, "prior_project_key_malformed" if not key.startswith("supp:"): prior_keys.add(key) + path = identity.get("path") + if ( + isinstance(path, str) + and path + and not Path(path).is_absolute() + and ".." not in Path(path).parts + ): + prior_paths[key] = path baseline_generated_at = generated_at source_summary = prior.get("source_summary") @@ -396,30 +413,90 @@ def _workspace_inventory_gap( ) if prior_gap is not None: if not isinstance(prior_gap, dict): - return None + return None, "prior_inventory_gap_malformed" missing = prior_gap.get("missing_project_keys") baseline = prior_gap.get("baseline_generated_at") + missing_paths = prior_gap.get("missing_project_paths", {}) if ( - set(prior_gap) != {"missing_project_keys", "baseline_generated_at"} + set(prior_gap) + not in ( + {"missing_project_keys", "baseline_generated_at"}, + { + "missing_project_keys", + "baseline_generated_at", + "missing_project_paths", + }, + ) or not isinstance(missing, list) or any(not isinstance(key, str) or not key for key in missing) or missing != sorted(set(missing)) or any(key.startswith("supp:") for key in missing) or set(missing).intersection(prior_keys) + or not isinstance(missing_paths, dict) + or not set(missing_paths).issubset(missing) + or any( + not isinstance(path, str) + or not path + or Path(path).is_absolute() + or ".." in Path(path).parts + for path in missing_paths.values() + ) or not isinstance(baseline, str) or not baseline.strip() ): - return None - prior_keys.update(key for key in missing if not key.startswith("supp:")) + return None, "prior_inventory_gap_malformed" + prior_paths.update(missing_paths) + prior_keys.update(missing) baseline_generated_at = baseline missing_keys = sorted(prior_keys - current_project_keys) + missing_keys = [ + key + for key in missing_keys + if ( + key not in prior_paths + or ( + prior_paths[key] not in current_project_paths + and _workspace_path_is_directory(workspace_root, prior_paths[key]) + ) + ) + ] if not missing_keys: - return None - return { + return None, None + gap: dict[str, object] = { "missing_project_keys": missing_keys, "baseline_generated_at": baseline_generated_at, } + missing_paths = { + key: prior_paths[key] for key in missing_keys if key in prior_paths + } + if missing_paths: + gap["missing_project_paths"] = missing_paths + return gap, None + + +def _workspace_path_is_directory(workspace_root: Path, relative_path: str) -> bool: + """Return false only when a relative workspace path is confirmed absent.""" + path = Path(relative_path) + if path.is_absolute() or ".." in path.parts or not path.parts: + return True + try: + root = workspace_root.expanduser().resolve() + root_mode = root.stat().st_mode + except (OSError, RuntimeError, ValueError): + return True + if not stat.S_ISDIR(root_mode): + return True + try: + candidate = (root / path).resolve() + if not candidate.is_relative_to(root): + return True + mode = candidate.stat().st_mode + except (FileNotFoundError, NotADirectoryError): + return False + except (OSError, RuntimeError, ValueError): + return True + return stat.S_ISDIR(mode) def publish_portfolio_truth( @@ -654,13 +731,23 @@ def _publish_portfolio_truth_locked( for project in build_result.snapshot.projects if not project.identity.project_key.startswith("supp:") } - inventory_gap = _workspace_inventory_gap( + inventory_gap, inventory_comparison_unavailable = _workspace_inventory_gap( latest_path, workspace_root=workspace_root, current_project_keys=current_project_keys, + current_project_paths={ + project.identity.path + for project in build_result.snapshot.projects + if not project.identity.project_key.startswith("supp:") + }, ) if inventory_gap is not None: build_result.snapshot.source_summary["workspace_inventory_gap"] = inventory_gap + if inventory_comparison_unavailable is not None: + build_result.snapshot.source_summary["workspace_inventory_comparison"] = { + "state": "unavailable", + "reason": inventory_comparison_unavailable, + } validate_truth_snapshot( build_result.snapshot, security_max_age_hours=security_max_age_hours, diff --git a/src/github_repo_auditor/portfolio_truth_validate.py b/src/github_repo_auditor/portfolio_truth_validate.py index 70b5afa3..7ec023ab 100644 --- a/src/github_repo_auditor/portfolio_truth_validate.py +++ b/src/github_repo_auditor/portfolio_truth_validate.py @@ -536,15 +536,28 @@ def _validate_snapshot_metadata( if workspace_inventory_gap is not None: if not isinstance(workspace_inventory_gap, dict) or set( workspace_inventory_gap - ) != {"missing_project_keys", "baseline_generated_at"}: + ) not in ( + {"missing_project_keys", "baseline_generated_at"}, + { + "missing_project_keys", + "baseline_generated_at", + "missing_project_paths", + }, + ): raise ValueError("PortfolioTruth workspace inventory gap is malformed.") missing_project_keys = workspace_inventory_gap.get("missing_project_keys") baseline_generated_at = workspace_inventory_gap.get("baseline_generated_at") + missing_project_paths = workspace_inventory_gap.get("missing_project_paths", {}) current_project_keys = { project.identity.project_key for project in snapshot.projects if not project.identity.project_key.startswith("supp:") } + current_project_paths = { + project.identity.path + for project in snapshot.projects + if not project.identity.project_key.startswith("supp:") + } if ( not isinstance(missing_project_keys, list) or any(not isinstance(key, str) or not key for key in missing_project_keys) @@ -553,8 +566,38 @@ def _validate_snapshot_metadata( or current_project_keys.intersection(missing_project_keys) or not isinstance(baseline_generated_at, str) or not baseline_generated_at.strip() + or not isinstance(missing_project_paths, dict) + or not set(missing_project_paths).issubset(missing_project_keys) + or any( + not isinstance(path, str) + or not path + or Path(path).is_absolute() + or ".." in Path(path).parts + for path in missing_project_paths.values() + ) + or current_project_paths.intersection(missing_project_paths.values()) ): raise ValueError("PortfolioTruth workspace inventory gap is invalid.") + workspace_inventory_comparison = summary.get("workspace_inventory_comparison") + valid_comparison_reasons = { + "prior_snapshot_unreadable", + "prior_snapshot_malformed", + "prior_workspace_unknown", + "workspace_root_changed", + "workspace_root_unavailable", + "prior_generation_time_missing", + "prior_project_inventory_malformed", + "prior_project_identity_malformed", + "prior_project_key_malformed", + "prior_inventory_gap_malformed", + } + if workspace_inventory_comparison is not None and ( + not isinstance(workspace_inventory_comparison, dict) + or set(workspace_inventory_comparison) != {"state", "reason"} + or workspace_inventory_comparison.get("state") != "unavailable" + or workspace_inventory_comparison.get("reason") not in valid_comparison_reasons + ): + raise ValueError("PortfolioTruth workspace inventory comparison is malformed.") expected_summary = build_source_summary( workspace_root=snapshot.workspace_root, projects=snapshot.projects, @@ -565,6 +608,7 @@ def _validate_snapshot_metadata( notion_context_carried_forward=notion_context_carried_forward, checkout_collisions=summary["checkout_collisions"]["groups"], workspace_inventory_gap=workspace_inventory_gap, + workspace_inventory_comparison=workspace_inventory_comparison, ) if "security_coverage_clock" in summary: clock = summary["security_coverage_clock"] @@ -2073,6 +2117,9 @@ def canonicalize_prior_security_truth_payload( ], checkout_collisions=[], workspace_inventory_gap=summary.get("workspace_inventory_gap"), + workspace_inventory_comparison=summary.get( + "workspace_inventory_comparison" + ), ) expected_legacy_summary = deepcopy(current_summary) expected_legacy_summary.pop("checkout_collisions") @@ -2138,6 +2185,9 @@ def canonicalize_prior_security_truth_payload( ], checkout_collisions=[], workspace_inventory_gap=summary.get("workspace_inventory_gap"), + workspace_inventory_comparison=summary.get( + "workspace_inventory_comparison" + ), ) migrated["exclusions"] = build_exclusions(dict(exclusion_counts)) canonicalize_truth_snapshot_payload( diff --git a/tests/test_workspace_inventory_gap.py b/tests/test_workspace_inventory_gap.py index 9f62ac8f..46d63edb 100644 --- a/tests/test_workspace_inventory_gap.py +++ b/tests/test_workspace_inventory_gap.py @@ -4,11 +4,19 @@ import shutil from datetime import datetime, timezone from pathlib import Path +from unittest.mock import patch + +import pytest from github_repo_auditor.portfolio_truth_publish import ( + _workspace_path_is_directory, _workspace_inventory_gap, publish_portfolio_truth, ) +from github_repo_auditor.portfolio_truth_validate import ( + _snapshot_from_payload, + validate_truth_snapshot, +) def _make_project(workspace: Path, name: str) -> None: @@ -55,29 +63,48 @@ def test_workspace_inventory_gap_tracks_disappearance_recovery_and_root_change( if project["identity"]["display_name"] == "Gamma" ) - shutil.rmtree(workspace / "Beta") - second = _publish(workspace, output, first_at.replace(minute=1)) + original_is_dir = Path.is_dir + + def omit_beta(path: Path) -> bool: + if path == workspace / "Beta": + return False + return original_is_dir(path) + + with patch.object(Path, "is_dir", omit_beta): + second = _publish(workspace, output, first_at.replace(minute=1)) assert second["source_summary"]["workspace_inventory_gap"] == { "missing_project_keys": [beta_key], "baseline_generated_at": first["generated_at"], + "missing_project_paths": {beta_key: "Beta"}, } - shutil.rmtree(workspace / "Gamma") - repeated = _publish(workspace, output, first_at.replace(minute=2)) + def omit_beta_and_gamma(path: Path) -> bool: + if path in {workspace / "Beta", workspace / "Gamma"}: + return False + return original_is_dir(path) + + with patch.object(Path, "is_dir", omit_beta_and_gamma): + repeated = _publish(workspace, output, first_at.replace(minute=2)) assert repeated["source_summary"]["workspace_inventory_gap"] == { "missing_project_keys": sorted([beta_key, gamma_key]), "baseline_generated_at": first["generated_at"], + "missing_project_paths": {beta_key: "Beta", gamma_key: "Gamma"}, } + shutil.rmtree(workspace / "Beta") + shutil.rmtree(workspace / "Gamma") + deleted = _publish(workspace, output, first_at.replace(minute=3)) + assert "workspace_inventory_gap" not in deleted["source_summary"] + _make_project(workspace, "Beta") _make_project(workspace, "Gamma") - recovered = _publish(workspace, output, first_at.replace(minute=3)) + recovered = _publish(workspace, output, first_at.replace(minute=4)) assert "workspace_inventory_gap" not in recovered["source_summary"] other_workspace = tmp_path / "other-workspace" other_workspace.mkdir() _make_project(other_workspace, "Alpha") - different_root = _publish(other_workspace, output, first_at.replace(minute=4)) + different_root = _publish(other_workspace, output, first_at.replace(minute=5)) assert "workspace_inventory_gap" not in different_root["source_summary"] @@ -101,16 +128,175 @@ def test_workspace_inventory_gap_excludes_supplementary_keys_and_ignores_bad_pri encoding="utf-8", ) assert _workspace_inventory_gap( - latest, workspace_root=workspace, current_project_keys=set() - ) == { - "missing_project_keys": ["repo:saagar/alpha"], - "baseline_generated_at": "2026-11-06T10:00:00+00:00", - } + latest, + workspace_root=workspace, + current_project_keys=set(), + current_project_paths=set(), + ) == ( + { + "missing_project_keys": ["repo:saagar/alpha"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + }, + None, + ) latest.write_bytes(b"\xff\xfe invalid utf-8") - assert ( - _workspace_inventory_gap( - latest, workspace_root=workspace, current_project_keys=set() - ) - is None + assert _workspace_inventory_gap( + latest, + workspace_root=workspace, + current_project_keys=set(), + current_project_paths=set(), + ) == (None, "prior_snapshot_unreadable") + + +@pytest.mark.parametrize( + "malformed_gap", + [ + { + "missing_project_keys": ["Alpha", "Alpha"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + }, + ["not", "a", "mapping"], + ], +) +def test_same_root_malformed_gap_marks_comparison_unavailable( + tmp_path: Path, malformed_gap: object +) -> None: + workspace = tmp_path / "workspace" + _make_project(workspace, "Alpha") + output = tmp_path / "output" + generated_at = datetime(2026, 11, 6, 10, tzinfo=timezone.utc) + baseline = _publish(workspace, output, generated_at) + latest = output / "portfolio-truth-latest.json" + baseline["source_summary"]["workspace_inventory_gap"] = malformed_gap + latest.write_text(json.dumps(baseline), encoding="utf-8") + + recovered = _publish(workspace, output, generated_at.replace(minute=1)) + + assert recovered["source_summary"]["workspace_inventory_comparison"] == { + "state": "unavailable", + "reason": "prior_inventory_gap_malformed", + } + assert "workspace_inventory_gap" not in recovered["source_summary"] + + +@pytest.mark.parametrize( + "gap", + [ + {"missing_project_keys": ["Beta"], "baseline_generated_at": ""}, + { + "missing_project_keys": ["Beta", "Beta"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + }, + { + "missing_project_keys": ["Gamma", "Beta"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + }, + { + "missing_project_keys": ["supp:calendar"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + }, + { + "missing_project_keys": ["Alpha"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + }, + { + "missing_project_keys": ["Beta"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + "missing_project_paths": {"Alpha": "Alpha"}, + }, + { + "missing_project_keys": ["Beta"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + "missing_project_paths": {"Beta": "../outside"}, + }, + { + "missing_project_keys": ["Beta"], + "baseline_generated_at": "2026-11-06T10:00:00+00:00", + "missing_project_paths": {"Beta": "Alpha"}, + }, + ], +) +def test_invalid_workspace_inventory_gap_is_rejected( + tmp_path: Path, gap: dict[str, object] +) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_project(workspace, "Alpha") + _make_project(workspace, "Beta") + snapshot = _publish( + workspace, + tmp_path / "output", + datetime(2026, 11, 6, 10, tzinfo=timezone.utc), + ) + snapshot["source_summary"]["workspace_inventory_gap"] = gap + + with pytest.raises(ValueError, match="workspace inventory gap"): + validate_truth_snapshot(_snapshot_from_payload(snapshot)) + + +def test_invalid_workspace_inventory_comparison_is_rejected(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + _make_project(workspace, "Alpha") + snapshot = _publish( + workspace, + tmp_path / "output", + datetime(2026, 11, 6, 10, tzinfo=timezone.utc), ) + snapshot["source_summary"]["workspace_inventory_comparison"] = { + "state": "complete", + "reason": "unknown", + } + + with pytest.raises(ValueError, match="workspace inventory comparison"): + validate_truth_snapshot(_snapshot_from_payload(snapshot)) + + +def test_workspace_path_removal_requires_a_confirmed_missing_path( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "Project").mkdir() + + assert _workspace_path_is_directory(workspace, "Project") + assert _workspace_path_is_directory(workspace, "../outside") + assert _workspace_path_is_directory(tmp_path / "unavailable", "Project") + + with patch.object(Path, "stat", side_effect=PermissionError("denied")): + assert _workspace_path_is_directory(workspace, "Project") + + (workspace / "Project").rmdir() + assert not _workspace_path_is_directory(workspace, "Project") + + +def test_workspace_path_rekey_at_same_path_resolves_prior_key(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + project_path = workspace / "Project" + project_path.mkdir(parents=True) + latest = tmp_path / "portfolio-truth-latest.json" + latest.write_text( + json.dumps( + { + "workspace_root": str(workspace), + "generated_at": "2026-11-06T10:00:00+00:00", + "projects": [ + { + "identity": { + "project_key": "repo:old-name", + "path": "Project", + } + } + ], + } + ), + encoding="utf-8", + ) + + assert _workspace_inventory_gap( + latest, + workspace_root=workspace, + current_project_keys={"repo:new-name"}, + current_project_paths={"Project"}, + ) == (None, None) From 5f0f26fde73bdafc0327fcd88781b5c3d21116c4 Mon Sep 17 00:00:00 2001 From: saagpatel <269905221+saagpatel@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:37:37 -0800 Subject: [PATCH 4/6] Clarify first-run inventory comparison state --- .../portfolio_truth_publish.py | 31 +++++++++++--- .../portfolio_truth_validate.py | 18 ++++++-- tests/test_workspace_inventory_gap.py | 41 +++++++++++++++++++ 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/src/github_repo_auditor/portfolio_truth_publish.py b/src/github_repo_auditor/portfolio_truth_publish.py index b24c51a9..641a23ce 100644 --- a/src/github_repo_auditor/portfolio_truth_publish.py +++ b/src/github_repo_auditor/portfolio_truth_publish.py @@ -35,6 +35,7 @@ render_registry_markdown, ) from github_repo_auditor.portfolio_truth_lineage import resolve_notion_origin +from github_repo_auditor.portfolio_truth_sources import workspace_exclusion_reason from github_repo_auditor.portfolio_truth_types import truth_latest_path from github_repo_auditor.producer_preflight import ( ProducerEvidence, @@ -357,7 +358,7 @@ def _workspace_inventory_gap( try: prior = json.loads(latest_path.read_text(encoding="utf-8")) except FileNotFoundError: - return None, None + return None, "prior_snapshot_missing" except (OSError, UnicodeError, json.JSONDecodeError): return None, "prior_snapshot_unreadable" if not isinstance(prior, dict): @@ -457,6 +458,7 @@ def _workspace_inventory_gap( key not in prior_paths or ( prior_paths[key] not in current_project_paths + and not _workspace_path_is_excluded(prior_paths[key]) and _workspace_path_is_directory(workspace_root, prior_paths[key]) ) ) @@ -487,18 +489,33 @@ def _workspace_path_is_directory(workspace_root: Path, relative_path: str) -> bo return True if not stat.S_ISDIR(root_mode): return True + entry = root / path + try: + entry.stat() + except (FileNotFoundError, NotADirectoryError): + return False + except (OSError, RuntimeError, ValueError): + return True try: - candidate = (root / path).resolve() + candidate = entry.resolve() if not candidate.is_relative_to(root): return True mode = candidate.stat().st_mode - except (FileNotFoundError, NotADirectoryError): - return False except (OSError, RuntimeError, ValueError): return True return stat.S_ISDIR(mode) +def _workspace_path_is_excluded(relative_path: str) -> bool: + """Return true only for a path component explicitly excluded by discovery.""" + path = Path(relative_path) + return ( + not path.is_absolute() + and ".." not in path.parts + and any(workspace_exclusion_reason(part) for part in path.parts) + ) + + def publish_portfolio_truth( *, workspace_root: Path, @@ -743,7 +760,11 @@ def _publish_portfolio_truth_locked( ) if inventory_gap is not None: build_result.snapshot.source_summary["workspace_inventory_gap"] = inventory_gap - if inventory_comparison_unavailable is not None: + if inventory_comparison_unavailable == "prior_snapshot_missing": + build_result.snapshot.source_summary["workspace_inventory_comparison"] = { + "state": "first_run", + } + elif inventory_comparison_unavailable is not None: build_result.snapshot.source_summary["workspace_inventory_comparison"] = { "state": "unavailable", "reason": inventory_comparison_unavailable, diff --git a/src/github_repo_auditor/portfolio_truth_validate.py b/src/github_repo_auditor/portfolio_truth_validate.py index 7ec023ab..7fa3082a 100644 --- a/src/github_repo_auditor/portfolio_truth_validate.py +++ b/src/github_repo_auditor/portfolio_truth_validate.py @@ -593,9 +593,21 @@ def _validate_snapshot_metadata( } if workspace_inventory_comparison is not None and ( not isinstance(workspace_inventory_comparison, dict) - or set(workspace_inventory_comparison) != {"state", "reason"} - or workspace_inventory_comparison.get("state") != "unavailable" - or workspace_inventory_comparison.get("reason") not in valid_comparison_reasons + or ( + set(workspace_inventory_comparison) == {"state"} + and workspace_inventory_comparison.get("state") != "first_run" + ) + or ( + set(workspace_inventory_comparison) != {"state"} + and set(workspace_inventory_comparison) != {"state", "reason"} + ) + or ( + workspace_inventory_comparison.get("state") == "unavailable" + and workspace_inventory_comparison.get("reason") + not in valid_comparison_reasons + ) + or workspace_inventory_comparison.get("state") + not in {"first_run", "unavailable"} ): raise ValueError("PortfolioTruth workspace inventory comparison is malformed.") expected_summary = build_source_summary( diff --git a/tests/test_workspace_inventory_gap.py b/tests/test_workspace_inventory_gap.py index 46d63edb..689dd4ae 100644 --- a/tests/test_workspace_inventory_gap.py +++ b/tests/test_workspace_inventory_gap.py @@ -52,6 +52,9 @@ def test_workspace_inventory_gap_tracks_disappearance_recovery_and_root_change( first = _publish(workspace, output, first_at) assert "workspace_inventory_gap" not in first["source_summary"] + assert first["source_summary"]["workspace_inventory_comparison"] == { + "state": "first_run", + } beta_key = next( project["identity"]["project_key"] for project in first["projects"] @@ -106,6 +109,10 @@ def omit_beta_and_gamma(path: Path) -> bool: _make_project(other_workspace, "Alpha") different_root = _publish(other_workspace, output, first_at.replace(minute=5)) assert "workspace_inventory_gap" not in different_root["source_summary"] + assert different_root["source_summary"]["workspace_inventory_comparison"] == { + "state": "unavailable", + "reason": "workspace_root_changed", + } def test_workspace_inventory_gap_excludes_supplementary_keys_and_ignores_bad_prior( @@ -270,6 +277,9 @@ def test_workspace_path_removal_requires_a_confirmed_missing_path( (workspace / "Project").rmdir() assert not _workspace_path_is_directory(workspace, "Project") + (workspace / "DanglingProject").symlink_to(tmp_path / "removed-target") + assert not _workspace_path_is_directory(workspace, "DanglingProject") + def test_workspace_path_rekey_at_same_path_resolves_prior_key(tmp_path: Path) -> None: workspace = tmp_path / "workspace" @@ -300,3 +310,34 @@ def test_workspace_path_rekey_at_same_path_resolves_prior_key(tmp_path: Path) -> current_project_keys={"repo:new-name"}, current_project_paths={"Project"}, ) == (None, None) + + +def test_workspace_path_excluded_by_current_policy_resolves_gap(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + excluded_path = workspace / "smoke-export" + excluded_path.mkdir(parents=True) + latest = tmp_path / "portfolio-truth-latest.json" + latest.write_text( + json.dumps( + { + "workspace_root": str(workspace), + "generated_at": "2026-11-06T10:00:00+00:00", + "projects": [ + { + "identity": { + "project_key": "repo:old-smoke-project", + "path": "smoke-export", + } + } + ], + } + ), + encoding="utf-8", + ) + + assert _workspace_inventory_gap( + latest, + workspace_root=workspace, + current_project_keys=set(), + current_project_paths=set(), + ) == (None, None) From 36dab21e8cb43ebb3595ae2e90dcd99cfd37d1f0 Mon Sep 17 00:00:00 2001 From: saagpatel <269905221+saagpatel@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:38:41 -0800 Subject: [PATCH 5/6] Keep out-of-root inventory paths uncertain --- src/github_repo_auditor/portfolio_truth_publish.py | 11 +++-------- tests/test_workspace_inventory_gap.py | 7 ++++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/github_repo_auditor/portfolio_truth_publish.py b/src/github_repo_auditor/portfolio_truth_publish.py index 641a23ce..6bec681a 100644 --- a/src/github_repo_auditor/portfolio_truth_publish.py +++ b/src/github_repo_auditor/portfolio_truth_publish.py @@ -489,18 +489,13 @@ def _workspace_path_is_directory(workspace_root: Path, relative_path: str) -> bo return True if not stat.S_ISDIR(root_mode): return True - entry = root / path try: - entry.stat() - except (FileNotFoundError, NotADirectoryError): - return False - except (OSError, RuntimeError, ValueError): - return True - try: - candidate = entry.resolve() + candidate = (root / path).resolve() if not candidate.is_relative_to(root): return True mode = candidate.stat().st_mode + except (FileNotFoundError, NotADirectoryError): + return False except (OSError, RuntimeError, ValueError): return True return stat.S_ISDIR(mode) diff --git a/tests/test_workspace_inventory_gap.py b/tests/test_workspace_inventory_gap.py index 689dd4ae..85105551 100644 --- a/tests/test_workspace_inventory_gap.py +++ b/tests/test_workspace_inventory_gap.py @@ -277,9 +277,14 @@ def test_workspace_path_removal_requires_a_confirmed_missing_path( (workspace / "Project").rmdir() assert not _workspace_path_is_directory(workspace, "Project") - (workspace / "DanglingProject").symlink_to(tmp_path / "removed-target") + (workspace / "DanglingProject").symlink_to(workspace / "removed-target") assert not _workspace_path_is_directory(workspace, "DanglingProject") + (workspace / "OutsideDanglingProject").symlink_to( + tmp_path / "outside" / "removed-target" + ) + assert _workspace_path_is_directory(workspace, "OutsideDanglingProject") + def test_workspace_path_rekey_at_same_path_resolves_prior_key(tmp_path: Path) -> None: workspace = tmp_path / "workspace" From 25f5b0ac6307a4fbc1ae447434fe0a06a4b0c34a Mon Sep 17 00:00:00 2001 From: saagpatel <269905221+saagpatel@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:49:25 -0800 Subject: [PATCH 6/6] Resolve inventory gaps from checkout evidence --- .../portfolio_truth_publish.py | 53 ++++-- .../portfolio_truth_sources.py | 25 +++ .../portfolio_truth_validate.py | 36 ++-- tests/test_portfolio_truth.py | 10 ++ tests/test_workspace_inventory_gap.py | 155 +++++++++++++++++- 5 files changed, 240 insertions(+), 39 deletions(-) diff --git a/src/github_repo_auditor/portfolio_truth_publish.py b/src/github_repo_auditor/portfolio_truth_publish.py index 6bec681a..bdebd19e 100644 --- a/src/github_repo_auditor/portfolio_truth_publish.py +++ b/src/github_repo_auditor/portfolio_truth_publish.py @@ -35,7 +35,10 @@ render_registry_markdown, ) from github_repo_auditor.portfolio_truth_lineage import resolve_notion_origin -from github_repo_auditor.portfolio_truth_sources import workspace_exclusion_reason +from github_repo_auditor.portfolio_truth_sources import ( + checkout_collision_paths, + workspace_exclusion_reason, +) from github_repo_auditor.portfolio_truth_types import truth_latest_path from github_repo_auditor.producer_preflight import ( ProducerEvidence, @@ -351,7 +354,9 @@ def _workspace_inventory_gap( Missing keys remain gaps while their recorded paths still exist in the workspace. A missing path or a path now represented by a different current key is treated as a confirmed removal or rename. Unreadable paths and paths - outside the workspace retain uncertainty and stay in the gap. The baseline + outside the workspace retain uncertainty and stay in the gap. Existing paths + that stop qualifying as projects stay uncertain unless discovery's explicit + exclusion rules or collision evidence account for them. The baseline timestamp marks the start of the continuous unresolved gap chain, so later missing keys can share that earlier baseline until the chain fully clears. """ @@ -458,7 +463,7 @@ def _workspace_inventory_gap( key not in prior_paths or ( prior_paths[key] not in current_project_paths - and not _workspace_path_is_excluded(prior_paths[key]) + and not _workspace_path_is_excluded(workspace_root, prior_paths[key]) and _workspace_path_is_directory(workspace_root, prior_paths[key]) ) ) @@ -501,14 +506,21 @@ def _workspace_path_is_directory(workspace_root: Path, relative_path: str) -> bo return stat.S_ISDIR(mode) -def _workspace_path_is_excluded(relative_path: str) -> bool: - """Return true only for a path component explicitly excluded by discovery.""" +def _workspace_path_is_excluded(workspace_root: Path, relative_path: str) -> bool: + """Return true only for a path explicitly excluded by workspace discovery.""" path = Path(relative_path) - return ( - not path.is_absolute() - and ".." not in path.parts - and any(workspace_exclusion_reason(part) for part in path.parts) - ) + if path.is_absolute() or ".." in path.parts or not path.parts: + return False + if any(workspace_exclusion_reason(part) for part in path.parts): + return True + try: + root = workspace_root.expanduser().resolve() + entry = root / path + if not stat.S_ISLNK(entry.lstat().st_mode): + return False + return entry.resolve().is_relative_to(root) + except (OSError, RuntimeError, ValueError): + return False def publish_portfolio_truth( @@ -743,15 +755,26 @@ def _publish_portfolio_truth_locked( for project in build_result.snapshot.projects if not project.identity.project_key.startswith("supp:") } + current_project_paths = { + project.identity.path + for project in build_result.snapshot.projects + if not project.identity.project_key.startswith("supp:") + } + collision_summary = build_result.snapshot.source_summary.get( + "checkout_collisions", {} + ) + current_project_paths.update( + checkout_collision_paths( + collision_summary.get("groups") + if isinstance(collision_summary, dict) + else None + ) + ) inventory_gap, inventory_comparison_unavailable = _workspace_inventory_gap( latest_path, workspace_root=workspace_root, current_project_keys=current_project_keys, - current_project_paths={ - project.identity.path - for project in build_result.snapshot.projects - if not project.identity.project_key.startswith("supp:") - }, + current_project_paths=current_project_paths, ) if inventory_gap is not None: build_result.snapshot.source_summary["workspace_inventory_gap"] = inventory_gap diff --git a/src/github_repo_auditor/portfolio_truth_sources.py b/src/github_repo_auditor/portfolio_truth_sources.py index bc471e9f..4a027892 100644 --- a/src/github_repo_auditor/portfolio_truth_sources.py +++ b/src/github_repo_auditor/portfolio_truth_sources.py @@ -414,6 +414,31 @@ def checkout_collision_summary( } +def checkout_collision_paths(groups: object) -> set[str]: + """Return safe workspace-relative paths observed in collision evidence.""" + if not isinstance(groups, list): + return set() + paths: set[str] = set() + for group in groups: + if not isinstance(group, dict): + continue + checkouts = group.get("checkouts") + if not isinstance(checkouts, list): + continue + for checkout in checkouts: + if not isinstance(checkout, dict): + continue + path = checkout.get("path") + if ( + isinstance(path, str) + and path + and not Path(path).is_absolute() + and ".." not in Path(path).parts + ): + paths.add(path) + return paths + + def _checkout_representative( group: list[dict[str, Any]], origin_key: str ) -> dict[str, Any]: diff --git a/src/github_repo_auditor/portfolio_truth_validate.py b/src/github_repo_auditor/portfolio_truth_validate.py index 7fa3082a..7b7ad8e6 100644 --- a/src/github_repo_auditor/portfolio_truth_validate.py +++ b/src/github_repo_auditor/portfolio_truth_validate.py @@ -48,7 +48,10 @@ REQUIRED_PROJECT_PROVENANCE_KEYS, ) from github_repo_auditor.portfolio_truth_render import registry_project_labels -from github_repo_auditor.portfolio_truth_sources import WORKSPACE_EXCLUSION_REASONS +from github_repo_auditor.portfolio_truth_sources import ( + WORKSPACE_EXCLUSION_REASONS, + checkout_collision_paths, +) from github_repo_auditor.portfolio_truth_types import ( CHECKOUT_COLLISION_SUMMARY_SCHEMA_VERSION, DERIVATION_POLICY_VERSION, @@ -558,6 +561,8 @@ def _validate_snapshot_metadata( for project in snapshot.projects if not project.identity.project_key.startswith("supp:") } + collision_groups = summary["checkout_collisions"]["groups"] + current_project_paths.update(checkout_collision_paths(collision_groups)) if ( not isinstance(missing_project_keys, list) or any(not isinstance(key, str) or not key for key in missing_project_keys) @@ -591,24 +596,17 @@ def _validate_snapshot_metadata( "prior_project_key_malformed", "prior_inventory_gap_malformed", } - if workspace_inventory_comparison is not None and ( - not isinstance(workspace_inventory_comparison, dict) - or ( - set(workspace_inventory_comparison) == {"state"} - and workspace_inventory_comparison.get("state") != "first_run" - ) - or ( - set(workspace_inventory_comparison) != {"state"} - and set(workspace_inventory_comparison) != {"state", "reason"} - ) - or ( - workspace_inventory_comparison.get("state") == "unavailable" - and workspace_inventory_comparison.get("reason") - not in valid_comparison_reasons - ) - or workspace_inventory_comparison.get("state") - not in {"first_run", "unavailable"} - ): + comparison_valid = workspace_inventory_comparison is None + if isinstance(workspace_inventory_comparison, dict): + if workspace_inventory_comparison.get("state") == "first_run": + comparison_valid = set(workspace_inventory_comparison) == {"state"} + elif workspace_inventory_comparison.get("state") == "unavailable": + comparison_valid = ( + set(workspace_inventory_comparison) == {"state", "reason"} + and workspace_inventory_comparison.get("reason") + in valid_comparison_reasons + ) + if not comparison_valid: raise ValueError("PortfolioTruth workspace inventory comparison is malformed.") expected_summary = build_source_summary( workspace_root=snapshot.workspace_root, diff --git a/tests/test_portfolio_truth.py b/tests/test_portfolio_truth.py index 11dbc87f..56c89bdd 100644 --- a/tests/test_portfolio_truth.py +++ b/tests/test_portfolio_truth.py @@ -59,6 +59,7 @@ ) from github_repo_auditor.portfolio_truth_validate import ( _path_identity, + _snapshot_from_payload, _same_repository_path, canonicalize_prior_security_truth_payload, validate_portfolio_report_markdown, @@ -1142,6 +1143,15 @@ def test_checkout_collision_flows_through_truth_validation_and_report( assert widget_projects[0].repository_state["checkout_authority"] == group validate_truth_snapshot(result.snapshot) + payload = result.snapshot.to_dict() + payload["source_summary"]["workspace_inventory_gap"] = { + "baseline_generated_at": payload["generated_at"], + "missing_project_keys": ["local:archive-widget"], + "missing_project_paths": {"local:archive-widget": "Archive/Widget"}, + } + with pytest.raises(ValueError, match="workspace inventory gap"): + validate_truth_snapshot(_snapshot_from_payload(payload)) + markdown = render_portfolio_report_markdown(result.snapshot, "output/x.json") assert "## Checkout Authority" in markdown assert "`owner/Widget`" in markdown diff --git a/tests/test_workspace_inventory_gap.py b/tests/test_workspace_inventory_gap.py index 85105551..3ca7fbd4 100644 --- a/tests/test_workspace_inventory_gap.py +++ b/tests/test_workspace_inventory_gap.py @@ -2,6 +2,7 @@ import json import shutil +import subprocess from datetime import datetime, timezone from pathlib import Path from unittest.mock import patch @@ -9,10 +10,12 @@ import pytest from github_repo_auditor.portfolio_truth_publish import ( + _workspace_path_is_excluded, _workspace_path_is_directory, _workspace_inventory_gap, publish_portfolio_truth, ) +from github_repo_auditor.portfolio_truth_sources import checkout_collision_paths from github_repo_auditor.portfolio_truth_validate import ( _snapshot_from_payload, validate_truth_snapshot, @@ -242,7 +245,16 @@ def test_invalid_workspace_inventory_gap_is_rejected( validate_truth_snapshot(_snapshot_from_payload(snapshot)) -def test_invalid_workspace_inventory_comparison_is_rejected(tmp_path: Path) -> None: +@pytest.mark.parametrize( + "comparison", + [ + {"state": "complete", "reason": "unknown"}, + {"state": "first_run", "reason": "arbitrary"}, + ], +) +def test_invalid_workspace_inventory_comparison_is_rejected( + tmp_path: Path, comparison: dict[str, str] +) -> None: workspace = tmp_path / "workspace" workspace.mkdir() _make_project(workspace, "Alpha") @@ -251,10 +263,7 @@ def test_invalid_workspace_inventory_comparison_is_rejected(tmp_path: Path) -> N tmp_path / "output", datetime(2026, 11, 6, 10, tzinfo=timezone.utc), ) - snapshot["source_summary"]["workspace_inventory_comparison"] = { - "state": "complete", - "reason": "unknown", - } + snapshot["source_summary"]["workspace_inventory_comparison"] = comparison with pytest.raises(ValueError, match="workspace inventory comparison"): validate_truth_snapshot(_snapshot_from_payload(snapshot)) @@ -317,6 +326,94 @@ def test_workspace_path_rekey_at_same_path_resolves_prior_key(tmp_path: Path) -> ) == (None, None) +def test_added_same_origin_checkout_resolves_old_identity_from_collision_evidence( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + (workspace / "foo-old").mkdir(parents=True) + (workspace / "foo").mkdir() + latest = tmp_path / "portfolio-truth-latest.json" + latest.write_text( + json.dumps( + { + "workspace_root": str(workspace), + "generated_at": "2026-11-06T10:00:00+00:00", + "projects": [ + { + "identity": { + "project_key": "local:foo-old", + "path": "foo-old", + } + } + ], + } + ), + encoding="utf-8", + ) + collisions = [ + { + "origin": "owner/foo", + "checkouts": [{"path": "foo"}, {"path": "foo-old"}], + } + ] + + current_paths = {"foo"} | checkout_collision_paths(collisions) + assert _workspace_inventory_gap( + latest, + workspace_root=workspace, + current_project_keys={"repo:owner/foo"}, + current_project_paths=current_paths, + ) == (None, None) + + +def test_adding_same_origin_clone_does_not_create_a_permanent_inventory_gap( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + old_checkout = workspace / "foo-old" + old_checkout.mkdir(parents=True) + (old_checkout / "README.md").write_text("# foo-old\n") + output = tmp_path / "output" + generated_at = datetime(2026, 11, 6, 10, tzinfo=timezone.utc) + + baseline = _publish(workspace, output, generated_at) + old_key = baseline["projects"][0]["identity"]["project_key"] + assert old_key == "foo-old" + + for checkout in (old_checkout, workspace / "foo"): + if checkout != old_checkout: + checkout.mkdir() + (checkout / "README.md").write_text("# foo\n") + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=checkout, check=True) + subprocess.run( + ["git", "remote", "add", "origin", "git@github.com:owner/foo.git"], + cwd=checkout, + check=True, + ) + subprocess.run(["git", "add", "README.md"], cwd=checkout, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "commit", + "-q", + "-m", + "fixture", + ], + cwd=checkout, + check=True, + ) + + current = _publish(workspace, output, generated_at.replace(minute=1)) + + assert "workspace_inventory_gap" not in current["source_summary"] + collision_groups = current["source_summary"]["checkout_collisions"]["groups"] + assert checkout_collision_paths(collision_groups) == {"foo", "foo-old"} + + def test_workspace_path_excluded_by_current_policy_resolves_gap(tmp_path: Path) -> None: workspace = tmp_path / "workspace" excluded_path = workspace / "smoke-export" @@ -346,3 +443,51 @@ def test_workspace_path_excluded_by_current_policy_resolves_gap(tmp_path: Path) current_project_keys=set(), current_project_paths=set(), ) == (None, None) + + +def test_existing_unclassified_directory_keeps_gap_uncertain(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + (workspace / "FormerProject").mkdir(parents=True) + latest = tmp_path / "portfolio-truth-latest.json" + latest.write_text( + json.dumps( + { + "workspace_root": str(workspace), + "generated_at": "2026-11-06T10:00:00+00:00", + "projects": [ + { + "identity": { + "project_key": "local:FormerProject", + "path": "FormerProject", + } + } + ], + } + ), + encoding="utf-8", + ) + + gap, comparison = _workspace_inventory_gap( + latest, + workspace_root=workspace, + current_project_keys=set(), + current_project_paths=set(), + ) + assert comparison is None + assert gap is not None + assert gap["missing_project_keys"] == ["local:FormerProject"] + + +def test_in_root_symlink_is_explicitly_excluded_but_outside_symlink_stays_uncertain( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + (workspace / "Target").mkdir(parents=True) + (workspace / "LinkedProject").symlink_to(workspace / "Target") + outside = tmp_path / "outside" + outside.mkdir() + (workspace / "OutsideProject").symlink_to(outside) + + assert _workspace_path_is_excluded(workspace, "LinkedProject") + assert not _workspace_path_is_excluded(workspace, "OutsideProject") + assert _workspace_path_is_directory(workspace, "OutsideProject")