From 18b76597712e9e9f57d0cfaf6e1739853ae8c0d7 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:44:05 +0200 Subject: [PATCH 01/10] Select explicit temporary gov exclusion consistently with coverage metadata --- scripts/build_nsdb.py | 57 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/scripts/build_nsdb.py b/scripts/build_nsdb.py index 47cbdf9..ddbd4cc 100644 --- a/scripts/build_nsdb.py +++ b/scripts/build_nsdb.py @@ -284,6 +284,44 @@ def infer_apex_from_filename(filename: str) -> str | None: return None +def select_zone_inputs( + zones_dir: Path, excluded_zones: tuple[str, ...] = () +) -> tuple[list[Path], tuple[str, ...]]: + """Apply exact, explicit exclusions to the unchanged CZDS download manifest. + + All manifest entries must still exist; only a known temporary exception + can be omitted from parsing. Preflight and aggregation call this same + selector to prevent disagreement about coverage. + """ + requested = set(excluded_zones) + if requested - {"gov"}: + raise ValueError("unsupported CZDS zone exclusion") + manifest = json.loads((zones_dir / "manifest.json").read_text(encoding="utf-8")) + entries = manifest.get("files", []) + if not isinstance(entries, list) or not entries: + raise ValueError("CZDS manifest contains no files") + selected = [] + found = set() + for entry in entries: + if not isinstance(entry, dict): + raise ValueError("malformed CZDS manifest entry") + filename = entry.get("filename") + if not isinstance(filename, str) or not filename or Path(filename).name != filename: + raise ValueError("CZDS manifest filename must be a basename") + path = zones_dir / filename + if path.is_symlink() or not path.is_file(): + raise ValueError("CZDS manifest references missing or nonregular file") + if infer_apex_from_filename(filename) in requested: + found.add(infer_apex_from_filename(filename)) + else: + selected.append(path) + if requested != found: + raise ValueError("requested CZDS exclusion was not found in manifest") + if not selected: + raise ValueError("no CZDS zones remain after exclusion") + return sorted(selected), tuple(sorted(found)) + + def process_zone_stream( stream, label: str, @@ -493,19 +531,12 @@ def main(): p.add_argument("--private-counts", required=True, type=Path) p.add_argument("--coverage", required=True, type=Path) p.add_argument("--counts", required=True, type=Path) + p.add_argument("--exclude-zone", choices=("gov",), action="append", default=[]) args = p.parse_args() - manifest_path = args.zones_dir / "manifest.json" - if not manifest_path.exists(): - raise SystemExit(f"missing {manifest_path}") - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - files = [args.zones_dir / item["filename"] for item in manifest.get("files", [])] - if not files: - raise SystemExit("zone manifest is empty") - - for path in files: - if not path.is_file(): - raise SystemExit(f"zone file missing: {path}") + files, excluded = select_zone_inputs(args.zones_dir, tuple(args.exclude_zone)) + coverage_complete = "false" if excluded else "true" + excluded_text = ",".join(excluded) if excluded else "none" args.private_counts.parent.mkdir(parents=True, exist_ok=True) args.coverage.parent.mkdir(parents=True, exist_ok=True) @@ -541,6 +572,8 @@ def main(): with open(args.coverage, "w", encoding="utf-8", newline="\n") as fh: fh.write("# format=zone|soa_serial|records|delegated_names|delegation_edges\n") + fh.write(f"# coverage_complete={coverage_complete}\n") + fh.write(f"# excluded_zones={excluded_text}\n") for row in coverage: fh.write( f"{row['zone']}|{row['soa_serial']}|{row['records']}|" @@ -548,6 +581,8 @@ def main(): ) with open(args.counts, "w", encoding="utf-8", newline="\n") as fh: + fh.write(f"coverage_complete={coverage_complete}\n") + fh.write(f"excluded_zones={excluded_text}\n") fh.write(f"zones={len(coverage)}\n") fh.write(f"delegated_names={delegated_names}\n") fh.write(f"delegation_edges={delegation_edges}\n") From 17deeb9535621f4558057c1d94fe461bf4d844b5 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:44:19 +0200 Subject: [PATCH 02/10] Share manifest exclusion between preflight and aggregation --- scripts/preflight_czds.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/scripts/preflight_czds.py b/scripts/preflight_czds.py index 6704e24..0602294 100644 --- a/scripts/preflight_czds.py +++ b/scripts/preflight_czds.py @@ -17,6 +17,7 @@ load_special_zones, open_text_stream, process_input, + select_zone_inputs, ) @@ -40,21 +41,17 @@ def has_nonleading_soa(path: Path) -> bool: return fields[3].lower() != "soa" or fields[2].lower() != "in" -def preflight(zones_dir: Path, special_zones_file: Path) -> int: - manifest = json.loads((zones_dir / "manifest.json").read_text(encoding="utf-8")) - entries = manifest.get("files", []) - if not entries: - raise ValueError("CZDS manifest contains no files") +def preflight( + zones_dir: Path, special_zones_file: Path, excluded_zones: tuple[str, ...] = () +) -> int: + files, excluded = select_zone_inputs(zones_dir, excluded_zones) + if excluded: + print("Temporary CZDS exclusion: gov (partial coverage)", flush=True) special_zones = load_special_zones(special_zones_file) completed = 0 - for entry in entries: - filename = entry["filename"] - if Path(filename).name != filename: - raise ValueError("manifest contains a non-basename path") - path = zones_dir / filename - if not path.is_file(): - raise ValueError(f"missing zone file: {filename}") + for path in files: + filename = path.name if not has_nonleading_soa(path): continue if infer_apex_from_filename(filename) is None: @@ -81,10 +78,11 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--zones-dir", type=Path, required=True) parser.add_argument("--special-zones", type=Path, required=True) + parser.add_argument("--exclude-zone", choices=("gov",), action="append", default=[]) args = parser.parse_args() try: - preflight(args.zones_dir, args.special_zones) + preflight(args.zones_dir, args.special_zones, tuple(args.exclude_zone)) except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: raise SystemExit( f"NSDB preflight failed (reason={public_validation_code(exc)})" From 731baf34863cfab0d59ec20f2bbbe749618de764 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:45:23 +0200 Subject: [PATCH 03/10] Propagate partial coverage into snapshots and change artifacts --- scripts/publish_nsdb.py | 68 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/scripts/publish_nsdb.py b/scripts/publish_nsdb.py index 46d03b3..3ca3be9 100644 --- a/scripts/publish_nsdb.py +++ b/scripts/publish_nsdb.py @@ -78,10 +78,12 @@ def write_gzip_copy(path: Path): shutil.copyfileobj(src, dst) -def write_public_snapshot(path: Path, counts: dict[str, int], k: int, current_tag: str): +def write_public_snapshot(path: Path, counts: dict[str, int], k: int, current_tag: str, coverage_complete: str = "true", excluded_zones: str = "none"): with open(path, "w", encoding="utf-8", newline="\n") as fh: fh.write(f"# k={k}\n") fh.write(f"# snapshot={current_tag}\n") + fh.write(f"# coverage_complete={coverage_complete}\n") + fh.write(f"# excluded_zones={excluded_zones}\n") fh.write("# format=nameserver|delegations\n") for name in sorted(counts): fh.write(f"{name}|{counts[name]}\n") @@ -115,6 +117,33 @@ def load_coverage(path: Path | None) -> set[str]: return zones +def coverage_status(path: Path, *, required: bool = True) -> tuple[str, str]: + """Read explicit, public coverage metadata from the coverage artifact.""" + if not path.exists(): + if required: + raise ValueError("coverage file is missing") + return "true", "none" + metadata = {} + with open(path, "r", encoding="utf-8") as source: + for line in source: + if not line.startswith("# "): + continue + item = line[2:].strip() + if "=" in item: + key, value = item.split("=", 1) + if key in ("coverage_complete", "excluded_zones"): + if key in metadata: + raise ValueError("duplicate coverage status field") + metadata[key] = value + if not metadata and not required: + return "true", "none" + complete = metadata.get("coverage_complete") + excluded = metadata.get("excluded_zones") + if (complete, excluded) not in (("true", "none"), ("false", "gov")): + raise ValueError("invalid or inconsistent coverage status metadata") + return complete, excluded + + def bool_text(value: bool) -> str: return "true" if value else "false" @@ -137,6 +166,17 @@ def main(): if args.k < 1: raise SystemExit("k must be >= 1") + current_coverage_status, current_excluded = coverage_status(args.current_coverage) + current_coverage = load_coverage(args.current_coverage) + current_full = load_key_values(args.current_full_counts) + if ( + current_full.get("coverage_complete") != current_coverage_status + or current_full.get("excluded_zones") != current_excluded + or current_full.get("zones") != str(len(current_coverage)) + or (current_excluded != "none" and current_excluded in current_coverage) + ): + raise SystemExit("current private counts and CZDS coverage status disagree") + args.out.mkdir(parents=True, exist_ok=True) current_visible = load_visible(args.current_private, args.k) @@ -175,7 +215,7 @@ def main(): raise SystemExit("previous tag supplied without previous public snapshot") current_snapshot = args.out / "nameservers.txt" - write_public_snapshot(current_snapshot, current_visible, args.k, args.current_tag) + write_public_snapshot(current_snapshot, current_visible, args.k, args.current_tag, current_coverage_status, current_excluded) has_previous = args.previous_public is not None and args.previous_public.exists() if has_previous: @@ -192,9 +232,20 @@ def main(): k_changed = previous_k is not None and previous_k != args.k previous_coverage = load_coverage(args.previous_coverage) - current_coverage = load_coverage(args.current_coverage) - coverage_changed = bool(previous_coverage) and previous_coverage != current_coverage - comparison_break = coverage_changed or comparison_mode == "k-lowered" + previous_status, previous_excluded = ( + coverage_status(args.previous_coverage, required=False) + if args.previous_coverage is not None else ("true", "none") + ) + coverage_changed = bool(previous_coverage) and ( + previous_coverage != current_coverage + or (previous_status, previous_excluded) != (current_coverage_status, current_excluded) + ) + # Partial releases are explicitly not comparable to a complete dataset. + comparison_break = ( + coverage_changed or comparison_mode == "k-lowered" + or current_coverage_status == "false" + or (has_previous and previous_status == "false") + ) diff_path = args.out / "delegation-changes.txt" with open(diff_path, "w", encoding="utf-8", newline="\n") as fh: @@ -202,6 +253,8 @@ def main(): fh.write(f"# previous_k={previous_k_text}\n") fh.write(f"# previous_visibility_k={previous_visibility_k if has_previous else 'none'}\n") fh.write(f"# current_visibility_k={args.k}\n") + fh.write(f"# coverage_complete={current_coverage_status}\n") + fh.write(f"# excluded_zones={current_excluded}\n") fh.write(f"# comparison_mode={comparison_mode}\n") fh.write(f"# k_changed={bool_text(k_changed)}\n") fh.write(f"# coverage_changed={bool_text(coverage_changed)}\n") @@ -220,6 +273,8 @@ def main(): fh.write(f"# previous_k={previous_k_text}\n") fh.write(f"# previous_visibility_k={previous_visibility_k if has_previous else 'none'}\n") fh.write(f"# current_visibility_k={args.k}\n") + fh.write(f"# coverage_complete={current_coverage_status}\n") + fh.write(f"# excluded_zones={current_excluded}\n") fh.write(f"# comparison_mode={comparison_mode}\n") fh.write(f"# comparison_break={bool_text(comparison_break)}\n") fh.write(f"# previous={args.previous_tag}\n") @@ -231,7 +286,6 @@ def main(): shutil.copyfile(args.current_coverage, args.out / "coverage.txt") - current_full = load_key_values(args.current_full_counts) previous_full = load_key_values(args.previous_full_counts) counts_path = args.out / "counts.txt" with open(counts_path, "w", encoding="utf-8", newline="\n") as fh: @@ -261,6 +315,8 @@ def int_or_none(values, key): fh.write(f"k={args.k}\n") fh.write(f"previous_k={previous_k_text}\n") fh.write(f"comparison_mode={comparison_mode}\n") + fh.write(f"coverage_complete={current_coverage_status}\n") + fh.write(f"excluded_zones={current_excluded}\n") fh.write(f"k_changed={bool_text(k_changed)}\n") fh.write(f"coverage_changed={bool_text(coverage_changed)}\n") fh.write(f"comparison_break={bool_text(comparison_break)}\n") From 15961f04fea18f4dde064796ffce8518007e0e1f Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:46:00 +0200 Subject: [PATCH 04/10] Temporarily exclude gov from both daily stages and label releases partial --- .github/workflows/daily-release.yml | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily-release.yml b/.github/workflows/daily-release.yml index dd03451..cb0fdd1 100644 --- a/.github/workflows/daily-release.yml +++ b/.github/workflows/daily-release.yml @@ -26,6 +26,7 @@ jobs: ICANN_USERNAME: ${{ secrets.ICANN_USERNAME }} ICANN_PASSWORD: ${{ secrets.ICANN_PASSWORD }} NSDB_STATE_DIR: ${{ vars.NSDB_STATE_DIR || '/var/lib/nsdb' }} + NSDB_TEMP_EXCLUDE_ZONE: gov steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -69,7 +70,8 @@ jobs: run: | python scripts/preflight_czds.py \ --zones-dir "$NSDB_STATE_DIR/zones/$DAY" \ - --special-zones config/locally-served-dns-zones.txt + --special-zones config/locally-served-dns-zones.txt \ + --exclude-zone "$NSDB_TEMP_EXCLUDE_ZONE" - name: Build private exact aggregate env: @@ -78,6 +80,7 @@ jobs: python scripts/build_nsdb.py \ --zones-dir "$NSDB_STATE_DIR/zones/$DAY" \ --special-zones config/locally-served-dns-zones.txt \ + --exclude-zone "$NSDB_TEMP_EXCLUDE_ZONE" \ --private-counts "$NSDB_STATE_DIR/private/$DAY/nameserver-counts.txt" \ --coverage "$NSDB_STATE_DIR/private/$DAY/coverage.txt" \ --counts "$NSDB_STATE_DIR/private/$DAY/counts.txt" @@ -194,10 +197,29 @@ jobs: DAY: ${{ steps.date.outputs.day }} PREVIOUS_TAG: ${{ steps.previous.outputs.tag }} run: | + COVERAGE_STATUS="$(sed -n 's/^coverage_complete=//p' dist/counts.txt)" + EXCLUDED_ZONES="$(sed -n 's/^excluded_zones=//p' dist/counts.txt)" + case "$COVERAGE_STATUS:$EXCLUDED_ZONES" in + true:none) + RELEASE_TITLE="NSDB snapshot $DAY" + COVERAGE_LABEL="Complete approved CZDS manifest" + ;; + false:gov) + RELEASE_TITLE="NSDB snapshot $DAY (PARTIAL: .gov excluded)" + COVERAGE_LABEL="PARTIAL - .gov excluded" + ;; + *) + echo "Invalid release coverage metadata" >&2 + exit 1 + ;; + esac + NOTES="$(cat </dev/null 2>&1; then - gh release edit "$TAG" --notes "$NOTES" + gh release edit "$TAG" --title "$RELEASE_TITLE" --notes "$NOTES" gh release upload "$TAG" dist/* --clobber else gh release create "$TAG" dist/* \ - --title "NSDB snapshot $DAY" \ + --title "$RELEASE_TITLE" \ --notes "$NOTES" fi From 8f72c80e6e7125cb32b1677e34a65b7ed5bf7a63 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:47:24 +0200 Subject: [PATCH 05/10] Test exact temporary gov exclusion, unchanged manifest, and fail-closed parsing --- tests/test_preflight.py | 113 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/tests/test_preflight.py b/tests/test_preflight.py index ff1ba77..71afa67 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -13,7 +13,7 @@ class PreflightTests(unittest.TestCase): @staticmethod - def invoke(directory: pathlib.Path) -> subprocess.CompletedProcess: + def invoke(directory: pathlib.Path, *extra: str) -> subprocess.CompletedProcess: return subprocess.run( [ sys.executable, @@ -22,6 +22,7 @@ def invoke(directory: pathlib.Path) -> subprocess.CompletedProcess: str(directory / "zones"), "--special-zones", str(directory / "special-zones.txt"), + *extra, ], capture_output=True, text=True, @@ -29,7 +30,7 @@ def invoke(directory: pathlib.Path) -> subprocess.CompletedProcess: ) @staticmethod - def invoke_build(directory: pathlib.Path) -> subprocess.CompletedProcess: + def invoke_build(directory: pathlib.Path, *extra: str) -> subprocess.CompletedProcess: return subprocess.run( [ sys.executable, @@ -44,6 +45,7 @@ def invoke_build(directory: pathlib.Path) -> subprocess.CompletedProcess: str(directory / "coverage.txt"), "--counts", str(directory / "counts.txt"), + *extra, ], capture_output=True, text=True, @@ -61,6 +63,113 @@ def setup_fixture(directory: pathlib.Path, filename: str, records: list[str]) -> with gzip.open(zones / filename, "wt", encoding="utf-8") as out: out.write("\n".join(records) + "\n") + @staticmethod + def setup_many(directory: pathlib.Path, by_filename: dict[str, list[str]]) -> None: + zones = directory / "zones" + zones.mkdir() + (directory / "special-zones.txt").write_text("home.arpa\n", encoding="utf-8") + (zones / "manifest.json").write_text( + json.dumps({"files": [{"filename": name} for name in sorted(by_filename)]}), + encoding="utf-8", + ) + for filename, records in by_filename.items(): + with gzip.open(zones / filename, "wt", encoding="utf-8") as out: + out.write("\n".join(records) + "\n") + + def test_temporary_gov_exclusion_is_exact_and_both_stages_match(self): + gov = [ + "gov.\t3600\tIN\tNS\tns.gov.", + "confidential.gov. 3600 IN NS ns.secret.example.", + "gov.\t3600\tIN\tSOA\tns.gov. hostmaster.gov. 123 2 3 4 5", + ] + example = [ + "example.\t3600\tIN\tSOA\tns.example. hostmaster.example. 11 2 3 4 5", + "child.example.\t3600\tIN\tNS\tns1.example.net.", + "example.\t3600\tIN\tSOA\tns.example. hostmaster.example. 11 2 3 4 5", + ] + govuk = [ + "govuk.\t3600\tIN\tSOA\tns.govuk. hostmaster.govuk. 12 2 3 4 5", + "child.govuk.\t3600\tIN\tNS\tns2.example.net.", + "govuk.\t3600\tIN\tSOA\tns.govuk. hostmaster.govuk. 12 2 3 4 5", + ] + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + self.setup_many(root, { + "gov.txt.gz": gov, "example.txt.gz": example, "govuk.txt.gz": govuk + }) + manifest = (root / "zones" / "manifest.json").read_bytes() + original_gov = (root / "zones" / "gov.txt.gz").read_bytes() + extra = ("--exclude-zone", "gov") + for invocation in (self.invoke, self.invoke_build): + failed = invocation(root) + self.assertNotEqual(failed.returncode, 0) + self.assertIn("reason=record_fields_space_delimited", failed.stderr) + self.assertNotIn("confidential.gov.", failed.stdout + failed.stderr) + preflight = self.invoke(root, *extra) + full = self.invoke_build(root, *extra) + self.assertEqual(preflight.returncode, 0, preflight.stderr) + self.assertEqual(full.returncode, 0, full.stderr) + self.assertIn("Temporary CZDS exclusion: gov", preflight.stdout) + coverage = (root / "coverage.txt").read_text(encoding="utf-8") + counts = (root / "counts.txt").read_text(encoding="utf-8") + self.assertIn("# coverage_complete=false", coverage) + self.assertIn("# excluded_zones=gov", coverage) + self.assertIn("coverage_complete=false", counts) + self.assertIn("excluded_zones=gov", counts) + self.assertIn("zones=2", counts) + self.assertIn("example|", coverage) + self.assertIn("govuk|", coverage) + self.assertNotIn("\ngov|", coverage) + self.assertEqual((root / "zones" / "manifest.json").read_bytes(), manifest) + self.assertEqual((root / "zones" / "gov.txt.gz").read_bytes(), original_gov) + private = (root / "private-counts.txt").read_text(encoding="utf-8") + self.assertIn("ns1.example.net|1", private) + self.assertIn("ns2.example.net|1", private) + self.assertNotIn("ns.secret.example", private) + + def test_exclusion_fails_if_requested_zone_missing_or_is_only_zone(self): + valid = [ + "example.\t3600\tIN\tSOA\tns.example. hostmaster.example. 1 2 3 4 5", + "a.example.\t3600\tIN\tNS\tns1.example.net.", + "example.\t3600\tIN\tSOA\tns.example. hostmaster.example. 1 2 3 4 5", + ] + for filenames in ({"example.txt.gz": valid}, { + "gov.txt.gz": [ + "gov.\t3600\tIN\tNS\tns.gov.", + "gov.\t3600\tIN\tSOA\tns.gov. hostmaster.gov. 1 2 3 4 5", + ], + }): + with self.subTest(names=tuple(filenames)), tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + self.setup_many(root, filenames) + for invocation in (self.invoke, self.invoke_build): + failed = invocation(root, "--exclude-zone", "gov") + self.assertNotEqual(failed.returncode, 0) + self.assertFalse((root / "counts.txt").exists()) + self.assertNotIn("ns1.example.net", failed.stdout + failed.stderr) + unsupported = invocation(root, "--exclude-zone", "com") + self.assertNotEqual(unsupported.returncode, 0) + + def test_exclusion_never_hides_other_malformed_zones(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + self.setup_many(root, { + "gov.txt.gz": [ + "gov.\t3600\tIN\tNS\tns.gov.", + "gov.\t3600\tIN\tSOA\tns.gov. hostmaster.gov. 1 2 3 4 5", + ], + "example.txt.gz": [ + "example.\t3600\tIN\tNS\tns.example.", + "private.example. 3600 IN NS ns.secret.example.", + "example.\t3600\tIN\tSOA\tns.example. hostmaster.example. 1 2 3 4 5", + ], + }) + for invocation in (self.invoke, self.invoke_build): + failed = invocation(root, "--exclude-zone", "gov") + self.assertNotEqual(failed.returncode, 0) + self.assertIn("reason=record_fields_space_delimited", failed.stderr) + self.assertNotIn("private.example.", failed.stdout + failed.stderr) + def test_nonleading_soa_is_validated_without_private_output(self): with tempfile.TemporaryDirectory() as d: root = pathlib.Path(d) From dd46e6cce10767e59bd5bad7d42c64dd41080cf6 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:48:05 +0200 Subject: [PATCH 06/10] Test partial public snapshots, explicit coverage break, and metadata mismatch rejection --- tests/test_first_release.py | 98 ++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/tests/test_first_release.py b/tests/test_first_release.py index c77719a..cb8f02c 100644 --- a/tests/test_first_release.py +++ b/tests/test_first_release.py @@ -20,9 +20,9 @@ def test_first_release_is_verified_and_contains_no_private_counts(self): private = root / "private.txt" private.write_text("# format=nameserver|delegations\nns.hidden.example|4\nns.visible.example|5\n") counts = root / "counts.txt" - counts.write_text("zones=1\ndelegation_edges=9\nnameservers_total=2\n") + counts.write_text("zones=1\ncoverage_complete=true\nexcluded_zones=none\ndelegation_edges=9\nnameservers_total=2\n") coverage = root / "coverage.txt" - coverage.write_text("# format=zone|soa_serial|records|delegated_names|delegation_edges\nexample|1|10|6|9\n") + coverage.write_text("# format=zone|soa_serial|records|delegated_names|delegation_edges\n# coverage_complete=true\n# excluded_zones=none\nexample|1|10|6|9\n") out = root / "dist" subprocess.run([ sys.executable, str(ROOT / "scripts" / "publish_nsdb.py"), @@ -46,6 +46,100 @@ def test_first_release_is_verified_and_contains_no_private_counts(self): fh.write("ns.hidden.example|<5|5|NA\n") self.assertNotEqual(subprocess.run(verify, capture_output=True).returncode, 0) + def test_partial_snapshot_is_labeled_and_restoration_breaks_comparison(self): + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + private = root / "private.txt" + private.write_text("# format=nameserver|delegations\nns.visible.example|5\n") + counts = root / "counts.txt" + coverage = root / "coverage.txt" + + def source(complete, excluded, rows, edges, zones): + counts.write_text( + f"zones={zones}\ncoverage_complete={complete}\n" + f"excluded_zones={excluded}\ndelegation_edges={edges}\n" + "nameservers_total=1\n", encoding="utf-8" + ) + coverage.write_text( + "# format=zone|soa_serial|records|delegated_names|delegation_edges\n" + f"# coverage_complete={complete}\n# excluded_zones={excluded}\n" + + rows, + encoding="utf-8", + ) + + def publish(directory, previous=None): + command = [ + sys.executable, str(ROOT / "scripts" / "publish_nsdb.py"), + "--current-private", str(private), + "--current-full-counts", str(counts), + "--current-coverage", str(coverage), + "--k", "5", "--current-tag", "nsdb-2026-10-22", + "--previous-tag", "none" if previous is None else "nsdb-2026-10-21", + "--out", str(directory), + ] + if previous is not None: + command += [ + "--previous-public", str(previous / "nameservers.txt"), + "--previous-full-counts", str(previous / "counts.txt"), + "--previous-coverage", str(previous / "coverage.txt"), + ] + return subprocess.run(command, capture_output=True, text=True) + + source("false", "gov", "example|1|10|6|5\n", 5, 1) + previous = root / "partial" + self.assertEqual(publish(previous).returncode, 0) + for file, prefix in ( + ("coverage.txt", "# "), ("nameservers.txt", "# "), + ("delegation-changes.txt", "# "), + ("nameserver-changes.txt", "# "), + ("counts.txt", ""), ("changes-counts.txt", ""), + ): + contents = (previous / file).read_text(encoding="utf-8") + self.assertIn(prefix + "coverage_complete=false", contents) + self.assertIn(prefix + "excluded_zones=gov", contents) + self.assertIn("comparison_break=true", (previous / "changes-counts.txt").read_text()) + + valid = root / "complete" + source("true", "none", "example|1|10|6|5\ngov|1|10|0|0\n", 5, 2) + outcome = publish(valid, previous) + self.assertEqual(outcome.returncode, 0, outcome.stderr) + changes = (valid / "changes-counts.txt").read_text() + self.assertIn("coverage_complete=true", changes) + self.assertIn("excluded_zones=none", changes) + self.assertIn("coverage_changed=true", changes) + self.assertIn("comparison_break=true", changes) + check = subprocess.run([ + sys.executable, str(ROOT / "scripts" / "verify_public.py"), + "--previous", str(previous / "nameservers.txt"), + "--current", str(valid / "nameservers.txt"), + "--delegation-changes", str(valid / "delegation-changes.txt"), + "--nameserver-changes", str(valid / "nameserver-changes.txt"), + ], capture_output=True, text=True) + self.assertEqual(check.returncode, 0, check.stderr) + + def test_inconsistent_partial_coverage_metadata_is_rejected(self): + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + private = root / "private.txt" + private.write_text("# format=nameserver|delegations\nns.secret.example|7\n") + counts = root / "counts.txt" + counts.write_text("zones=1\ncoverage_complete=true\nexcluded_zones=none\n") + coverage = root / "coverage.txt" + coverage.write_text( + "# format=zone|soa_serial|records|delegated_names|delegation_edges\n" + "# coverage_complete=false\n# excluded_zones=gov\nexample|1|10|6|7\n" + ) + result = subprocess.run([ + sys.executable, str(ROOT / "scripts" / "publish_nsdb.py"), + "--current-private", str(private), + "--current-full-counts", str(counts), + "--current-coverage", str(coverage), + "--k", "5", "--current-tag", "nsdb-2026-10-22", + "--out", str(root / "dist"), + ], capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("ns.secret.example", result.stdout + result.stderr) + def test_release_payload_checksum_and_attestation_input(self): with tempfile.TemporaryDirectory() as temp: out = pathlib.Path(temp) From e8c19c0166061bb2ed7ff5960e2f537d3aad1652 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:48:34 +0200 Subject: [PATCH 07/10] Document temporary gov exception and public coverage-contract semantics --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/README.md b/README.md index 37c14fe..a1df44e 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,11 @@ coverage_changed=true comparison_break=true ``` +When a zone is intentionally excluded, each public release artifact includes +the `coverage_complete=false` and `excluded_zones=gov` status, regardless +of whether a prior release exists. Such comparisons are conservatively marked +`comparison_break=true` even if both releases omit the same zone. + This distinguishes nameserver movement from changes caused by CZDS access or source coverage. ## Aggregate counts @@ -345,6 +350,41 @@ source digest in cache invalidation. Normal first-SOA archives are not fully re-parsed during this preflight. A passing preflight therefore does **not** certify the entire CZDS dataset, and the full build remains mandatory. The current aggregation uses an ephemeral global SQLite database, so a failed run reprocesses earlier zones; per-zone private checkpoints are a separate future reliability improvement. +## Temporary .gov omission (partial-coverage releases) + +**As of October 22, 2026**, protected `gov.txt.gz` fails the strict CZDS record +structure check. Until the defect is understood, the daily-release workflow +passes `--exclude-zone gov` to **both** preflight and private aggregation +through the pinned `NSDB_TEMP_EXCLUDE_ZONE=gov` setting. This is an +explicit, temporary exception tracked by [issue #11](https://github.com/soft-cloud-dev/nsdb/issues/11). +No other zone may be excluded by this option. + +The approved download manifest and raw, licensed zone files are **not** +modified, deleted or republished. The selector requires that `gov` be +present in that manifest and that at least one other zone remain; it still +checks the existence of every manifest file, including the excluded one. +All included zones undergo their normal SOA, DNS ordering and deduplication +validation. Failing included zones continue to block publication. + +Private `counts.txt` and `coverage.txt`, and the public snapshot, +coverage, diffs and summary counts are explicitly labeled with +`coverage_complete=false` and `excluded_zones=gov`. The term +`coverage_complete` refers only to intentional exclusions from the +**approved CZDS download manifest**, not to all Internet TLDs. Partial +releases display `PARTIAL: .gov excluded` in the release title and notes; +`comparison_break=true` remains set while either compared snapshot has +partial coverage. A restored `.gov` release changes the coverage set and +is explicitly flagged with `coverage_changed=true`. + +The CLI defaults to **no exclusions**. For a protected, manual partial run, +pass `--exclude-zone gov` to both `scripts/preflight_czds.py` and +`scripts/build_nsdb.py`. Passing any other zone, requesting `gov` when +it is missing, or excluding the sole manifest entry fails closed. Remove +both workflow arguments and the pinned environment setting after the +protected .gov input is validated; [issue #11](https://github.com/soft-cloud-dev/nsdb/issues/11) +tracks the restoration and its regression tests. This mechanism does **not** +fix or silently accept the malformed .gov source. + ## Same-day reruns `scripts/fetch_czds.py` writes a persistent per-day manifest. If that day's download set is complete, a rerun reuses it without contacting the zone-download endpoints again. From 58fecda7a13cf58fac9e37ddbec9347be5cc6cf3 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:49:56 +0200 Subject: [PATCH 08/10] Verify partial coverage markers agree across all public release artifacts --- scripts/verify_release_assets.py | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/scripts/verify_release_assets.py b/scripts/verify_release_assets.py index 7dd778f..0933db6 100644 --- a/scripts/verify_release_assets.py +++ b/scripts/verify_release_assets.py @@ -35,6 +35,37 @@ def file_sha256(path: Path) -> str: return digest.hexdigest() +def public_coverage_status(path: Path, *, headers: bool) -> tuple[str, str]: + """Validate explicit coverage fields without exposing any source records.""" + values = {} + with path.open("r", encoding="utf-8") as source: + for line in source: + content = line.strip() + if headers: + if not content.startswith("# "): + continue + content = content[2:] + if "=" not in content: + continue + key, value = content.split("=", 1) + if key not in ("coverage_complete", "excluded_zones"): + continue + if key in values: + raise ValueError("duplicate public coverage metadata") + values[key] = value + status = values.get("coverage_complete"), values.get("excluded_zones") + if status not in (("true", "none"), ("false", "gov")): + raise ValueError("invalid public coverage metadata") + return status + + +def require_comparison_break(path: Path, *, headers: bool) -> None: + marker = "# comparison_break=true" if headers else "comparison_break=true" + with path.open("r", encoding="utf-8") as source: + if marker not in {line.strip() for line in source}: + raise ValueError("partial coverage must be flagged as comparison_break") + + def verify(directory: Path) -> None: if directory.is_symlink() or not directory.is_dir(): raise ValueError("release output directory is missing or is a symlink") @@ -70,6 +101,26 @@ def verify(directory: Path) -> None: if file_sha256(directory / name) != expected_digest: raise ValueError(f"SHA-256 mismatch: {name}") + statuses = { + name: public_coverage_status(directory / name, headers=name not in ( + "counts.txt", "changes-counts.txt" + )) + for name in ( + "nameservers.txt", "coverage.txt", "delegation-changes.txt", + "nameserver-changes.txt", "counts.txt", "changes-counts.txt", + ) + } + if len(set(statuses.values())) != 1: + raise ValueError("release coverage status differs across public artifacts") + complete, excluded = statuses["coverage.txt"] + if complete == "false": + with (directory / "coverage.txt").open("r", encoding="utf-8") as coverage: + if any(line.startswith("gov|") for line in coverage): + raise ValueError("excluded gov zone present in public coverage") + for name in ("delegation-changes.txt", "nameserver-changes.txt"): + require_comparison_break(directory / name, headers=True) + require_comparison_break(directory / "changes-counts.txt", headers=False) + for name, compressed in GZIP_PAIRS: with (directory / compressed).open("rb") as source: header = source.read(10) From cd051e8bfa5ccd4c35e54218fc995ac0e54b0168 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:50:27 +0200 Subject: [PATCH 09/10] Update checksum fixture to include required public coverage metadata --- tests/test_first_release.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_first_release.py b/tests/test_first_release.py index cb8f02c..09a3787 100644 --- a/tests/test_first_release.py +++ b/tests/test_first_release.py @@ -143,8 +143,17 @@ def test_inconsistent_partial_coverage_metadata_is_rejected(self): def test_release_payload_checksum_and_attestation_input(self): with tempfile.TemporaryDirectory() as temp: out = pathlib.Path(temp) + metadata = { + "nameservers.txt": "# k=5\n# coverage_complete=true\n# excluded_zones=none\n", + "coverage.txt": "# coverage_complete=true\n# excluded_zones=none\n", + "delegation-changes.txt": "# coverage_complete=true\n# excluded_zones=none\n", + "nameserver-changes.txt": "# coverage_complete=true\n# excluded_zones=none\n", + "counts.txt": "coverage_complete=true\nexcluded_zones=none\n", + "changes-counts.txt": "coverage_complete=true\nexcluded_zones=none\n", + } for name in ASSETS.ARTIFACTS: - (out / name).write_bytes(b"public\n") + (out / name).write_text(metadata.get(name, "public\n"), encoding="utf-8") + original_snapshot = (out / "nameservers.txt").read_bytes() for name, compressed in ASSETS.GZIP_PAIRS: with (out / compressed).open("wb") as raw: with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=0) as stream: @@ -158,7 +167,7 @@ def test_release_payload_checksum_and_attestation_input(self): (out / "nameservers.txt").write_text("tampered\n") with self.assertRaisesRegex(ValueError, "SHA-256 mismatch"): ASSETS.verify(out) - (out / "nameservers.txt").write_text("public\n") + (out / "nameservers.txt").write_bytes(original_snapshot) (out / "private-counts.txt").write_text("secret\n") with self.assertRaisesRegex(ValueError, "allowlist"): ASSETS.verify(out) From a08490bd960f8f5ce130b23f8f8b78ddd59df695 Mon Sep 17 00:00:00 2001 From: "Lukasz D. Tulikowski" Date: Thu, 22 Oct 2026 09:52:05 +0200 Subject: [PATCH 10/10] Regression test public asset verifier rejects partial-coverage mislabeling --- tests/test_first_release.py | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_first_release.py b/tests/test_first_release.py index 09a3787..3826afb 100644 --- a/tests/test_first_release.py +++ b/tests/test_first_release.py @@ -140,6 +140,63 @@ def test_inconsistent_partial_coverage_metadata_is_rejected(self): self.assertNotEqual(result.returncode, 0) self.assertNotIn("ns.secret.example", result.stdout + result.stderr) + def test_partial_release_verifier_rejects_mislabeled_or_unmarked_assets(self): + with tempfile.TemporaryDirectory() as temp: + out = pathlib.Path(temp) + fixture = { + "nameservers.txt": "# k=5\n# coverage_complete=false\n# excluded_zones=gov\n", + "coverage.txt": ( + "# format=zone|soa_serial|records|delegated_names|delegation_edges\n" + "# coverage_complete=false\n# excluded_zones=gov\nexample|1|1|1|1\n" + ), + "delegation-changes.txt": ( + "# coverage_complete=false\n# excluded_zones=gov\n# comparison_break=true\n" + ), + "nameserver-changes.txt": ( + "# coverage_complete=false\n# excluded_zones=gov\n# comparison_break=true\n" + ), + "counts.txt": "coverage_complete=false\nexcluded_zones=gov\n", + "changes-counts.txt": ( + "coverage_complete=false\nexcluded_zones=gov\ncomparison_break=true\n" + ), + } + for name in ASSETS.ARTIFACTS: + (out / name).write_text(fixture.get(name, "public\n"), encoding="utf-8") + for name, compressed in ASSETS.GZIP_PAIRS: + with (out / compressed).open("wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=0) as stream: + stream.write((out / name).read_bytes()) + + def sums(): + (out / "SHA256SUMS").write_text("".join( + f"{hashlib.sha256((out / name).read_bytes()).hexdigest()} {name}\n" + for name in ASSETS.ARTIFACTS + ), encoding="utf-8") + + sums() + ASSETS.verify(out) + + (out / "counts.txt").write_text("coverage_complete=true\nexcluded_zones=none\n") + sums() + with self.assertRaisesRegex(ValueError, "coverage"): + ASSETS.verify(out) + + (out / "counts.txt").write_text(fixture["counts.txt"]) + (out / "changes-counts.txt").write_text( + "coverage_complete=false\nexcluded_zones=gov\ncomparison_break=false\n" + ) + sums() + with self.assertRaisesRegex(ValueError, "comparison_break"): + ASSETS.verify(out) + + (out / "changes-counts.txt").write_text(fixture["changes-counts.txt"]) + (out / "coverage.txt").write_text( + fixture["coverage.txt"] + "gov|1|1|1|1\n" + ) + sums() + with self.assertRaisesRegex(ValueError, "excluded gov zone"): + ASSETS.verify(out) + def test_release_payload_checksum_and_attestation_input(self): with tempfile.TemporaryDirectory() as temp: out = pathlib.Path(temp)