diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5856e0bfee..e3ed2f5c22 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -79,7 +79,6 @@ jobs: echo "Wheel contains generated or temporary script files" exit 1 fi - unzip -l dist/*.whl | grep 'scripts/dupscore/dupscore.toml' unzip -l dist/*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md' python -m venv /tmp/sqlbuild-wheel-smoke /tmp/sqlbuild-wheel-smoke/bin/python -m pip install dist/*.whl @@ -91,7 +90,6 @@ jobs: printf 'MODEL();\nSELECT 1 AS id\n' > /tmp/sqlbuild-scope-smoke/models/orders.sql /tmp/sqlbuild-wheel-smoke/bin/sqb --project-dir /tmp/sqlbuild-scope-smoke scope model:orders --json /tmp/sqlbuild-wheel-smoke/bin/sqlbuild --project-dir /tmp/sqlbuild-scope-smoke scope model:orders - /tmp/sqlbuild-wheel-smoke/bin/dupscore --help - uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.platform.name }} @@ -118,7 +116,6 @@ jobs: echo "Source distribution contains generated or temporary script files" exit 1 fi - tar -tzf dist/*.tar.gz | grep 'scripts/dupscore/dupscore.toml' - uses: actions/upload-artifact@v4 with: name: sdist diff --git a/AGENTS.md b/AGENTS.md index bb1e028155..c74814585b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,6 +114,13 @@ - Keep fixes within the requested scope. Do not expand supported behavior, permissions, operational cost, or system ownership without a short product decision from the user. - Do not add safeguards solely for impossible or unsupported states. Record residual risks when a concern is real but outside the current contract. +## Duplicated Code + +- Run `uv run fensu dupes --since origin/main` before review, and `uv run fensu dupes --path ''` before adding helpers or logic to an area that may already implement them. The report is advisory: unlike `fensu check`, it never gates and never needs to reach zero. +- Treat a genuine duplicate as evidence of possible wider drift, not only a cleanup task: a missing shared owner, parallel implementations of one concept, logic on the wrong side of a boundary, or copies that have already diverged (use `--diff`; a fix present in only one copy may be a bug in the others). Diagnose and record that wider smell before consolidating, because removing the copies erases the only deterministic signal of it. +- Consolidate genuine duplication the change introduces or touches. Report unrelated findings as follow-ups instead of refactoring them opportunistically. +- Members marked `[forced]` are adapter contract overrides required by `test_strict_adapter.py`. Record other intentional mirrors in `[dupes]` in `fensu.toml` with a reason. + ## Bounded Review Process Orchestrating agents must run reviews as a single bounded cycle, not an open-ended loop: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1ddfa641fb..ed81fbfd22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,10 +43,11 @@ pytest tests/unit/src/sqlbuild/compiler/compile/test_main.py To check whether a branch introduces copied or drifting code in Python or Rust, run: ```bash -uv run dupscore clones --since origin/main +uv run fensu dupes --since origin/main ``` -`make dupscore` reports the whole repository. See `scripts/dupscore/README.md` for options. +`make dupes` reports the whole repository. The report is advisory and never needs to reach zero; +`[dupes]` in `fensu.toml` configures exclusions and contract exemptions. See `fensu dupes --help`. ## Tests diff --git a/Makefile b/Makefile index 8542b00209..701ed84de0 100644 --- a/Makefile +++ b/Makefile @@ -448,8 +448,8 @@ check-fensu: uv run fensu check -dupscore: - uv run dupscore clones +dupes: + uv run fensu dupes check: diff --git a/fensu.toml b/fensu.toml index 528c17c8a2..6d449018d8 100644 --- a/fensu.toml +++ b/fensu.toml @@ -271,3 +271,9 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types" crate_names = [ "sqlbuild-rules-native", ] + +[[dupes.contract_exemptions]] +contract = "src/sqlbuild/adapter/contract/classes/strict_adapter.py:StrictAdapter" +forbidden_owners = ["src/sqlbuild/adapter/contract/classes/base_adapter.py:BaseAdapter"] +paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/classes/*"] +reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." diff --git a/pyproject.toml b/pyproject.toml index 51c155cad2..14cd88b6a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ rivers = [ [project.scripts] sqb = "sqlbuild.cli.entry.main.entry:main" sqlbuild = "sqlbuild.cli.entry.main.entry:main" -dupscore = "scripts.dupscore.main.dupscore:dupscore" [dependency-groups] dev = [ @@ -78,7 +77,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.22.0", + "fensu>=0.23.0", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] @@ -90,8 +89,6 @@ python-source = "src" features = ["extension-module"] include = [ { path = "LICENSE", format = ["sdist", "wheel"] }, - { path = "scripts/dupscore/**/*.py", format = ["sdist", "wheel"] }, - { path = "scripts/dupscore/dupscore.toml", format = ["sdist", "wheel"] }, ] [tool.commitizen] diff --git a/scripts/dupscore/README.md b/scripts/dupscore/README.md deleted file mode 100644 index 5c95b32c65..0000000000 --- a/scripts/dupscore/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# dupscore - -Duplication advisory tool for the SQLBuild repository. `make dupscore` runs the default -`clones` mode. - -## `dupscore clones` (default) - -Reports concrete function-level clones: - -- Units are Python functions and methods under `src/sqlbuild` (via `ast`; decorators are outside - the line range) and Rust `fn` items with bodies under `crates/*/src` (free functions, impl and - trait methods; via a small lexer that handles nested block comments, raw and byte strings, and - chars versus lifetimes). Test code is skipped by default: Python `tests/`, Rust `tests/` and - `benches/` directories, `#[cfg(test)]` items and modules, and `#[test]` functions. - `--include-tests` analyzes them too. The dbt integration is included, unlike in `report`. -- Token streams are normalised: local names, parameters, receivers, and plain attribute reads - become one placeholder, literals one placeholder per kind, and keywords and operators stay. - Call targets keep their names, so two functions with the same layout that call different - helpers do not look identical: Python `f(...)` and `obj.f(...)` keep `f`; Rust keeps any - identifier directly followed by `(` or `!` (functions, methods, tuple constructors, macros) - except the name after `fn`. Comments, Python docstrings, annotations, and `->` are dropped. -- Candidates come from MOSS-style winnowed fingerprints: 12-token k-grams, minimum of every - window of 6 k-grams, so any shared run of 17 or more normalised tokens yields a shared - fingerprint. Fingerprints present in more than 25 units are boilerplate and ignored. Pairs - sharing fingerprints with a Jaccard index of at least 0.2 are scored with the `difflib` - ratio of their normalised streams. -- Categories: `exact` (identical tokens after dropping comments, docstrings, and annotations), - `renamed` (identical normalised stream, so only abstracted names and literals differ), - `near-miss` (ratio at or above `--min-similarity`, default 0.8). Units under `--min-tokens` - normalised tokens (default 60) are ignored. A near-miss pair whose smaller unit has fewer - than 80 tokens must reach a ratio of at least 0.9: short wrappers share most of their - skeleton by idiom, so this removes a noisy low end without affecting exact or renamed pairs. -- Similar pairs are grouped into transitive clusters and ranked by estimated duplicated tokens. - -Useful options: - -```bash -uv run dupscore clones --since origin/main # only clusters touching lines changed since a rev -uv run dupscore clones --lang rust --top 10 -uv run dupscore clones --path 'src/sqlbuild/compiler/*' --json -``` - -`--since` diffs the worktree against the revision, so uncommitted edits and untracked files count. -Changed members are marked `[new]` or `[changed]`. - -Intentional mirrors can be suppressed in `dupscore.toml`: - -```toml -[[clone_allowlist]] -paths = ["src/sqlbuild/example/backends/*"] -reason = "Backends intentionally mirror one protocol." -``` - -A pair is suppressed when both sides match the entry's globs (`*` also matches `/`). - -Contract methods that every implementation must define itself are declared once: - -```toml -[[contract_exemption]] -contract = "src/sqlbuild/adapter/contract/classes/strict_adapter.py:StrictAdapter" -paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/classes/*"] -forbidden_owners = ["src/sqlbuild/adapter/contract/classes/base_adapter.py:BaseAdapter"] -reason = "The strict adapter contract test requires each adapter to define every method." -``` - -The contract method names are the class's abstract methods (`@abstractmethod` or -`@abc.abstractmethod`, including those inherited from resolvable bases and not overridden), -parsed with `ast` from the analysed worktree; nothing is imported. A method is a forced override -when its top-level class lies under `paths`, derives from the contract class, and would -otherwise inherit that method from the contract, a `forbidden_owners` class, or nowhere. Links -between two forced overrides are hidden and clusters left without links disappear; links from -a forced override to any other unit (a private helper copy, or an override that could have been -inherited instead) are still reported. The summary counts forced overrides hidden this way. -Forced overrides that remain in a reported cluster are marked `[forced]` in text output -(`"forced_override": true` in JSON) and listed after the other members, so a cluster's first -lines show what to fix. They also add nothing to the duplicated-token estimate that ranks -clusters: a forced override already serves as the copy that must stay. -An entry whose contract file is absent from the analysed tree is inactive; a missing class in an -existing file is a configuration error. - -## `dupscore report` and `dupscore pair A B` - -The package-pair ranking fused from call-graph, state fan-in, dataclass overlap, same-name, and -co-change signals, and drill-down evidence for one package pair. diff --git a/scripts/dupscore/__init__.py b/scripts/dupscore/__init__.py deleted file mode 100644 index e577394238..0000000000 --- a/scripts/dupscore/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dupscore duplication-risk advisory tooling.""" diff --git a/scripts/dupscore/_helpers/__init__.py b/scripts/dupscore/_helpers/__init__.py deleted file mode 100644 index c290e8d2aa..0000000000 --- a/scripts/dupscore/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dupscore signal and infrastructure helpers.""" diff --git a/scripts/dupscore/_helpers/clones/__init__.py b/scripts/dupscore/_helpers/clones/__init__.py deleted file mode 100644 index 43717e6c56..0000000000 --- a/scripts/dupscore/_helpers/clones/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dupscore clones helpers.""" diff --git a/scripts/dupscore/_helpers/clones/clustering.py b/scripts/dupscore/_helpers/clones/clustering.py deleted file mode 100644 index a4f000fa78..0000000000 --- a/scripts/dupscore/_helpers/clones/clustering.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Group clone pairs into transitive clusters and rank them.""" - -from __future__ import annotations - -from collections.abc import Callable - -from scripts.dupscore.constants import CATEGORY_ORDER -from scripts.dupscore.models import ( - CloneCluster, - CloneMember, - ClonePair, - ClonePairLink, - CloneUnit, -) - - -def cluster_clone_pairs( - *, - units: list[CloneUnit], - pairs: list[ClonePair], - change_of: Callable[[CloneUnit], str | None], - forced: frozenset[int], -) -> list[CloneCluster]: - """Group pairs into connected components ordered by estimated duplicated tokens.""" - - clusters: list[CloneCluster] = [ - _build_cluster(units=units, pairs=component, change_of=change_of, forced=forced) - for component in _connected_pair_groups(pairs) - ] - return sorted(clusters, key=_cluster_sort_key) - - -def _connected_pair_groups(pairs: list[ClonePair]) -> list[list[ClonePair]]: - neighbours: dict[int, list[int]] = {} - for pair in pairs: - neighbours.setdefault(pair.left, []).append(pair.right) - neighbours.setdefault(pair.right, []).append(pair.left) - component_of: dict[int, int] = {} - for root in sorted(neighbours): - if root in component_of: - continue - pending: list[int] = [root] - while pending: - node: int = pending.pop() - if node in component_of: - continue - component_of[node] = root - pending.extend(neighbours[node]) - grouped: dict[int, list[ClonePair]] = {} - for pair in pairs: - grouped.setdefault(component_of[pair.left], []).append(pair) - return list(grouped.values()) - - -def _unit_indexes(pairs: list[ClonePair]) -> set[int]: - indexes: set[int] = set() - for pair in pairs: - indexes.update((pair.left, pair.right)) - return indexes - - -def _build_cluster( - *, - units: list[CloneUnit], - pairs: list[ClonePair], - change_of: Callable[[CloneUnit], str | None], - forced: frozenset[int], -) -> CloneCluster: - unit_indexes: list[int] = sorted( - _unit_indexes(pairs), - key=lambda index: ( - index in forced, - units[index].path, - units[index].start_line, - units[index].name, - ), - ) - position: dict[int, int] = {unit_index: slot for slot, unit_index in enumerate(unit_indexes)} - best_similarity: dict[int, float] = {} - for pair in pairs: - for unit_index in (pair.left, pair.right): - best_similarity[unit_index] = max(best_similarity.get(unit_index, 0.0), pair.similarity) - weights: list[float] = [ - len(units[unit_index].normalized) * best_similarity[unit_index] - for unit_index in unit_indexes - if unit_index not in forced - ] - links: list[ClonePairLink] = sorted( - ( - ClonePairLink( - left=min(position[pair.left], position[pair.right]), - right=max(position[pair.left], position[pair.right]), - similarity=pair.similarity, - category=pair.category, - ) - for pair in pairs - ), - key=lambda link: (link.left, link.right), - ) - similarities: list[float] = [pair.similarity for pair in pairs] - category: str = max( - (pair.category for pair in pairs), key=lambda value: CATEGORY_ORDER.index(value) - ) - members: tuple[CloneMember, ...] = tuple( - CloneMember( - language=units[unit_index].language, - path=units[unit_index].path, - name=units[unit_index].name, - start_line=units[unit_index].start_line, - end_line=units[unit_index].end_line, - tokens=len(units[unit_index].normalized), - change=change_of(units[unit_index]), - forced_override=unit_index in forced, - ) - for unit_index in unit_indexes - ) - return CloneCluster( - category=category, - similarity_min=min(similarities), - similarity_max=max(similarities), - duplicated_tokens=_duplicated_tokens( - weights=weights, has_forced=len(weights) < len(members) - ), - members=members, - links=tuple(links), - ) - - -def _duplicated_tokens(*, weights: list[float], has_forced: bool) -> int: - """Estimate removable tokens: keep one copy, or a required forced override when present.""" - - if has_forced: - return round(sum(weights)) - return round(sum(weights) - max(weights)) - - -def _cluster_sort_key(cluster: CloneCluster) -> tuple[int, float, str, int]: - first: CloneMember = cluster.members[0] - return (-cluster.duplicated_tokens, -cluster.similarity_max, first.path, first.start_line) diff --git a/scripts/dupscore/_helpers/clones/filters.py b/scripts/dupscore/_helpers/clones/filters.py deleted file mode 100644 index 54c14dea16..0000000000 --- a/scripts/dupscore/_helpers/clones/filters.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Report filters: lines changed since a git revision, allowlist globs, and path globs.""" - -from __future__ import annotations - -import re -from fnmatch import fnmatchcase -from pathlib import Path - -from scripts.dupscore._helpers.inputs.source_provider import run_git -from scripts.dupscore.constants import CHANGE_CHANGED, CHANGE_NEW -from scripts.dupscore.models import CloneAllowlistEntry, CloneUnit, FileChanges - -_HUNK_PATTERN: re.Pattern[str] = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") -_TARGET_PREFIX: str = "+++ " -_TARGET_PATH_PREFIX: str = "b/" -_NULL_PATH: str = "/dev/null" -_GIT_OPTIONS: tuple[str, ...] = ("-c", "core.quotePath=false") - - -def collect_changes_since(*, repo_root: Path, revision: str) -> dict[str, FileChanges]: - """Map paths to worktree lines, including untracked files, changed since ``revision``.""" - - _ = run_git(repo_root=repo_root, arguments=["rev-parse", "--verify", f"{revision}^{{commit}}"]) - diff_text: str = run_git( - repo_root=repo_root, - arguments=[ - *_GIT_OPTIONS, - "diff", - "-U0", - "--no-color", - "--no-ext-diff", - "--find-renames", - "--src-prefix=a/", - "--dst-prefix=b/", - revision, - "--", - ], - ) - changes: dict[str, FileChanges] = parse_unified_diff(diff_text) - untracked: str = run_git( - repo_root=repo_root, - arguments=["ls-files", "--others", "--exclude-standard", "-z"], - ) - for relative in untracked.split("\0"): - if relative: - changes[relative] = FileChanges(whole_file=True) - return changes - - -def parse_unified_diff(diff_text: str) -> dict[str, FileChanges]: - """Parse ``git diff -U0`` output into added line ranges and pure-deletion points.""" - - added: dict[str, list[tuple[int, int]]] = {} - deletions: dict[str, list[int]] = {} - current: str | None = None - for line in diff_text.splitlines(): - if line.startswith(_TARGET_PREFIX): - target: str = line[len(_TARGET_PREFIX) :] - current = None if target == _NULL_PATH else target.removeprefix(_TARGET_PATH_PREFIX) - continue - match: re.Match[str] | None = _HUNK_PATTERN.match(line) - if match is None or current is None: - continue - start: int = int(match.group(1)) - count: int = int(match.group(2)) if match.group(2) is not None else 1 - if count == 0: - deletions.setdefault(current, []).append(start) - else: - added.setdefault(current, []).append((start, start + count - 1)) - return { - path: FileChanges( - added_ranges=tuple(added.get(path, ())), - deletion_points=tuple(deletions.get(path, ())), - ) - for path in sorted(set(added) | set(deletions)) - } - - -def classify_unit_change(*, unit: CloneUnit, changes: FileChanges | None) -> str | None: - """Return ``new`` when every unit line was added, ``changed`` on any overlap.""" - - if changes is None: - return None - if changes.whole_file: - return CHANGE_NEW - covered: int = 0 - for range_start, range_end in changes.added_ranges: - overlap: int = min(range_end, unit.end_line) - max(range_start, unit.start_line) + 1 - covered += max(overlap, 0) - if covered >= unit.end_line - unit.start_line + 1: - return CHANGE_NEW - if covered > 0: - return CHANGE_CHANGED - for point in changes.deletion_points: - if unit.start_line <= point < unit.end_line: - return CHANGE_CHANGED - return None - - -def path_matches_any(*, path: str, globs: tuple[str, ...]) -> bool: - """Match a repo-relative POSIX path against globs where ``*`` also crosses ``/``.""" - - return any(fnmatchcase(path, pattern) for pattern in globs) - - -def allowlist_reason( - *, - left_path: str, - right_path: str, - entries: tuple[CloneAllowlistEntry, ...], -) -> str | None: - """Return the reason of the first entry whose globs cover both sides of a pair.""" - - for entry in entries: - if path_matches_any(path=left_path, globs=entry.paths) and path_matches_any( - path=right_path, globs=entry.paths - ): - return entry.reason - return None diff --git a/scripts/dupscore/_helpers/clones/fingerprints.py b/scripts/dupscore/_helpers/clones/fingerprints.py deleted file mode 100644 index eb733dfcb9..0000000000 --- a/scripts/dupscore/_helpers/clones/fingerprints.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Winnowed k-gram fingerprints over normalised token streams (MOSS style).""" - -from __future__ import annotations - -import zlib -from functools import cache - -from scripts.dupscore.constants import CLONE_KGRAM_SIZE, CLONE_WINNOW_WINDOW - - -def fingerprint_tokens(normalized: tuple[str, ...]) -> frozenset[int]: - """Select the minimum k-gram hash of every window of consecutive k-grams.""" - - identifiers: list[int] = [_token_identifier(text) for text in normalized] - gram_count: int = len(identifiers) - CLONE_KGRAM_SIZE + 1 - if gram_count <= 0: - return frozenset() - hashes: list[int] = [ - hash(tuple(identifiers[offset : offset + CLONE_KGRAM_SIZE])) for offset in range(gram_count) - ] - if gram_count <= CLONE_WINNOW_WINDOW: - return frozenset({min(hashes)}) - return frozenset( - min(hashes[offset : offset + CLONE_WINNOW_WINDOW]) - for offset in range(gram_count - CLONE_WINNOW_WINDOW + 1) - ) - - -@cache -def _token_identifier(text: str) -> int: - return zlib.crc32(text.encode("utf-8")) diff --git a/scripts/dupscore/_helpers/clones/matching.py b/scripts/dupscore/_helpers/clones/matching.py deleted file mode 100644 index 5342bb48cb..0000000000 --- a/scripts/dupscore/_helpers/clones/matching.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Find similar unit pairs from normalised identity keys and shared fingerprints.""" - -from __future__ import annotations - -from collections import Counter -from difflib import SequenceMatcher -from itertools import combinations - -from scripts.dupscore.constants import ( - CATEGORY_EXACT, - CATEGORY_NEAR_MISS, - CATEGORY_RENAMED, - CLONE_CANDIDATE_MIN_JACCARD, - CLONE_MAX_FINGERPRINT_UNITS, - CLONE_MIN_FINGERPRINTS, - CLONE_SMALL_NEAR_MISS_MIN_SIMILARITY, - CLONE_SMALL_NEAR_MISS_TOKENS, -) -from scripts.dupscore.models import ClonePair, CloneUnit - -_MIN_POSTING_UNITS: int = 2 - - -def find_clone_pairs( - *, - units: list[CloneUnit], - fingerprints: list[frozenset[int]], - min_similarity: float, -) -> list[ClonePair]: - """Return identical-stream pairs plus near-miss pairs above the similarity threshold.""" - - pairs: dict[tuple[int, int], ClonePair] = {} - for left, right in _identical_stream_pairs(units): - category: str = ( - CATEGORY_EXACT - if units[left].concrete_key == units[right].concrete_key - else CATEGORY_RENAMED - ) - pairs[(left, right)] = ClonePair(left=left, right=right, similarity=1.0, category=category) - - postings: dict[int, list[int]] = {} - for unit_index, unit_fingerprints in enumerate(fingerprints): - for fingerprint in unit_fingerprints: - postings.setdefault(fingerprint, []).append(unit_index) - boilerplate: set[int] = { - fingerprint - for fingerprint, members in postings.items() - if len(members) > CLONE_MAX_FINGERPRINT_UNITS - } - distinctive_sizes: list[int] = [ - len(unit_fingerprints - boilerplate) for unit_fingerprints in fingerprints - ] - shared_counts: Counter[tuple[int, int]] = Counter() - for fingerprint, members in postings.items(): - if fingerprint in boilerplate or len(members) < _MIN_POSTING_UNITS: - continue - shared_counts.update(combinations(members, 2)) - - for (left, right), shared in shared_counts.items(): - if (left, right) in pairs or units[left].language != units[right].language: - continue - left_size: int = distinctive_sizes[left] - right_size: int = distinctive_sizes[right] - if min(left_size, right_size) < CLONE_MIN_FINGERPRINTS: - continue - if shared / (left_size + right_size - shared) < CLONE_CANDIDATE_MIN_JACCARD: - continue - required: float = _required_similarity( - smaller=min(len(units[left].normalized), len(units[right].normalized)), - min_similarity=min_similarity, - ) - similarity: float = _stream_similarity( - left=units[left].normalized, - right=units[right].normalized, - min_similarity=required, - ) - if similarity >= required: - pairs[(left, right)] = ClonePair( - left=left, - right=right, - similarity=round(similarity, 4), - category=CATEGORY_NEAR_MISS, - ) - return [pairs[key] for key in sorted(pairs)] - - -def _required_similarity(*, smaller: int, min_similarity: float) -> float: - """Demand closer matches from short units, whose shared skeletons are mostly idioms.""" - - if smaller < CLONE_SMALL_NEAR_MISS_TOKENS: - return max(min_similarity, CLONE_SMALL_NEAR_MISS_MIN_SIMILARITY) - return min_similarity - - -def _stream_similarity( - *, - left: tuple[str, ...], - right: tuple[str, ...], - min_similarity: float, -) -> float: - shorter, longer = sorted((len(left), len(right))) - if 2 * shorter / (shorter + longer) < min_similarity: - return 0.0 - return SequenceMatcher(None, left, right, autojunk=False).ratio() - - -def _identical_stream_pairs(units: list[CloneUnit]) -> list[tuple[int, int]]: - groups: dict[tuple[str, str], list[int]] = {} - for unit_index, unit in enumerate(units): - groups.setdefault((unit.language, unit.normalized_key), []).append(unit_index) - identical: list[tuple[int, int]] = [] - for members in groups.values(): - identical.extend(combinations(members, 2)) - return identical diff --git a/scripts/dupscore/_helpers/clones/python_units.py b/scripts/dupscore/_helpers/clones/python_units.py deleted file mode 100644 index f9c93474b4..0000000000 --- a/scripts/dupscore/_helpers/clones/python_units.py +++ /dev/null @@ -1,262 +0,0 @@ -"""Extract normalised function and method units from Python sources.""" - -from __future__ import annotations - -import ast -import io -import keyword -import token -import tokenize -from bisect import bisect_left -from dataclasses import dataclass - -from scripts.dupscore._helpers.clones.tokens import build_clone_unit -from scripts.dupscore.constants import ( - LANGUAGE_PYTHON, - PLACEHOLDER_FSTRING, - PLACEHOLDER_IDENTIFIER, - PLACEHOLDER_NUMBER, - PLACEHOLDER_STRING, -) -from scripts.dupscore.models import CloneUnit - -_KEYWORDS: frozenset[str] = frozenset(keyword.kwlist) -_DROPPED_TYPES: frozenset[int] = frozenset( - {tokenize.NL, tokenize.COMMENT, tokenize.ENDMARKER, tokenize.ENCODING} -) -_STRUCTURE_MARKERS: dict[int, str] = { - tokenize.NEWLINE: "$nl", - tokenize.INDENT: "$in", - tokenize.DEDENT: "$de", -} -_LITERAL_PLACEHOLDERS: dict[int, str] = { - tokenize.NUMBER: PLACEHOLDER_NUMBER, - tokenize.STRING: PLACEHOLDER_STRING, -} -_FSTRING_START_TYPES: frozenset[int] = frozenset( - getattr(token, name) for name in ("FSTRING_START", "TSTRING_START") if hasattr(token, name) -) -_FSTRING_BODY_TYPES: frozenset[int] = frozenset( - getattr(token, name) - for name in ("FSTRING_MIDDLE", "FSTRING_END", "TSTRING_MIDDLE", "TSTRING_END") - if hasattr(token, name) -) -_RETURN_ARROW: str = "->" - -type _Position = tuple[int, int] -type _Span = tuple[_Position, _Position] -type _FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef - - -@dataclass(frozen=True, slots=True) -class _IgnoredSpan: - start: _Position - end: _Position - statement: bool - - -@dataclass(frozen=True, slots=True) -class _UnitNode: - node: _FunctionNode - qualified_name: str - - -def extract_python_units(*, relative_path: str, source: str) -> tuple[CloneUnit, ...]: - """Return one unit per top-level function or class method, without decorators.""" - - try: - tree: ast.Module = ast.parse(source) - tokens: list[tokenize.TokenInfo] = [ - item - for item in tokenize.generate_tokens(io.StringIO(source).readline) - if item.type not in _DROPPED_TYPES - ] - except (SyntaxError, tokenize.TokenError): - return () - starts: list[_Position] = [item.start for item in tokens] - call_targets: frozenset[_Position] = _call_target_positions( - tree=tree, lines=io.StringIO(source).readlines() - ) - normalized: list[str | None] = [ - _normalize(item=item, call_target=item.start in call_targets) for item in tokens - ] - for span in _ignored_spans(tree.body): - end: int = bisect_left(starts, span.end) - if span.statement and end < len(tokens) and tokens[end].type == tokenize.NEWLINE: - end += 1 - for index in range(bisect_left(starts, span.start), end): - normalized[index] = None - units: list[CloneUnit] = [] - for unit_node in _collect_unit_nodes(body=tree.body, prefix=""): - node: _FunctionNode = unit_node.node - end_line: int = node.end_lineno if node.end_lineno is not None else node.lineno - end_column: int = node.end_col_offset if node.end_col_offset is not None else 0 - first: int = bisect_left(starts, (node.lineno, node.col_offset)) - last: int = bisect_left(starts, (end_line, end_column)) - kept: list[tuple[str, str]] = [ - (text, tokens[index].string) - for index in range(first, last) - if (text := normalized[index]) is not None - ] - units.append( - build_clone_unit( - language=LANGUAGE_PYTHON, - path=relative_path, - name=unit_node.qualified_name, - start_line=node.lineno, - end_line=end_line, - normalized=[text for text, _ in kept], - concrete=[text for _, text in kept], - ) - ) - return tuple(units) - - -def _normalize(*, item: tokenize.TokenInfo, call_target: bool) -> str | None: - if item.type == tokenize.NAME: - if call_target or item.string in _KEYWORDS: - return item.string - return PLACEHOLDER_IDENTIFIER - if item.type == tokenize.OP: - return None if item.string == _RETURN_ARROW else item.string - if item.type in _FSTRING_BODY_TYPES: - return None - if item.type in _FSTRING_START_TYPES: - return PLACEHOLDER_FSTRING - marker: str | None = _STRUCTURE_MARKERS.get(item.type) - if marker is not None: - return marker - return _LITERAL_PLACEHOLDERS.get(item.type, item.string) - - -def _call_target_positions(*, tree: ast.Module, lines: list[str]) -> frozenset[_Position]: - """Locate the name tokens of called functions (``f(...)``) and methods (``x.f(...)``).""" - - positions: set[_Position] = set() - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - function: ast.expr = node.func - if isinstance(function, ast.Name): - positions.add( - _token_position(lines=lines, line=function.lineno, byte_column=function.col_offset) - ) - elif ( - isinstance(function, ast.Attribute) - and function.end_lineno is not None - and function.end_col_offset is not None - ): - end: _Position = _token_position( - lines=lines, line=function.end_lineno, byte_column=function.end_col_offset - ) - positions.add((end[0], end[1] - len(function.attr))) - return frozenset(positions) - - -def _token_position(*, lines: list[str], line: int, byte_column: int) -> _Position: - """Convert an ``ast`` UTF-8 byte column into the character column ``tokenize`` reports.""" - - text: str = lines[line - 1] if 0 < line <= len(lines) else "" - if text.isascii(): - return (line, byte_column) - return (line, len(text.encode("utf-8")[:byte_column].decode("utf-8", errors="replace"))) - - -def _collect_unit_nodes(*, body: list[ast.stmt], prefix: str) -> list[_UnitNode]: - collected: list[_UnitNode] = [] - for statement in body: - if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): - collected.append(_UnitNode(node=statement, qualified_name=prefix + statement.name)) - elif isinstance(statement, ast.ClassDef): - collected.extend( - _collect_unit_nodes(body=statement.body, prefix=f"{prefix}{statement.name}.") - ) - elif isinstance(statement, (ast.If, ast.Try)): - collected.extend(_collect_unit_nodes(body=statement.body, prefix=prefix)) - collected.extend(_collect_unit_nodes(body=statement.orelse, prefix=prefix)) - return collected - - -def _ignored_spans(body: list[ast.stmt]) -> list[_IgnoredSpan]: - spans: list[_IgnoredSpan] = [] - for statement in body: - if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - docstring: ast.Expr | None = _docstring_expression(statement) - if docstring is not None: - docstring_start, docstring_end = _span_of(docstring) - spans.append(_IgnoredSpan(start=docstring_start, end=docstring_end, statement=True)) - annotation_spans: list[_Span] = [] - if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): - annotation_spans = _signature_annotation_spans(statement) - elif isinstance(statement, ast.AnnAssign): - annotation_spans = [ - _annotation_span(owner=statement.target, annotation=statement.annotation) - ] - spans.extend( - _IgnoredSpan(start=start, end=end, statement=False) for start, end in annotation_spans - ) - for child_body in _child_bodies(statement): - spans.extend(_ignored_spans(child_body)) - return spans - - -def _child_bodies(statement: ast.stmt) -> list[list[ast.stmt]]: - bodies: list[list[ast.stmt]] = [] - for field_name in ("body", "orelse", "finalbody"): - value: object = getattr(statement, field_name, None) - if isinstance(value, list): - bodies.append([item for item in value if isinstance(item, ast.stmt)]) - if isinstance(statement, (ast.Try, ast.TryStar)): - bodies.extend(handler.body for handler in statement.handlers) - elif isinstance(statement, ast.Match): - bodies.extend(case.body for case in statement.cases) - return bodies - - -def _signature_annotation_spans(node: _FunctionNode) -> list[_Span]: - spans: list[_Span] = [_span_of(node.returns)] if node.returns is not None else [] - arguments: ast.arguments = node.args - for argument in ( - *arguments.posonlyargs, - *arguments.args, - *arguments.kwonlyargs, - *(item for item in (arguments.vararg, arguments.kwarg) if item is not None), - ): - if argument.annotation is not None: - spans.append(_annotation_span(owner=argument, annotation=argument.annotation)) - return spans - - -def _annotation_span(*, owner: ast.expr | ast.arg, annotation: ast.expr) -> _Span: - """Cover the ``:`` separator and the annotation that follow a name.""" - - owner_end: _Position = ( - (owner.lineno, owner.col_offset + len(owner.arg)) - if isinstance(owner, ast.arg) - else ( - owner.end_lineno if owner.end_lineno is not None else owner.lineno, - owner.end_col_offset if owner.end_col_offset is not None else owner.col_offset, - ) - ) - return (owner_end, _span_of(annotation)[1]) - - -def _docstring_expression( - node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, -) -> ast.Expr | None: - if not node.body: - return None - first: ast.stmt = node.body[0] - if ( - isinstance(first, ast.Expr) - and isinstance(first.value, ast.Constant) - and isinstance(first.value.value, str) - ): - return first - return None - - -def _span_of(node: ast.expr | ast.stmt) -> _Span: - end_line: int = node.end_lineno if node.end_lineno is not None else node.lineno - end_column: int = node.end_col_offset if node.end_col_offset is not None else node.col_offset - return ((node.lineno, node.col_offset), (end_line, end_column)) diff --git a/scripts/dupscore/_helpers/clones/rendering.py b/scripts/dupscore/_helpers/clones/rendering.py deleted file mode 100644 index 256ff5bae7..0000000000 --- a/scripts/dupscore/_helpers/clones/rendering.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Render clone reports as concise text or deterministic JSON.""" - -from __future__ import annotations - -import json -from dataclasses import asdict - -from scripts.dupscore.constants import CLONE_TEXT_MAX_MEMBERS -from scripts.dupscore.models import CloneCluster, CloneMember, CloneReport - -_INDENT: str = " " -_FORCED_MARKER: str = " [forced]" - - -def render_clone_text(*, report: CloneReport, top: int) -> str: - """Render the ranked clusters with one line per member.""" - - counts: str = ", ".join( - f"{language} {count} units" for language, count in report.unit_counts.items() - ) - scope: str = f" changed since {report.since}" if report.since is not None else "" - lines: list[str] = [ - f"dupscore clones{scope}: {len(report.clusters)} clusters " - f"({counts}; {report.allowlisted_pairs} allowlisted pairs hidden; " - f"{report.contract_exempt_members} contract-exempt members hidden)" - ] - for position, cluster in enumerate(report.clusters[:top], start=1): - lines.append(f"{position:3d}. {_cluster_heading(cluster)}") - shown: tuple[CloneMember, ...] = cluster.members[:CLONE_TEXT_MAX_MEMBERS] - lines.extend(_INDENT + _member_line(member) for member in shown) - hidden: int = len(cluster.members) - len(shown) - if hidden: - lines.append(f"{_INDENT}... {hidden} more members (see --json)") - if not report.clusters: - lines.append("no clones found") - elif len(report.clusters) > top: - lines.append(f"... {len(report.clusters) - top} more clusters (use --top)") - return "\n".join(lines) - - -def render_clone_json(*, report: CloneReport, top: int) -> str: - """Render the ranked clusters as deterministic JSON.""" - - payload: dict[str, object] = { - "since": report.since, - "unit_counts": report.unit_counts, - "allowlisted_pairs": report.allowlisted_pairs, - "contract_exempt_members": report.contract_exempt_members, - "total_clusters": len(report.clusters), - "clusters": [ - {"rank": position, **asdict(cluster)} - for position, cluster in enumerate(report.clusters[:top], start=1) - ], - } - return json.dumps(payload, indent=1, sort_keys=True) - - -def _cluster_heading(cluster: CloneCluster) -> str: - lowest: str = f"{cluster.similarity_min:.2f}" - highest: str = f"{cluster.similarity_max:.2f}" - similarity: str = highest if lowest == highest else f"{lowest}-{highest}" - return ( - f"{cluster.category} sim {similarity}, ~{cluster.duplicated_tokens} duplicated tokens, " - f"{len(cluster.members)} members" - ) - - -def _member_line(member: CloneMember) -> str: - change: str = f" [{member.change}]" if member.change is not None else "" - forced: str = _FORCED_MARKER if member.forced_override else "" - return ( - f"{member.path}:{member.start_line}-{member.end_line} {member.name} " - f"({member.tokens} tokens){forced}{change}" - ) diff --git a/scripts/dupscore/_helpers/clones/rust_lexer.py b/scripts/dupscore/_helpers/clones/rust_lexer.py deleted file mode 100644 index 25f0a519ea..0000000000 --- a/scripts/dupscore/_helpers/clones/rust_lexer.py +++ /dev/null @@ -1,251 +0,0 @@ -"""A small Rust lexer that is precise enough for brace matching and clone tokens.""" - -from __future__ import annotations - -from scripts.dupscore.constants import ( - RUST_KIND_BYTE, - RUST_KIND_BYTE_STRING, - RUST_KIND_CHAR, - RUST_KIND_IDENTIFIER, - RUST_KIND_LIFETIME, - RUST_KIND_NUMBER, - RUST_KIND_PUNCTUATION, - RUST_KIND_STRING, -) -from scripts.dupscore.models import RustToken - -_THREE_CHAR_OPERATORS: frozenset[str] = frozenset({"..=", "...", "<<=", ">>="}) -_TWO_CHAR_OPERATORS: frozenset[str] = frozenset( - { - "::", - "->", - "=>", - "==", - "!=", - "<=", - ">=", - "&&", - "||", - "+=", - "-=", - "*=", - "/=", - "%=", - "^=", - "&=", - "|=", - "..", - } -) -_STRING_PREFIX_KINDS: dict[str, str] = { - "b": RUST_KIND_BYTE_STRING, - "br": RUST_KIND_BYTE_STRING, - "c": RUST_KIND_STRING, - "cr": RUST_KIND_STRING, - "r": RUST_KIND_STRING, -} -_RAW_MARKER: str = "r" -_BYTE_PREFIX: str = "b" -_QUOTE: str = '"' -_APOSTROPHE: str = "'" -_BACKSLASH: str = "\\" -_HASH: str = "#" -_NEWLINE: str = "\n" -_UNDERSCORE: str = "_" -_DECIMAL_POINT: str = "." -_EXPONENT_MARKERS: frozenset[str] = frozenset({"e", "E"}) -_EXPONENT_SIGNS: frozenset[str] = frozenset({"+", "-"}) -_HEX_PREFIXES: tuple[str, ...] = ("0x", "0X") -_LINE_COMMENT: str = "//" -_BLOCK_COMMENT_OPEN: str = "/*" -_BLOCK_COMMENT_CLOSE: str = "*/" -_ESCAPE_WIDTH: int = 2 -_EXPONENT_WIDTH: int = 2 -_CHAR_AND_CLOSING_WIDTH: int = 2 -_OPERATOR_WIDTHS: tuple[tuple[int, frozenset[str]], ...] = ( - (3, _THREE_CHAR_OPERATORS), - (2, _TWO_CHAR_OPERATORS), -) - - -def lex_rust(source: str) -> list[RustToken]: - """Tokenise Rust source, dropping whitespace and all comments.""" - - lexer: _Lexer = _Lexer(source=source) - return lexer.run() - - -def _is_word_start(character: str) -> bool: - return character == _UNDERSCORE or character.isalpha() - - -def _is_word_part(character: str) -> bool: - return character == _UNDERSCORE or character.isalnum() - - -class _Lexer: - def __init__(self, *, source: str) -> None: - self._source: str = source - self._index: int = 0 - self._line: int = 1 - self._tokens: list[RustToken] = [] - - def run(self) -> list[RustToken]: - length: int = len(self._source) - while self._index < length: - character: str = self._source[self._index] - if character == _NEWLINE: - self._line += 1 - self._index += 1 - elif character.isspace(): - self._index += 1 - elif self._source.startswith(_LINE_COMMENT, self._index): - self._skip_line_comment() - elif self._source.startswith(_BLOCK_COMMENT_OPEN, self._index): - self._skip_block_comment() - elif character == _QUOTE: - self._lex_quoted(kind=RUST_KIND_STRING, prefix_length=0) - elif character == _APOSTROPHE: - self._lex_apostrophe(prefix_length=0) - elif character.isdigit(): - self._lex_number() - elif _is_word_start(character): - self._lex_word() - else: - self._lex_punctuation() - return self._tokens - - def _emit(self, *, kind: str, start: int, line: int) -> None: - text: str = self._source[start : self._index] - self._tokens.append(RustToken(kind=kind, text=text, line=line)) - - def _advance_to(self, end: int) -> None: - self._line += self._source.count(_NEWLINE, self._index, end) - self._index = end - - def _word_end(self, start: int) -> int: - cursor: int = start - while cursor < len(self._source) and _is_word_part(self._source[cursor]): - cursor += 1 - return cursor - - def _skip_line_comment(self) -> None: - end: int = self._source.find(_NEWLINE, self._index) - self._index = len(self._source) if end < 0 else end - - def _skip_block_comment(self) -> None: - depth: int = 0 - cursor: int = self._index - length: int = len(self._source) - while cursor < length: - if self._source.startswith(_BLOCK_COMMENT_OPEN, cursor): - depth += 1 - cursor += len(_BLOCK_COMMENT_OPEN) - elif self._source.startswith(_BLOCK_COMMENT_CLOSE, cursor): - depth -= 1 - cursor += len(_BLOCK_COMMENT_CLOSE) - if depth == 0: - break - else: - cursor += 1 - self._advance_to(cursor) - - def _lex_word(self) -> None: - start: int = self._index - line: int = self._line - cursor: int = self._word_end(start) - word: str = self._source[start:cursor] - following: str = self._source[cursor : cursor + 1] - prefix_kind: str | None = _STRING_PREFIX_KINDS.get(word) - if prefix_kind is not None and following in (_QUOTE, _HASH): - if word.endswith(_RAW_MARKER): - if self._lex_raw_string(kind=prefix_kind, prefix_end=cursor): - return - elif following == _QUOTE: - self._lex_quoted(kind=prefix_kind, prefix_length=cursor - start) - return - if word == _BYTE_PREFIX and following == _APOSTROPHE: - self._lex_apostrophe(prefix_length=len(_BYTE_PREFIX)) - return - if word == _RAW_MARKER and following == _HASH and self._is_raw_identifier(cursor): - cursor = self._word_end(cursor + 1) - self._index = cursor - self._emit(kind=RUST_KIND_IDENTIFIER, start=start, line=line) - - def _is_raw_identifier(self, hash_index: int) -> bool: - after: int = hash_index + 1 - return after < len(self._source) and _is_word_start(self._source[after]) - - def _lex_raw_string(self, *, kind: str, prefix_end: int) -> bool: - start: int = self._index - line: int = self._line - cursor: int = prefix_end - hashes: int = 0 - while cursor < len(self._source) and self._source[cursor] == _HASH: - hashes += 1 - cursor += 1 - if cursor >= len(self._source) or self._source[cursor] != _QUOTE: - return False - terminator: str = _QUOTE + _HASH * hashes - end: int = self._source.find(terminator, cursor + 1) - end = len(self._source) if end < 0 else end + len(terminator) - self._advance_to(end) - self._emit(kind=kind, start=start, line=line) - return True - - def _lex_quoted(self, *, kind: str, prefix_length: int) -> None: - start: int = self._index - line: int = self._line - cursor: int = start + prefix_length + 1 - length: int = len(self._source) - while cursor < length and self._source[cursor] != _QUOTE: - cursor += _ESCAPE_WIDTH if self._source[cursor] == _BACKSLASH else 1 - self._advance_to(min(cursor + 1, length)) - self._emit(kind=kind, start=start, line=line) - - def _lex_apostrophe(self, *, prefix_length: int) -> None: - start: int = self._index - line: int = self._line - after: int = start + prefix_length + 1 - length: int = len(self._source) - kind: str = RUST_KIND_BYTE if prefix_length else RUST_KIND_CHAR - if after < length and self._source[after] == _BACKSLASH: - closing: int = self._source.find(_APOSTROPHE, after + _ESCAPE_WIDTH) - self._index = length if closing < 0 else closing + 1 - self._emit(kind=kind, start=start, line=line) - return - if after + 1 < length and self._source[after + 1] == _APOSTROPHE: - self._index = after + _CHAR_AND_CLOSING_WIDTH - self._emit(kind=kind, start=start, line=line) - return - self._index = self._word_end(after) - self._emit(kind=RUST_KIND_LIFETIME, start=start, line=line) - - def _lex_number(self) -> None: - start: int = self._index - line: int = self._line - cursor: int = start - length: int = len(self._source) - hexadecimal: bool = self._source.startswith(_HEX_PREFIXES, start) - while cursor < length: - character: str = self._source[cursor] - following: str = self._source[cursor + 1 : cursor + 2] - if not hexadecimal and character in _EXPONENT_MARKERS and following in _EXPONENT_SIGNS: - cursor += _EXPONENT_WIDTH - elif _is_word_part(character) or (character == _DECIMAL_POINT and following.isdigit()): - cursor += 1 - else: - break - self._index = cursor - self._emit(kind=RUST_KIND_NUMBER, start=start, line=line) - - def _lex_punctuation(self) -> None: - start: int = self._index - line: int = self._line - for width, operators in _OPERATOR_WIDTHS: - if self._source[start : start + width] in operators: - self._index = start + width - self._emit(kind=RUST_KIND_PUNCTUATION, start=start, line=line) - return - self._index = start + 1 - self._emit(kind=RUST_KIND_PUNCTUATION, start=start, line=line) diff --git a/scripts/dupscore/_helpers/clones/rust_units.py b/scripts/dupscore/_helpers/clones/rust_units.py deleted file mode 100644 index 681ea75e9e..0000000000 --- a/scripts/dupscore/_helpers/clones/rust_units.py +++ /dev/null @@ -1,365 +0,0 @@ -"""Extract normalised `fn` item units from Rust sources.""" - -from __future__ import annotations - -from pathlib import PurePosixPath - -from scripts.dupscore._helpers.clones.rust_lexer import lex_rust -from scripts.dupscore._helpers.clones.tokens import build_clone_unit -from scripts.dupscore.constants import ( - LANGUAGE_RUST, - PLACEHOLDER_BYTE, - PLACEHOLDER_BYTE_STRING, - PLACEHOLDER_CHAR, - PLACEHOLDER_IDENTIFIER, - PLACEHOLDER_LIFETIME, - PLACEHOLDER_NUMBER, - PLACEHOLDER_STRING, - RUST_KIND_BYTE, - RUST_KIND_BYTE_STRING, - RUST_KIND_CHAR, - RUST_KIND_IDENTIFIER, - RUST_KIND_LIFETIME, - RUST_KIND_NUMBER, - RUST_KIND_PUNCTUATION, - RUST_KIND_STRING, -) -from scripts.dupscore.models import CloneUnit, RustFileUnits, RustToken - -_RUST_KEYWORDS: frozenset[str] = frozenset( - { - "as", - "async", - "await", - "break", - "const", - "continue", - "crate", - "dyn", - "else", - "enum", - "extern", - "false", - "fn", - "for", - "if", - "impl", - "in", - "let", - "loop", - "match", - "mod", - "move", - "mut", - "pub", - "ref", - "return", - "self", - "Self", - "static", - "struct", - "super", - "trait", - "true", - "type", - "unsafe", - "use", - "where", - "while", - } -) -_LITERAL_PLACEHOLDERS: dict[str, str] = { - RUST_KIND_STRING: PLACEHOLDER_STRING, - RUST_KIND_BYTE_STRING: PLACEHOLDER_BYTE_STRING, - RUST_KIND_CHAR: PLACEHOLDER_CHAR, - RUST_KIND_BYTE: PLACEHOLDER_BYTE, - RUST_KIND_NUMBER: PLACEHOLDER_NUMBER, - RUST_KIND_LIFETIME: PLACEHOLDER_LIFETIME, -} -_OPEN_BRACE: str = "{" -_SEMICOLON: str = ";" -_ATTRIBUTE_MARKER: str = "#" -_INNER_ATTRIBUTE_MARKER: str = "!" -_ATTRIBUTE_OPEN: str = "[" -_OPENERS: dict[str, str] = {"{": "}", "(": ")", "[": "]"} -_CLOSERS: frozenset[str] = frozenset(_OPENERS.values()) -_SIGNATURE_GROUP_OPENERS: frozenset[str] = frozenset({"(", "["}) -_BODY_OR_END: frozenset[str] = frozenset({_OPEN_BRACE, _SEMICOLON}) -_ANGLE_DEPTH_CHANGES: dict[str, int] = {"<": 1, ">": -1} -_FUNCTION_KEYWORD: str = "fn" -_OWNER_KEYWORDS: frozenset[str] = frozenset({"impl", "trait"}) -_MACRO_RULES: str = "macro_rules" -_MODULE_KEYWORD: str = "mod" -_FOR_KEYWORD: str = "for" -_WHERE_KEYWORD: str = "where" -_MODULE_ROOT_FILES: frozenset[str] = frozenset({"mod.rs", "lib.rs", "main.rs"}) -_TEST_ATTRIBUTE: str = "test" -_CFG_ATTRIBUTE: str = "cfg" -_NEGATION: str = "not" -_PATH_SEPARATOR: str = "::" -_RUST_SUFFIX: str = ".rs" -_DIRECTORY_SUFFIX: str = "/" -_OWNER_SEPARATOR: str = "::" -_CALL_SUFFIXES: frozenset[str] = frozenset({"(", "!"}) - - -def extract_rust_units( - *, - relative_path: str, - source: str, - include_tests: bool, -) -> RustFileUnits: - """Return every `fn` item with a body, skipping test-only items unless requested.""" - - tokens: list[RustToken] = lex_rust(source) - scanner: _ItemScanner = _ItemScanner( - tokens=tokens, - relative_path=relative_path, - include_tests=include_tests, - ) - scanner.scan(start=0, end=len(tokens), owner=None) - return RustFileUnits( - units=tuple(scanner.units), - test_module_prefixes=tuple(scanner.test_module_prefixes), - ) - - -def _normalize_stream(tokens: list[RustToken]) -> list[str]: - normalized: list[str] = [] - for index, item in enumerate(tokens): - if item.kind == RUST_KIND_IDENTIFIER: - keep: bool = item.text in _RUST_KEYWORDS or _is_call_target(tokens=tokens, index=index) - normalized.append(item.text if keep else PLACEHOLDER_IDENTIFIER) - else: - normalized.append(_LITERAL_PLACEHOLDERS.get(item.kind, item.text)) - return normalized - - -def _is_call_target(*, tokens: list[RustToken], index: int) -> bool: - """Name a called function, method, or macro: an identifier directly before ``(`` or ``!``.""" - - following: int = index + 1 - if following >= len(tokens): - return False - after: RustToken = tokens[following] - if after.kind != RUST_KIND_PUNCTUATION or after.text not in _CALL_SUFFIXES: - return False - return index == 0 or tokens[index - 1].text != _FUNCTION_KEYWORD - - -def _match_brackets(tokens: list[RustToken]) -> list[int]: - matches: list[int] = [-1] * len(tokens) - stack: list[int] = [] - for index, item in enumerate(tokens): - if item.kind != RUST_KIND_PUNCTUATION: - continue - if item.text in _OPENERS: - stack.append(index) - elif item.text in _CLOSERS and stack and _OPENERS[tokens[stack[-1]].text] == item.text: - opener: int = stack.pop() - matches[opener] = index - matches[index] = opener - return matches - - -def _is_test_attribute(texts: list[str]) -> bool: - if not texts: - return False - if texts[0] == _CFG_ATTRIBUTE: - return _TEST_ATTRIBUTE in texts and _NEGATION not in texts - return texts[-1] == _TEST_ATTRIBUTE and all( - text == _PATH_SEPARATOR or text.isidentifier() for text in texts - ) - - -def _strip_leading_generics(header: list[RustToken]) -> list[RustToken]: - depth: int = 0 - for position, item in enumerate(header): - depth += _ANGLE_DEPTH_CHANGES.get(item.text, 0) - if depth == 0: - return header if position == 0 else header[position + 1 :] - return [] - - -def _owner_name(header: list[RustToken]) -> str | None: - """Name the self type of an ``impl`` or ``trait`` header (tokens between it and ``{``).""" - - candidates: list[RustToken] = _strip_leading_generics(header) - depth: int = 0 - for position, item in enumerate(candidates): - depth += _ANGLE_DEPTH_CHANGES.get(item.text, 0) - if depth == 0 and item.text == _FOR_KEYWORD: - candidates = candidates[position + 1 :] - break - name: str | None = None - depth = 0 - for item in candidates: - depth += _ANGLE_DEPTH_CHANGES.get(item.text, 0) - if depth == 0 and item.text == _WHERE_KEYWORD: - break - if depth == 0 and item.kind == RUST_KIND_IDENTIFIER and item.text not in _RUST_KEYWORDS: - name = item.text - return name - - -class _ItemScanner: - def __init__( - self, - *, - tokens: list[RustToken], - relative_path: str, - include_tests: bool, - ) -> None: - self._tokens: list[RustToken] = tokens - self._matches: list[int] = _match_brackets(tokens) - self._relative_path: str = relative_path - self._include_tests: bool = include_tests - self.units: list[CloneUnit] = [] - self.test_module_prefixes: list[str] = [] - - def scan(self, *, start: int, end: int, owner: str | None) -> None: - index: int = start - test_item: bool = False - item_start: int | None = None - while index < end: - attribute_end: int | None = self._attribute_end(index) - if attribute_end is not None: - test_item = test_item or self._attribute_is_test(index=index, end=attribute_end) - index = attribute_end + 1 - continue - if self._is_punctuation(index=index, text=_SEMICOLON): - index += 1 - test_item, item_start = False, None - continue - if test_item and not self._include_tests: - self._record_test_module(index) - index = self._item_end(start=index, end=end) - test_item, item_start = False, None - continue - if item_start is None: - item_start = index - next_index: int | None = self._scan_item( - index=index, end=end, owner=owner, item_start=item_start - ) - if next_index is None: - index += 1 - continue - index = next_index - test_item, item_start = False, None - - def _scan_item(self, *, index: int, end: int, owner: str | None, item_start: int) -> int | None: - item: RustToken = self._tokens[index] - is_identifier: bool = item.kind == RUST_KIND_IDENTIFIER - if is_identifier and item.text == _FUNCTION_KEYWORD and self._is_identifier(index + 1): - body: int = self._find_body(start=index + 2, end=end) - if not self._is_punctuation(index=body, text=_OPEN_BRACE) or body >= end: - return body + 1 - close: int = self._closing(opener=body, end=end) - self._emit_unit(start=index, close=close, owner=owner, item_start=item_start) - return close + 1 - if is_identifier and item.text in _OWNER_KEYWORDS: - body = self._find_body(start=index + 1, end=end) - if not self._is_punctuation(index=body, text=_OPEN_BRACE) or body >= end: - return body + 1 - close = self._closing(opener=body, end=end) - self.scan( - start=body + 1, - end=close, - owner=_owner_name(self._tokens[index + 1 : body]), - ) - return close + 1 - if is_identifier and item.text == _MACRO_RULES: - body = index + 1 - while body < end and self._tokens[body].text not in _OPENERS: - body += 1 - return self._closing(opener=body, end=end) + 1 if body < end else end - if self._is_punctuation(index=index, text=_OPEN_BRACE): - close = self._closing(opener=index, end=end) - self.scan(start=index + 1, end=close, owner=owner) - return close + 1 - return None - - def _is_punctuation(self, *, index: int, text: str) -> bool: - return ( - index < len(self._tokens) - and self._tokens[index].kind == RUST_KIND_PUNCTUATION - and self._tokens[index].text == text - ) - - def _is_identifier(self, index: int) -> bool: - return index < len(self._tokens) and self._tokens[index].kind == RUST_KIND_IDENTIFIER - - def _attribute_open(self, index: int) -> int | None: - if not self._is_punctuation(index=index, text=_ATTRIBUTE_MARKER): - return None - opener: int = index + 1 - if self._is_punctuation(index=opener, text=_INNER_ATTRIBUTE_MARKER): - opener += 1 - return opener if self._is_punctuation(index=opener, text=_ATTRIBUTE_OPEN) else None - - def _attribute_end(self, index: int) -> int | None: - opener: int | None = self._attribute_open(index) - if opener is None: - return None - close: int = self._matches[opener] - return close if close > opener else None - - def _attribute_is_test(self, *, index: int, end: int) -> bool: - opener: int | None = self._attribute_open(index) - if opener is None: - return False - return _is_test_attribute([item.text for item in self._tokens[opener + 1 : end]]) - - def _closing(self, *, opener: int, end: int) -> int: - close: int = self._matches[opener] - return close if opener < close < end else end - 1 - - def _find_body(self, *, start: int, end: int) -> int: - index: int = start - while index < end: - item: RustToken = self._tokens[index] - if item.kind == RUST_KIND_PUNCTUATION and item.text in _BODY_OR_END: - return index - if item.kind == RUST_KIND_PUNCTUATION and item.text in _SIGNATURE_GROUP_OPENERS: - index = self._closing(opener=index, end=end) + 1 - continue - index += 1 - return end - - def _item_end(self, *, start: int, end: int) -> int: - body: int = self._find_body(start=start, end=end) - if body >= end: - return end - if self._is_punctuation(index=body, text=_OPEN_BRACE): - return self._closing(opener=body, end=end) + 1 - return body + 1 - - def _record_test_module(self, index: int) -> None: - if not ( - self._tokens[index].text == _MODULE_KEYWORD - and self._is_identifier(index + 1) - and self._is_punctuation(index=index + 2, text=_SEMICOLON) - ): - return - path: PurePosixPath = PurePosixPath(self._relative_path) - base: PurePosixPath = ( - path.parent if path.name in _MODULE_ROOT_FILES else path.parent / path.stem - ) - module_path: str = (base / self._tokens[index + 1].text).as_posix() - self.test_module_prefixes.append(module_path + _RUST_SUFFIX) - self.test_module_prefixes.append(module_path + _DIRECTORY_SUFFIX) - - def _emit_unit(self, *, start: int, close: int, owner: str | None, item_start: int) -> None: - function_name: str = self._tokens[start + 1].text - body: list[RustToken] = self._tokens[start : close + 1] - self.units.append( - build_clone_unit( - language=LANGUAGE_RUST, - path=self._relative_path, - name=f"{owner}{_OWNER_SEPARATOR}{function_name}" if owner else function_name, - start_line=self._tokens[item_start].line, - end_line=self._tokens[close].line, - normalized=_normalize_stream(body), - concrete=[item.text for item in body], - ) - ) diff --git a/scripts/dupscore/_helpers/clones/sources.py b/scripts/dupscore/_helpers/clones/sources.py deleted file mode 100644 index 98c8cc7784..0000000000 --- a/scripts/dupscore/_helpers/clones/sources.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Collect clone units from the Python package and Rust crates in the worktree.""" - -from __future__ import annotations - -from pathlib import Path - -from scripts.dupscore._helpers.clones.python_units import extract_python_units -from scripts.dupscore._helpers.clones.rust_units import extract_rust_units -from scripts.dupscore.constants import ( - LANGUAGE_PYTHON, - LANGUAGE_RUST, - PROJECT_PACKAGE, - PYTHON_TEST_ROOT, - RUST_CRATES_ROOT, - RUST_SOURCE_DIRECTORY, - RUST_TEST_DIRECTORIES, - SOURCE_ROOT, -) -from scripts.dupscore.models import CloneUnit, RustFileUnits - -_PYTHON_PATTERN: str = "*.py" -_RUST_PATTERN: str = "*.rs" -_CRATE_DIRECTORY_INDEX: int = 1 - - -def collect_clone_units( - *, - repo_root: Path, - languages: tuple[str, ...], - include_tests: bool, -) -> list[CloneUnit]: - """Extract units for the requested languages in deterministic path order.""" - - units: list[CloneUnit] = [] - if LANGUAGE_PYTHON in languages: - units.extend(_python_units(repo_root=repo_root, include_tests=include_tests)) - if LANGUAGE_RUST in languages: - units.extend(_rust_units(repo_root=repo_root, include_tests=include_tests)) - return units - - -def _python_units(*, repo_root: Path, include_tests: bool) -> list[CloneUnit]: - files: list[Path] = sorted((repo_root / SOURCE_ROOT / PROJECT_PACKAGE).rglob(_PYTHON_PATTERN)) - if include_tests: - files.extend(sorted((repo_root / PYTHON_TEST_ROOT).rglob(_PYTHON_PATTERN))) - units: list[CloneUnit] = [] - for path in files: - units.extend( - extract_python_units( - relative_path=path.relative_to(repo_root).as_posix(), - source=path.read_text(encoding="utf-8", errors="replace"), - ) - ) - return units - - -def _rust_units(*, repo_root: Path, include_tests: bool) -> list[CloneUnit]: - crates_root: Path = repo_root / RUST_CRATES_ROOT - files: list[Path] = sorted( - path - for path in crates_root.rglob(_RUST_PATTERN) - if _is_rust_candidate(path=path.relative_to(crates_root), include_tests=include_tests) - ) - units: list[CloneUnit] = [] - test_prefixes: list[str] = [] - for path in files: - extracted: RustFileUnits = extract_rust_units( - relative_path=path.relative_to(repo_root).as_posix(), - source=path.read_text(encoding="utf-8", errors="replace"), - include_tests=include_tests, - ) - units.extend(extracted.units) - test_prefixes.extend(extracted.test_module_prefixes) - prefixes: tuple[str, ...] = tuple(test_prefixes) - return [unit for unit in units if not unit.path.startswith(prefixes)] - - -def _is_rust_candidate(*, path: Path, include_tests: bool) -> bool: - parts: tuple[str, ...] = path.parts - if len(parts) <= _CRATE_DIRECTORY_INDEX: - return False - section: str = parts[_CRATE_DIRECTORY_INDEX] - if include_tests: - return section == RUST_SOURCE_DIRECTORY or section in RUST_TEST_DIRECTORIES - nested: tuple[str, ...] = parts[_CRATE_DIRECTORY_INDEX + 1 :] - return section == RUST_SOURCE_DIRECTORY and not RUST_TEST_DIRECTORIES.intersection(nested) diff --git a/scripts/dupscore/_helpers/clones/tokens.py b/scripts/dupscore/_helpers/clones/tokens.py deleted file mode 100644 index 7acfeca6bf..0000000000 --- a/scripts/dupscore/_helpers/clones/tokens.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Build clone units from lexed token streams.""" - -from __future__ import annotations - -import hashlib -from collections.abc import Iterable - -from scripts.dupscore.models import CloneUnit - -_KEY_SEPARATOR: str = "\x1f" -_DIGEST_BYTES: int = 16 - - -def build_clone_unit( - *, - language: str, - path: str, - name: str, - start_line: int, - end_line: int, - normalized: list[str], - concrete: list[str], -) -> CloneUnit: - """Assemble a unit with identity keys for its concrete and normalised streams.""" - - return CloneUnit( - language=language, - path=path, - name=name, - start_line=start_line, - end_line=end_line, - normalized=tuple(normalized), - concrete_key=_digest(concrete), - normalized_key=_digest(normalized), - ) - - -def _digest(parts: Iterable[str]) -> str: - joined: str = _KEY_SEPARATOR.join(parts) - return hashlib.blake2b(joined.encode("utf-8"), digest_size=_DIGEST_BYTES).hexdigest() diff --git a/scripts/dupscore/_helpers/contracts/__init__.py b/scripts/dupscore/_helpers/contracts/__init__.py deleted file mode 100644 index 43717e6c56..0000000000 --- a/scripts/dupscore/_helpers/contracts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dupscore clones helpers.""" diff --git a/scripts/dupscore/_helpers/contracts/forced_overrides.py b/scripts/dupscore/_helpers/contracts/forced_overrides.py deleted file mode 100644 index 8e470c037f..0000000000 --- a/scripts/dupscore/_helpers/contracts/forced_overrides.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Recognise forced overrides of contract methods, derived statically from the analysed source.""" - -from __future__ import annotations - -import ast -from dataclasses import dataclass -from pathlib import Path, PurePosixPath - -from scripts.dupscore._helpers.clones.filters import path_matches_any -from scripts.dupscore.constants import LANGUAGE_PYTHON, SOURCE_ROOT -from scripts.dupscore.exceptions import DupscoreConfigError -from scripts.dupscore.models import CloneUnit, ContractExemptionEntry - -_ABSTRACT_DECORATOR: str = "abstractmethod" -_MODULE_SEPARATOR: str = "." -_PYTHON_SUFFIX: str = ".py" -_PACKAGE_INIT: str = "__init__.py" -_METHOD_NAME_PARTS: int = 2 - -type _FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef -type _ClassKey = tuple[str, str] - - -@dataclass(frozen=True, slots=True) -class _ClassInfo: - bases: tuple[_ClassKey, ...] - concrete: frozenset[str] - abstract: frozenset[str] - complete: bool - - -@dataclass(frozen=True, slots=True) -class _ContractScope: - contract: _ClassKey - method_names: frozenset[str] - forbidden_owners: frozenset[_ClassKey] - paths: tuple[str, ...] - - -def find_forced_overrides( - *, - repo_root: Path, - entries: tuple[ContractExemptionEntry, ...], - units: list[CloneUnit], -) -> tuple[bool, ...]: - """Flag each unit that overrides a contract method it may not inherit from an ancestor.""" - - exemptions: _ContractExemptions = _ContractExemptions(repo_root=repo_root, entries=entries) - return tuple(exemptions.is_forced(unit) for unit in units) - - -class _ContractExemptions: - def __init__(self, *, repo_root: Path, entries: tuple[ContractExemptionEntry, ...]) -> None: - self._resolver: _ClassResolver = _ClassResolver(repo_root=repo_root) - active: list[ContractExemptionEntry] = [ - entry for entry in entries if (repo_root / entry.contract_path).is_file() - ] - self._scopes: tuple[_ContractScope, ...] = tuple(self._scope(entry) for entry in active) - - def is_forced(self, unit: CloneUnit) -> bool: - """Require a top-level class method under the paths that derives from the contract.""" - - if unit.language != LANGUAGE_PYTHON: - return False - parts: list[str] = unit.name.split(_MODULE_SEPARATOR) - if len(parts) != _METHOD_NAME_PARTS: - return False - class_name, method_name = parts - for scope in self._scopes: - if method_name not in scope.method_names: - continue - if not path_matches_any(path=unit.path, globs=scope.paths): - continue - if self._forced_in_scope( - scope=scope, key=(unit.path, class_name), method_name=method_name - ): - return True - return False - - def _forced_in_scope(self, *, scope: _ContractScope, key: _ClassKey, method_name: str) -> bool: - order: tuple[_ClassKey, ...] = self._resolver.method_resolution_order(key) - if scope.contract not in order: - return False - if not all(self._resolver.info(ancestor).complete for ancestor in order): - return False - for ancestor in order[1:]: - if method_name in self._resolver.info(ancestor).concrete: - return ancestor in scope.forbidden_owners - return True - - def _scope(self, entry: ContractExemptionEntry) -> _ContractScope: - contract: _ClassKey = (entry.contract_path, entry.contract_class) - forbidden: set[_ClassKey] = {contract} - for key in (contract, *entry.forbidden_owners): - if not self._resolver.exists(key): - raise DupscoreConfigError( - f"contract_exemption class {key[0]}:{key[1]} was not found in the analysed " - "source" - ) - forbidden.add(key) - return _ContractScope( - contract=contract, - method_names=self._resolver.abstract_methods(contract), - forbidden_owners=frozenset(forbidden), - paths=entry.paths, - ) - - -class _ClassResolver: - def __init__(self, *, repo_root: Path) -> None: - self._repo_root: Path = repo_root - self._trees: dict[str, ast.Module | None] = {} - self._infos: dict[_ClassKey, _ClassInfo] = {} - self._orders: dict[_ClassKey, tuple[_ClassKey, ...]] = {} - self._abstract: dict[_ClassKey, frozenset[str]] = {} - - def exists(self, key: _ClassKey) -> bool: - tree: ast.Module | None = self._parse(key[0]) - return tree is not None and _find_class(tree=tree, name=key[1]) is not None - - def info(self, key: _ClassKey) -> _ClassInfo: - if key not in self._infos: - self._infos[key] = self._build_info(key) - return self._infos[key] - - def abstract_methods(self, key: _ClassKey) -> frozenset[str]: - """Mirror ``__abstractmethods__``: own abstract methods plus unoverridden inherited ones.""" - - if key in self._abstract: - return self._abstract[key] - self._abstract[key] = frozenset() - info: _ClassInfo = self.info(key) - names: set[str] = set(info.abstract) - for base in info.bases: - names.update(self.abstract_methods(base) - info.concrete) - self._abstract[key] = frozenset(names) - return self._abstract[key] - - def method_resolution_order(self, key: _ClassKey) -> tuple[_ClassKey, ...]: - """C3-linearise the statically resolvable bases, as ``type.__mro__`` would.""" - - if key in self._orders: - return self._orders[key] - self._orders[key] = (key,) - bases: tuple[_ClassKey, ...] = self.info(key).bases - sequences: list[list[_ClassKey]] = [ - list(self.method_resolution_order(base)) for base in bases - ] - self._orders[key] = (key, *_c3_merge([*sequences, list(bases)])) - return self._orders[key] - - def _parse(self, relative_path: str) -> ast.Module | None: - if relative_path not in self._trees: - try: - source: str = (self._repo_root / relative_path).read_text(encoding="utf-8") - self._trees[relative_path] = ast.parse(source) - except (OSError, SyntaxError, UnicodeDecodeError, ValueError): - self._trees[relative_path] = None - return self._trees[relative_path] - - def _build_info(self, key: _ClassKey) -> _ClassInfo: - tree: ast.Module | None = self._parse(key[0]) - node: ast.ClassDef | None = None if tree is None else _find_class(tree=tree, name=key[1]) - if tree is None or node is None: - return _ClassInfo(bases=(), concrete=frozenset(), abstract=frozenset(), complete=False) - functions: list[_FunctionNode] = [ - item for item in node.body if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) - ] - bases: list[_ClassKey] = [] - complete: bool = True - for base in node.bases: - resolved: _ClassKey | None = self._resolve_base(tree=tree, importer=key[0], base=base) - if resolved is not None: - bases.append(resolved) - elif self._references_local_module(tree=tree, importer=key[0], base=base): - complete = False - return _ClassInfo( - bases=tuple(bases), - concrete=frozenset(item.name for item in functions if not _is_abstract(item)), - abstract=frozenset(item.name for item in functions if _is_abstract(item)), - complete=complete, - ) - - def _references_local_module(self, *, tree: ast.Module, importer: str, base: ast.expr) -> bool: - """Report bases that name this repository but could not be resolved to a class.""" - - root: ast.expr = base.value if isinstance(base, ast.Subscript) else base - while isinstance(root, ast.Attribute): - root = root.value - if not isinstance(root, ast.Name): - return True - if _find_class(tree=tree, name=root.id) is not None: - return True - for statement in tree.body: - if isinstance(statement, ast.ImportFrom): - for alias in statement.names: - if (alias.asname or alias.name) == root.id: - return ( - self._module_path( - importer=importer, module=statement.module, level=statement.level - ) - is not None - or statement.level > 0 - ) - elif isinstance(statement, ast.Import): - for alias in statement.names: - if (alias.asname or alias.name.split(_MODULE_SEPARATOR)[0]) == root.id: - return ( - self._module_path(importer=importer, module=alias.name, level=0) - is not None - ) - return False - - def _resolve_base(self, *, tree: ast.Module, importer: str, base: ast.expr) -> _ClassKey | None: - if isinstance(base, ast.Subscript): - base = base.value - if not isinstance(base, ast.Name): - return None - if _find_class(tree=tree, name=base.id) is not None: - return (importer, base.id) - for statement in tree.body: - if not isinstance(statement, ast.ImportFrom): - continue - for alias in statement.names: - if (alias.asname or alias.name) == base.id: - module_path: str | None = self._module_path( - importer=importer, module=statement.module, level=statement.level - ) - return None if module_path is None else (module_path, alias.name) - return None - - def _module_path(self, *, importer: str, module: str | None, level: int) -> str | None: - if not module: - return None - parts: list[str] = module.split(_MODULE_SEPARATOR) - if level: - anchor: PurePosixPath = PurePosixPath(importer).parent - for _ in range(level - 1): - anchor = anchor.parent - bases: list[PurePosixPath] = [anchor.joinpath(*parts)] - else: - bases = [PurePosixPath(SOURCE_ROOT).joinpath(*parts), PurePosixPath(*parts)] - for base in bases: - for candidate in (base.with_name(base.name + _PYTHON_SUFFIX), base / _PACKAGE_INIT): - if (self._repo_root / candidate).is_file(): - return candidate.as_posix() - return None - - -def _c3_merge(sequences: list[list[_ClassKey]]) -> list[_ClassKey]: - """Merge linearisations; an inconsistent hierarchy falls back to first-seen order.""" - - merged: list[_ClassKey] = [] - pending: list[list[_ClassKey]] = [sequence for sequence in sequences if sequence] - while pending: - head: _ClassKey = _c3_head(pending) - merged.append(head) - remaining: list[list[_ClassKey]] = [] - for sequence in pending: - rest: list[_ClassKey] = [item for item in sequence if item != head] - if rest: - remaining.append(rest) - pending = remaining - return merged - - -def _c3_head(pending: list[list[_ClassKey]]) -> _ClassKey: - tails: set[_ClassKey] = set() - for sequence in pending: - tails.update(sequence[1:]) - for sequence in pending: - if sequence[0] not in tails: - return sequence[0] - return pending[0][0] - - -def _find_class(*, tree: ast.Module, name: str) -> ast.ClassDef | None: - for statement in tree.body: - if isinstance(statement, ast.ClassDef) and statement.name == name: - return statement - return None - - -def _is_abstract(function: _FunctionNode) -> bool: - for decorator in function.decorator_list: - if isinstance(decorator, ast.Name) and decorator.id == _ABSTRACT_DECORATOR: - return True - if isinstance(decorator, ast.Attribute) and decorator.attr == _ABSTRACT_DECORATOR: - return True - return False diff --git a/scripts/dupscore/_helpers/inputs/__init__.py b/scripts/dupscore/_helpers/inputs/__init__.py deleted file mode 100644 index 6385831d4f..0000000000 --- a/scripts/dupscore/_helpers/inputs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dupscore inputs helpers.""" diff --git a/scripts/dupscore/_helpers/inputs/cli.py b/scripts/dupscore/_helpers/inputs/cli.py deleted file mode 100644 index e779c6a56b..0000000000 --- a/scripts/dupscore/_helpers/inputs/cli.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Parse and validate dupscore command-line arguments.""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -from scripts.dupscore.constants import ( - CLONES_MODE, - DEFAULT_CLONE_MIN_SIMILARITY, - DEFAULT_CLONE_MIN_TOKENS, - PAIR_MODE, - REPORT_MODE, - SUPPORTED_LANGUAGES, -) -from scripts.dupscore.models import CloneOptions - -_CLONE_ONLY_OPTIONS: dict[str, str] = { - "min_similarity": "--min-similarity", - "min_tokens": "--min-tokens", - "language": "--lang", - "path_globs": "--path", - "include_tests": "--include-tests", -} -_MAX_SIMILARITY: float = 1.0 -_MIN_POSITIVE_COUNT: int = 1 - - -def parse_arguments(argv: list[str] | None) -> argparse.Namespace: - """Parse CLI arguments, exiting with a usage error for invalid combinations.""" - - parser: argparse.ArgumentParser = _build_parser() - args: argparse.Namespace = parser.parse_args(argv) - problem: str | None = _argument_problem(args) - if problem is not None: - parser.error(problem) - return args - - -def clone_options_from(args: argparse.Namespace) -> CloneOptions: - """Build clone-mode options from parsed arguments and defaults.""" - - return CloneOptions( - languages=(args.language,) if args.language is not None else SUPPORTED_LANGUAGES, - include_tests=bool(args.include_tests), - min_tokens=args.min_tokens if args.min_tokens is not None else DEFAULT_CLONE_MIN_TOKENS, - min_similarity=( - args.min_similarity if args.min_similarity is not None else DEFAULT_CLONE_MIN_SIMILARITY - ), - path_globs=tuple(args.path_globs or ()), - since=args.since, - ) - - -def _build_parser() -> argparse.ArgumentParser: - parser: argparse.ArgumentParser = argparse.ArgumentParser( - prog="dupscore", - description=( - "Duplication advisory tool. 'clones' (default) reports concrete function-level " - "clones in Python and Rust; 'report' ranks package pairs; 'pair' drills into one." - ), - ) - parser.add_argument( - "mode", - nargs="?", - choices=[CLONES_MODE, REPORT_MODE, PAIR_MODE], - default=CLONES_MODE, - ) - parser.add_argument("packages", nargs="*", default=[]) - parser.add_argument("--top", type=int, default=None, help="Number of entries to print.") - parser.add_argument("--domain", type=str, default=None, help="report: package prefix.") - parser.add_argument( - "--since", - type=str, - default=None, - help="clones: only clusters touching lines changed since REV; report: delta vs REV.", - ) - parser.add_argument("--json", action="store_true", dest="as_json") - parser.add_argument("--repo-root", type=Path, default=None, help="Repository to analyze.") - parser.add_argument( - "--min-similarity", - type=float, - default=None, - help=f"clones: minimum token similarity (default {DEFAULT_CLONE_MIN_SIMILARITY}).", - ) - parser.add_argument( - "--min-tokens", - type=int, - default=None, - help=f"clones: minimum normalised tokens per unit (default {DEFAULT_CLONE_MIN_TOKENS}).", - ) - parser.add_argument( - "--lang", - choices=SUPPORTED_LANGUAGES, - default=None, - dest="language", - help="clones: analyze one language only.", - ) - parser.add_argument( - "--path", - action="append", - default=None, - dest="path_globs", - help="clones: only clusters with a member matching this glob (repeatable).", - ) - parser.add_argument( - "--include-tests", - action="store_true", - default=None, - help="clones: also analyze Python tests/ and Rust test modules.", - ) - return parser - - -def _argument_problem(args: argparse.Namespace) -> str | None: - if args.mode != CLONES_MODE: - misused: list[str] = [ - flag for attribute, flag in _CLONE_ONLY_OPTIONS.items() if getattr(args, attribute) - ] - return f"{', '.join(misused)} only apply to the {CLONES_MODE} mode" if misused else None - if args.packages: - return f"the {CLONES_MODE} mode takes no positional package names" - if args.domain is not None: - return f"--domain only applies to the {REPORT_MODE} mode" - if args.top is not None and args.top < _MIN_POSITIVE_COUNT: - return "--top must be at least 1" - if args.min_tokens is not None and args.min_tokens < _MIN_POSITIVE_COUNT: - return "--min-tokens must be at least 1" - if args.min_similarity is not None and not 0.0 < args.min_similarity <= _MAX_SIMILARITY: - return "--min-similarity must be in (0, 1]" - return None diff --git a/scripts/dupscore/_helpers/inputs/config.py b/scripts/dupscore/_helpers/inputs/config.py deleted file mode 100644 index 281e452e58..0000000000 --- a/scripts/dupscore/_helpers/inputs/config.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Load the dupscore TOML configuration.""" - -from __future__ import annotations - -import tomllib -from pathlib import Path - -from scripts.dupscore.exceptions import DupscoreConfigError -from scripts.dupscore.models import CloneAllowlistEntry, ContractExemptionEntry, DupscoreConfig - -_ALLOWLIST_KEY: str = "allowlist" -_SURFACES_KEY: str = "persisted_state_surfaces" -_PAIR_KEY: str = "pair" -_REASON_KEY: str = "reason" -_PAIR_LENGTH: int = 2 -_CLONE_ALLOWLIST_KEY: str = "clone_allowlist" -_PATHS_KEY: str = "paths" -_CLONE_ALLOWLIST_KEYS: frozenset[str] = frozenset({_PATHS_KEY, _REASON_KEY}) -_CONTRACT_EXEMPTION_KEY: str = "contract_exemption" -_CONTRACT_KEY: str = "contract" -_FORBIDDEN_OWNERS_KEY: str = "forbidden_owners" -_CONTRACT_EXEMPTION_KEYS: frozenset[str] = frozenset( - {_CONTRACT_KEY, _FORBIDDEN_OWNERS_KEY, _PATHS_KEY, _REASON_KEY} -) -_CLASS_SPEC_SEPARATOR: str = ":" -_CLASS_SPEC_PARTS: int = 2 -_CLASS_SPEC_FORMAT: str = "'relative/path.py:ClassName'" -_PYTHON_SUFFIX: str = ".py" -_ABSOLUTE_PATH_PREFIX: str = "/" - - -def load_config(config_path: Path) -> DupscoreConfig: - """Load and validate dupscore configuration from a TOML file.""" - - if not config_path.is_file(): - return DupscoreConfig() - raw: dict[str, object] = tomllib.loads(config_path.read_text(encoding="utf-8")) - surfaces_raw: object = raw.get(_SURFACES_KEY, []) - if not isinstance(surfaces_raw, list): - raise DupscoreConfigError(f"{_SURFACES_KEY} must be a list of module prefixes") - surfaces: list[str] = [] - for surface in surfaces_raw: - if not isinstance(surface, str): - raise DupscoreConfigError(f"{_SURFACES_KEY} entries must be strings") - surfaces.append(surface) - - allowlist_raw: object = raw.get(_ALLOWLIST_KEY, []) - if not isinstance(allowlist_raw, list): - raise DupscoreConfigError(f"{_ALLOWLIST_KEY} must be an array of tables") - allowlisted_pairs: dict[tuple[str, str], str] = {} - for item in allowlist_raw: - if not isinstance(item, dict): - raise DupscoreConfigError(f"{_ALLOWLIST_KEY} entries must be tables") - pair_raw: object = item.get(_PAIR_KEY) - reason_raw: object = item.get(_REASON_KEY) - if not isinstance(pair_raw, list) or len(pair_raw) != _PAIR_LENGTH: - raise DupscoreConfigError( - f"{_ALLOWLIST_KEY} entries need a two-element {_PAIR_KEY} list" - ) - if not isinstance(reason_raw, str) or not reason_raw: - raise DupscoreConfigError(f"{_ALLOWLIST_KEY} entries need a non-empty {_REASON_KEY}") - left, right = str(pair_raw[0]), str(pair_raw[1]) - ordered: tuple[str, str] = (left, right) if left <= right else (right, left) - allowlisted_pairs[ordered] = reason_raw - return DupscoreConfig( - persisted_state_surfaces=tuple(sorted(surfaces)), - allowlisted_pairs=allowlisted_pairs, - clone_allowlist=_load_clone_allowlist(raw.get(_CLONE_ALLOWLIST_KEY, [])), - contract_exemptions=_load_contract_exemptions(raw.get(_CONTRACT_EXEMPTION_KEY, [])), - ) - - -def _load_clone_allowlist(raw: object) -> tuple[CloneAllowlistEntry, ...]: - return tuple( - CloneAllowlistEntry( - paths=_glob_list(item=item, label=label), reason=_reason(item=item, label=label) - ) - for label, item in _table_entries( - raw=raw, key=_CLONE_ALLOWLIST_KEY, allowed=_CLONE_ALLOWLIST_KEYS - ) - ) - - -def _load_contract_exemptions(raw: object) -> tuple[ContractExemptionEntry, ...]: - entries: list[ContractExemptionEntry] = [] - for label, item in _table_entries( - raw=raw, key=_CONTRACT_EXEMPTION_KEY, allowed=_CONTRACT_EXEMPTION_KEYS - ): - contract_path, contract_class = _class_spec( - raw=item.get(_CONTRACT_KEY), label=f"{label} {_CONTRACT_KEY}" - ) - owners_raw: object = item.get(_FORBIDDEN_OWNERS_KEY, []) - if not isinstance(owners_raw, list): - raise DupscoreConfigError( - f"{label} {_FORBIDDEN_OWNERS_KEY} must be a list of {_CLASS_SPEC_FORMAT} strings" - ) - entries.append( - ContractExemptionEntry( - contract_path=contract_path, - contract_class=contract_class, - paths=_glob_list(item=item, label=label), - reason=_reason(item=item, label=label), - forbidden_owners=tuple( - _class_spec(raw=owner, label=f"{label} {_FORBIDDEN_OWNERS_KEY}") - for owner in owners_raw - ), - ) - ) - return tuple(entries) - - -def _table_entries( - *, raw: object, key: str, allowed: frozenset[str] -) -> list[tuple[str, dict[str, object]]]: - if not isinstance(raw, list): - raise DupscoreConfigError(f"{key} must be an array of tables") - entries: list[tuple[str, dict[str, object]]] = [] - for position, raw_item in enumerate(raw, start=1): - label: str = f"{key} entry {position}" - if not isinstance(raw_item, dict): - raise DupscoreConfigError(f"{label} must be a table") - item: dict[str, object] = {str(name): value for name, value in raw_item.items()} - unknown: set[str] = set(item) - allowed - if unknown: - raise DupscoreConfigError(f"{label} has unknown keys: {', '.join(sorted(unknown))}") - entries.append((label, item)) - return entries - - -def _glob_list(*, item: dict[str, object], label: str) -> tuple[str, ...]: - paths_raw: object = item.get(_PATHS_KEY) - if ( - not isinstance(paths_raw, list) - or not paths_raw - or not all(isinstance(path, str) and path.strip() for path in paths_raw) - ): - raise DupscoreConfigError( - f"{label} needs a non-empty {_PATHS_KEY} list of non-empty glob strings" - ) - return tuple(str(path) for path in paths_raw) - - -def _reason(*, item: dict[str, object], label: str) -> str: - reason_raw: object = item.get(_REASON_KEY) - if not isinstance(reason_raw, str) or not reason_raw.strip(): - raise DupscoreConfigError(f"{label} needs a non-empty {_REASON_KEY}") - return reason_raw.strip() - - -def _class_spec(*, raw: object, label: str) -> tuple[str, str]: - parts: list[str] = raw.split(_CLASS_SPEC_SEPARATOR) if isinstance(raw, str) else [] - if ( - len(parts) != _CLASS_SPEC_PARTS - or not parts[0].endswith(_PYTHON_SUFFIX) - or parts[0].startswith(_ABSOLUTE_PATH_PREFIX) - or not parts[1].isidentifier() - ): - raise DupscoreConfigError(f"{label} must be {_CLASS_SPEC_FORMAT}") - return parts[0], parts[1] diff --git a/scripts/dupscore/_helpers/inputs/source_provider.py b/scripts/dupscore/_helpers/inputs/source_provider.py deleted file mode 100644 index 7bb5575800..0000000000 --- a/scripts/dupscore/_helpers/inputs/source_provider.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Read analyzable Python sources from the worktree or a git revision.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -from scripts.dupscore.constants import ( - EXCLUDED_MODULE_PREFIX, - PACKAGE_DEPTH, - PROJECT_PACKAGE, - SOURCE_ROOT, -) -from scripts.dupscore.exceptions import DupscoreGitError - -_PYTHON_SUFFIX: str = ".py" -_INIT_MODULE: str = "__init__" -_MIN_SOURCE_PATH_PARTS: int = 2 - - -def module_name_for(relative_path: str) -> str | None: - """Translate a repo-relative file path into an analyzable module name.""" - - path: Path = Path(relative_path) - if path.suffix != _PYTHON_SUFFIX: - return None - parts: tuple[str, ...] = path.with_suffix("").parts - if ( - len(parts) < _MIN_SOURCE_PATH_PARTS - or parts[0] != SOURCE_ROOT - or parts[1] != PROJECT_PACKAGE - ): - return None - module_parts: tuple[str, ...] = parts[1:] - if module_parts[-1] == _INIT_MODULE: - module_parts = module_parts[:-1] - module: str = ".".join(module_parts) - if module == EXCLUDED_MODULE_PREFIX or module.startswith(EXCLUDED_MODULE_PREFIX + "."): - return None - return module - - -def read_worktree_sources(repo_root: Path) -> dict[str, str]: - """Read all analyzable module sources from the current worktree.""" - - sources: dict[str, str] = {} - package_root: Path = repo_root / SOURCE_ROOT / PROJECT_PACKAGE - for file_path in sorted(package_root.rglob("*" + _PYTHON_SUFFIX)): - relative: str = file_path.relative_to(repo_root).as_posix() - module: str | None = module_name_for(relative) - if module is None: - continue - sources[relative] = file_path.read_text(encoding="utf-8") - return sources - - -def read_revision_sources(*, repo_root: Path, revision: str) -> dict[str, str]: - """Read all analyzable module sources as stored at one git revision.""" - - listing: str = run_git( - repo_root=repo_root, arguments=["ls-tree", "-r", "--name-only", revision] - ) - sources: dict[str, str] = {} - for relative in sorted(listing.splitlines()): - module: str | None = module_name_for(relative) - if module is None: - continue - blob_reference: str = f"{revision}:{relative}" - sources[relative] = run_git(repo_root=repo_root, arguments=["show", blob_reference]) - return sources - - -def resolve_default_base_revision(repo_root: Path) -> str: - """Resolve the default comparison base as the merge-base with the main branch.""" - - return run_git(repo_root=repo_root, arguments=["merge-base", "main", "HEAD"]).strip() - - -def run_git(*, repo_root: Path, arguments: list[str]) -> str: - """Run one git command in the repository and return its stdout.""" - - completed: subprocess.CompletedProcess[str] = subprocess.run( - ["git", *arguments], - cwd=repo_root, - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - raise DupscoreGitError(f"git {' '.join(arguments)} failed: {completed.stderr.strip()}") - return completed.stdout - - -def package_of(module: str) -> str: - """Reduce a module name to its aggregation package prefix.""" - - parts: list[str] = module.split(".") - return ".".join(parts[:PACKAGE_DEPTH]) - - -def sorted_pair(*, left: str, right: str) -> tuple[str, str]: - """Return the pair in deterministic lexicographic order.""" - - if left <= right: - return (left, right) - return (right, left) diff --git a/scripts/dupscore/_helpers/package_pairs/__init__.py b/scripts/dupscore/_helpers/package_pairs/__init__.py deleted file mode 100644 index fb4b2f7c7f..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dupscore package pairs helpers.""" diff --git a/scripts/dupscore/_helpers/package_pairs/facts.py b/scripts/dupscore/_helpers/package_pairs/facts.py deleted file mode 100644 index 0ec1e87714..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/facts.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Extract module, function, call, and dataclass facts from Python sources.""" - -from __future__ import annotations - -import ast -import re - -from scripts.dupscore._helpers.inputs.source_provider import module_name_for -from scripts.dupscore.models import ClassFact, FunctionFact, ModuleFacts, ProjectFacts - -_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") -_DATACLASS_MARKER: str = "dataclass" -_MODEL_BASE_SUFFIXES: tuple[str, ...] = ("BaseModel", "NamedTuple", "TypedDict") - - -def extract_project_facts(sources: dict[str, str]) -> ProjectFacts: - """Extract facts for every parseable module in the given sources.""" - - modules: list[ModuleFacts] = [] - for relative_path in sorted(sources): - module: str | None = module_name_for(relative_path) - if module is None: - continue - parsed: ast.Module | None = _parse_module(sources[relative_path]) - if parsed is None: - continue - modules.append( - _extract_module_facts(module=module, relative_path=relative_path, tree=parsed) - ) - return ProjectFacts(modules=tuple(modules)) - - -def _parse_module(source: str) -> ast.Module | None: - try: - return ast.parse(source) - except SyntaxError: - return None - - -def _extract_module_facts(*, module: str, relative_path: str, tree: ast.Module) -> ModuleFacts: - imports: dict[str, str] = _collect_imports(tree) - local_function_names: set[str] = { - node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } - functions: list[FunctionFact] = [] - classes: list[ClassFact] = [] - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - functions.append( - _extract_function_fact( - node=node, - module=module, - owner_prefix="", - imports=imports, - local_function_names=local_function_names, - ) - ) - elif isinstance(node, ast.ClassDef): - classes.append( - _extract_class_fact( - node=node, - module=module, - imports=imports, - local_function_names=local_function_names, - ) - ) - imported_modules: set[str] = set() - for target in imports.values(): - imported_modules.add(target) - return ModuleFacts( - module=module, - relative_path=relative_path, - functions=tuple(functions), - classes=tuple(classes), - imported_modules=tuple(sorted(imported_modules)), - ) - - -def _collect_imports(tree: ast.Module) -> dict[str, str]: - imports: dict[str, str] = {} - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - local_name: str = alias.asname or alias.name.split(".")[0] - imports[local_name] = alias.name - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module is not None: - for alias in node.names: - local_name = alias.asname or alias.name - imports[local_name] = f"{node.module}.{alias.name}" - return imports - - -def _extract_class_fact( - *, - node: ast.ClassDef, - module: str, - imports: dict[str, str], - local_function_names: set[str], -) -> ClassFact: - methods: list[FunctionFact] = [] - field_names: list[str] = [] - for statement in node.body: - if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): - methods.append( - _extract_function_fact( - node=statement, - module=module, - owner_prefix=node.name + ".", - imports=imports, - local_function_names=local_function_names, - ) - ) - elif isinstance(statement, ast.AnnAssign) and isinstance(statement.target, ast.Name): - field_names.append(statement.target.id) - return ClassFact( - name=node.name, - qualified_name=f"{module}::{node.name}", - module=module, - public=not node.name.startswith("_"), - lineno=node.lineno, - dataclass_like=_is_dataclass_like(node), - field_names=tuple(field_names), - methods=tuple(methods), - ) - - -def _is_dataclass_like(node: ast.ClassDef) -> bool: - for decorator in node.decorator_list: - decorated: ast.expr = decorator.func if isinstance(decorator, ast.Call) else decorator - dotted: str | None = _dotted_name(decorated) - if dotted is not None and _DATACLASS_MARKER in dotted: - return True - for base in node.bases: - base_name: str | None = _dotted_name(base) - if base_name is not None and base_name.endswith(_MODEL_BASE_SUFFIXES): - return True - return False - - -def _extract_function_fact( - *, - node: ast.FunctionDef | ast.AsyncFunctionDef, - module: str, - owner_prefix: str, - imports: dict[str, str], - local_function_names: set[str], -) -> FunctionFact: - resolved_calls: set[str] = set() - bare_attribute_calls: set[str] = set() - for statement in node.body: - for descendant in ast.walk(statement): - if not isinstance(descendant, ast.Call): - continue - resolved, bare = _classify_call( - call=descendant, - module=module, - imports=imports, - local_function_names=local_function_names, - ) - if resolved is not None: - resolved_calls.add(resolved) - if bare is not None: - bare_attribute_calls.add(bare) - body_dump: str = ast.dump(ast.Module(body=node.body, type_ignores=[])) - body_tokens: frozenset[str] = frozenset(_IDENTIFIER_PATTERN.findall(body_dump)) - return FunctionFact( - name=node.name, - qualified_name=f"{module}::{owner_prefix}{node.name}", - module=module, - public=not node.name.startswith("_"), - lineno=node.lineno, - resolved_calls=tuple(sorted(resolved_calls)), - bare_attribute_calls=tuple(sorted(bare_attribute_calls)), - body_tokens=body_tokens, - ) - - -def _classify_call( - *, - call: ast.Call, - module: str, - imports: dict[str, str], - local_function_names: set[str], -) -> tuple[str | None, str | None]: - target: ast.expr = call.func - if isinstance(target, ast.Name): - if target.id in local_function_names: - return f"{module}::{target.id}", None - if target.id in imports and imports[target.id].startswith("sqlbuild"): - imported_target: str = imports[target.id] - owner, _, symbol = imported_target.rpartition(".") - return f"{owner}::{symbol}", None - return None, None - if isinstance(target, ast.Attribute): - receiver: str | None = _dotted_name(target.value) - if receiver is not None: - receiver_root: str = receiver.split(".")[0] - if receiver_root in imports and imports[receiver_root].startswith("sqlbuild"): - receiver_suffix: list[str] = receiver.split(".")[1:] - full_target: str = ".".join([imports[receiver_root], *receiver_suffix, target.attr]) - owner, _, symbol = full_target.rpartition(".") - return f"{owner}::{symbol}", None - return None, target.attr - return None, None - - -def _dotted_name(node: ast.expr) -> str | None: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - base: str | None = _dotted_name(node.value) - if base is None: - return None - return f"{base}.{node.attr}" - return None diff --git a/scripts/dupscore/_helpers/package_pairs/fusion.py b/scripts/dupscore/_helpers/package_pairs/fusion.py deleted file mode 100644 index 9dd5a92967..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/fusion.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Reciprocal-rank fusion of signal rankings into one combined report.""" - -from __future__ import annotations - -from scripts.dupscore.constants import RRF_RANK_OFFSET, SIGNAL_WEIGHTS -from scripts.dupscore.models import ( - CombinedPairEntry, - DupscoreConfig, - DupscoreReport, - SignalContribution, - SignalRanking, -) - - -def fuse_rankings( - *, - rankings: tuple[SignalRanking, ...], - config: DupscoreConfig, - revision_label: str, -) -> DupscoreReport: - """Fuse per-signal rankings into one deterministic combined report.""" - - contributions_by_pair: dict[tuple[str, str], list[SignalContribution]] = {} - for ranking in rankings: - weight: float = SIGNAL_WEIGHTS.get(ranking.signal_name, 1.0) - for rank, entry in enumerate(ranking.entries, start=1): - points: float = weight / (RRF_RANK_OFFSET + rank) - contribution: SignalContribution = SignalContribution( - signal_name=ranking.signal_name, - rank=rank, - points=points, - evidence=entry.evidence, - ) - contributions_by_pair.setdefault(entry.package_pair, []).append(contribution) - - entries: list[CombinedPairEntry] = [] - for package_pair in sorted(contributions_by_pair): - contributions: list[SignalContribution] = sorted( - contributions_by_pair[package_pair], - key=lambda contribution: contribution.signal_name, - ) - total: float = sum(contribution.points for contribution in contributions) - allowlist_reason: str | None = config.allowlisted_pairs.get(package_pair) - entries.append( - CombinedPairEntry( - package_pair=package_pair, - score=total, - allowlisted=allowlist_reason is not None, - allowlist_reason=allowlist_reason, - contributions=tuple(contributions), - ) - ) - entries.sort(key=lambda entry: (-entry.score, -len(entry.contributions), entry.package_pair)) - return DupscoreReport( - revision_label=revision_label, - total_pairs=len(entries), - entries=tuple(entries), - ) - - -def filter_report_to_domain(*, report: DupscoreReport, domain: str | None) -> DupscoreReport: - """Keep only entries whose pair mentions the requested domain substring.""" - - if domain is None: - return report - matching: list[CombinedPairEntry] = [] - for entry in report.entries: - joined_pair: str = " ".join(entry.package_pair) - if domain in joined_pair: - matching.append(entry) - return DupscoreReport( - revision_label=report.revision_label, - total_pairs=len(matching), - entries=tuple(matching), - ) diff --git a/scripts/dupscore/_helpers/package_pairs/rendering.py b/scripts/dupscore/_helpers/package_pairs/rendering.py deleted file mode 100644 index 9e14fcdcb6..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/rendering.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Render dupscore reports as human-readable text or JSON.""" - -from __future__ import annotations - -import json -from dataclasses import asdict - -from scripts.dupscore.models import ( - CombinedPairEntry, - DupscoreReport, - PairEvidenceReport, - ReportDelta, -) - -_SIGNAL_ABBREVIATIONS: dict[str, str] = { - "callgraph_shape": "cg", - "state_fanin": "st", - "dataclass_overlap": "dc", - "same_name_symbols": "nm", - "cochange": "cc", -} - - -def render_report_text(*, report: DupscoreReport, top: int) -> str: - """Render the combined ranking as aligned text lines.""" - - lines: list[str] = [ - f"dupscore report @ {report.revision_label}: {report.total_pairs} package pairs", - ] - for position, entry in enumerate(report.entries[:top], start=1): - lines.append(_format_entry_line(position=position, entry=entry)) - return "\n".join(lines) - - -def render_report_json(*, report: DupscoreReport, top: int) -> str: - """Render the combined ranking as deterministic JSON.""" - - payload: dict[str, object] = { - "revision_label": report.revision_label, - "total_pairs": report.total_pairs, - "entries": [asdict(entry) for entry in report.entries[:top]], - } - return json.dumps(payload, indent=1, sort_keys=True) - - -def render_pair_text(evidence: PairEvidenceReport) -> str: - """Render pair drill-down evidence as text lines.""" - - left, right = evidence.package_pair - rank_label: str = ( - str(evidence.combined_rank) if evidence.combined_rank is not None else "unranked" - ) - lines: list[str] = [f"pair {left} <-> {right} (combined rank {rank_label})"] - for contribution in evidence.contributions: - lines.append(f" {contribution.signal_name} rank {contribution.rank}:") - for item in contribution.evidence: - lines.append(f" {item}") - if not evidence.contributions: - lines.append(" no signal fired for this pair") - return "\n".join(lines) - - -def render_pair_json(evidence: PairEvidenceReport) -> str: - """Render pair drill-down evidence as deterministic JSON.""" - - return json.dumps(asdict(evidence), indent=1, sort_keys=True) - - -def render_delta_text(delta: ReportDelta) -> str: - """Render a two-revision comparison as text lines.""" - - lines: list[str] = [f"dupscore delta {delta.base_label} -> {delta.current_label}"] - for entry in delta.entered_top: - lines.append("+ entered top: " + _format_entry_line(position=0, entry=entry).strip()) - for entry in delta.left_top: - lines.append("- left top: " + _format_entry_line(position=0, entry=entry).strip()) - for item in delta.new_state_fanin_evidence: - lines.append("+ state fan-in: " + item) - for item in delta.new_same_name_evidence: - lines.append("+ same-name twin: " + item) - if len(lines) == 1: - lines.append("no material changes") - return "\n".join(lines) - - -def render_delta_json(delta: ReportDelta) -> str: - """Render a two-revision comparison as deterministic JSON.""" - - return json.dumps(asdict(delta), indent=1, sort_keys=True) - - -def _format_entry_line(*, position: int, entry: CombinedPairEntry) -> str: - left, right = entry.package_pair - signal_parts: list[str] = [] - for contribution in entry.contributions: - abbreviation: str = _SIGNAL_ABBREVIATIONS.get( - contribution.signal_name, contribution.signal_name - ) - signal_parts.append(f"{abbreviation}#{contribution.rank}") - allow_marker: str = " [allowlisted]" if entry.allowlisted else "" - prefix: str = f"{position:3d}. " if position else "" - return f"{prefix}{entry.score:.4f} [{','.join(signal_parts)}] {left} <-> {right}{allow_marker}" diff --git a/scripts/dupscore/_helpers/package_pairs/signal_callgraph.py b/scripts/dupscore/_helpers/package_pairs/signal_callgraph.py deleted file mode 100644 index da87a4f98e..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/signal_callgraph.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Signal: IDF-weighted shared-leaves / disjoint-middles call-graph shape.""" - -from __future__ import annotations - -import math -from collections.abc import Callable -from dataclasses import dataclass -from itertools import combinations - -from scripts.dupscore._helpers.inputs.source_provider import package_of, sorted_pair -from scripts.dupscore.constants import ( - MAX_LEAF_USERS_FOR_PAIRING, - MIN_REACHABLE_FUNCTIONS, - MIN_SHARED_LEAVES, - SIGNAL_NAME_CALLGRAPH, -) -from scripts.dupscore.models import ProjectFacts, SignalPairScore, SignalRanking - -_EVIDENCE_LEAF_SAMPLE: int = 6 - - -@dataclass(frozen=True, slots=True) -class _FunctionProfile: - leaves: frozenset[str] - middles: frozenset[str] - - -def score_callgraph_shape(facts: ProjectFacts) -> SignalRanking: - """Rank package pairs by parallel-derivation call-graph shape.""" - - edges, node_modules = _build_call_graph(facts) - reachable: dict[str, frozenset[str]] = _reachable_sets(edges) - profiles: dict[str, _FunctionProfile] = _build_profiles(edges=edges, reachable=reachable) - document_frequency: dict[str, int] = _document_frequency(profiles) - profile_count: int = len(profiles) - - def inverse_frequency(item: str) -> float: - return math.log1p(profile_count / (1 + document_frequency.get(item, 0))) - - best_per_pair: dict[tuple[str, str], SignalPairScore] = {} - for left, right in _candidate_pairs(profiles=profiles, node_modules=node_modules): - if right in reachable.get(left, frozenset()) or left in reachable.get(right, frozenset()): - continue - pair_score: SignalPairScore | None = _score_function_pair( - left=left, - right=right, - profiles=profiles, - node_modules=node_modules, - inverse_frequency=inverse_frequency, - ) - if pair_score is None: - continue - existing: SignalPairScore | None = best_per_pair.get(pair_score.package_pair) - if existing is None or (-pair_score.score, pair_score.evidence) < ( - -existing.score, - existing.evidence, - ): - best_per_pair[pair_score.package_pair] = pair_score - - ordered: list[SignalPairScore] = sorted( - best_per_pair.values(), - key=lambda entry: (-entry.score, entry.package_pair, entry.evidence), - ) - return SignalRanking(signal_name=SIGNAL_NAME_CALLGRAPH, entries=tuple(ordered)) - - -def _build_call_graph(facts: ProjectFacts) -> tuple[dict[str, tuple[str, ...]], dict[str, str]]: - node_modules: dict[str, str] = {} - raw_edges: dict[str, tuple[str, ...]] = {} - for module_facts in facts.modules: - for function in module_facts.functions: - node_modules[function.qualified_name] = module_facts.module - raw_edges[function.qualified_name] = function.resolved_calls - edges: dict[str, tuple[str, ...]] = {} - for node, calls in raw_edges.items(): - retained: list[str] = [] - for call in calls: - if call in node_modules and call != node: - retained.append(call) - edges[node] = tuple(retained) - return edges, node_modules - - -def _reachable_sets(edges: dict[str, tuple[str, ...]]) -> dict[str, frozenset[str]]: - memo: dict[str, frozenset[str]] = {} - for root in sorted(edges): - if root in memo: - continue - stack: list[str] = [root] - visiting: set[str] = set() - pending: dict[str, list[str]] = {} - accumulated: dict[str, set[str]] = {} - while stack: - node: str = stack[-1] - if node in memo: - _ = stack.pop() - continue - if node not in accumulated: - accumulated[node] = set() - visiting.add(node) - pending[node] = sorted(edges.get(node, ()), reverse=True) - if pending[node]: - successor: str = pending[node].pop() - accumulated[node].add(successor) - if successor in memo: - accumulated[node] |= memo[successor] - elif successor not in visiting: - stack.append(successor) - continue - _ = stack.pop() - visiting.discard(node) - memo[node] = frozenset(accumulated[node]) - if stack: - parent: str = stack[-1] - accumulated[parent] |= memo[node] - return memo - - -def _build_profiles( - *, - edges: dict[str, tuple[str, ...]], - reachable: dict[str, frozenset[str]], -) -> dict[str, _FunctionProfile]: - leaves_all: set[str] = {node for node, calls in edges.items() if not calls} - profiles: dict[str, _FunctionProfile] = {} - for node, reached in reachable.items(): - if len(reached) < MIN_REACHABLE_FUNCTIONS: - continue - leaves: frozenset[str] = frozenset(item for item in reached if item in leaves_all) - if len(leaves) < MIN_SHARED_LEAVES: - continue - middles: frozenset[str] = frozenset(item for item in reached if item not in leaves_all) - profiles[node] = _FunctionProfile(leaves=leaves, middles=middles) - return profiles - - -def _document_frequency(profiles: dict[str, _FunctionProfile]) -> dict[str, int]: - frequency: dict[str, int] = {} - for profile in profiles.values(): - for item in profile.leaves | profile.middles: - frequency[item] = frequency.get(item, 0) + 1 - return frequency - - -def _candidate_pairs( - *, - profiles: dict[str, _FunctionProfile], - node_modules: dict[str, str], -) -> list[tuple[str, str]]: - users_by_leaf: dict[str, list[str]] = {} - for node in sorted(profiles): - for leaf in profiles[node].leaves: - users_by_leaf.setdefault(leaf, []).append(node) - pairs: set[tuple[str, str]] = set() - for users in users_by_leaf.values(): - if len(users) > MAX_LEAF_USERS_FOR_PAIRING: - continue - for left, right in combinations(sorted(users), 2): - if node_modules[left] != node_modules[right]: - pairs.add((left, right)) - return sorted(pairs) - - -def _score_function_pair( - *, - left: str, - right: str, - profiles: dict[str, _FunctionProfile], - node_modules: dict[str, str], - inverse_frequency: Callable[[str], float], -) -> SignalPairScore | None: - left_profile: _FunctionProfile = profiles[left] - right_profile: _FunctionProfile = profiles[right] - shared_leaves: frozenset[str] = left_profile.leaves & right_profile.leaves - if len(shared_leaves) < MIN_SHARED_LEAVES: - return None - shared_weight: float = sum(inverse_frequency(item) for item in shared_leaves) - union_weight: float = sum( - inverse_frequency(item) for item in left_profile.leaves | right_profile.leaves - ) - if union_weight <= 0: - return None - leaf_similarity: float = shared_weight / union_weight - middle_union: frozenset[str] = left_profile.middles | right_profile.middles - middle_shared_weight: float = sum( - inverse_frequency(item) for item in left_profile.middles & right_profile.middles - ) - middle_union_weight: float = sum(inverse_frequency(item) for item in middle_union) - middle_similarity: float = ( - middle_shared_weight / middle_union_weight if middle_union_weight > 0 else 1.0 - ) - score: float = leaf_similarity * (1.0 - middle_similarity) * math.sqrt(max(shared_weight, 0.0)) - if score <= 0: - return None - rare_shared: list[str] = sorted( - shared_leaves, key=lambda item: (-inverse_frequency(item), item) - ) - evidence: tuple[str, ...] = ( - f"entrypoints {left} <-> {right}", - "shared rare leaves: " + ", ".join(rare_shared[:_EVIDENCE_LEAF_SAMPLE]), - ) - package_pair: tuple[str, str] = sorted_pair( - left=package_of(node_modules[left]), - right=package_of(node_modules[right]), - ) - return SignalPairScore(package_pair=package_pair, score=score, evidence=evidence) diff --git a/scripts/dupscore/_helpers/package_pairs/signal_cochange.py b/scripts/dupscore/_helpers/package_pairs/signal_cochange.py deleted file mode 100644 index 55684d24c1..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/signal_cochange.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Signal: git co-change coupling between packages without import coupling.""" - -from __future__ import annotations - -import subprocess -from itertools import combinations -from pathlib import Path - -from scripts.dupscore._helpers.inputs.source_provider import module_name_for, package_of -from scripts.dupscore.constants import ( - MAX_FILES_PER_COMMIT, - MIN_COCHANGES, - SIGNAL_NAME_COCHANGE, -) -from scripts.dupscore.exceptions import DupscoreGitError -from scripts.dupscore.models import ProjectFacts, SignalPairScore, SignalRanking - -_COMMIT_SENTINEL: str = "\x01" -_MIN_FILES_PER_COMMIT: int = 2 - - -def score_cochange(*, facts: ProjectFacts, repo_root: Path, revision: str) -> SignalRanking: - """Rank cross-package module pairs that co-change without import coupling.""" - - commits: list[list[str]] = _read_commit_modules(repo_root=repo_root, revision=revision) - live_modules: set[str] = {module_facts.module for module_facts in facts.modules} - import_edges: set[tuple[str, str]] = _module_import_edges(facts) - - pair_counts: dict[tuple[str, str], int] = {} - change_counts: dict[str, int] = {} - for commit_modules in commits: - unique_modules: list[str] = sorted(set(commit_modules)) - for module in unique_modules: - change_counts[module] = change_counts.get(module, 0) + 1 - if not (_MIN_FILES_PER_COMMIT <= len(unique_modules) <= MAX_FILES_PER_COMMIT): - continue - for left, right in combinations(unique_modules, 2): - pair_counts[(left, right)] = pair_counts.get((left, right), 0) + 1 - - best_per_pair: dict[tuple[str, str], SignalPairScore] = {} - for (left, right), count in sorted(pair_counts.items()): - if count < MIN_COCHANGES: - continue - if left not in live_modules or right not in live_modules: - continue - if (left, right) in import_edges or (right, left) in import_edges: - continue - left_package: str = package_of(left) - right_package: str = package_of(right) - if left_package == right_package: - continue - confidence: float = count / min(change_counts[left], change_counts[right]) - score: float = count * confidence - cochange_summary: str = f"co-changed in {count} commits (confidence {confidence:.2f})" - evidence: tuple[str, ...] = (f"modules {left} <-> {right} {cochange_summary}",) - package_pair: tuple[str, str] = ( - (left_package, right_package) - if left_package <= right_package - else (right_package, left_package) - ) - candidate: SignalPairScore = SignalPairScore( - package_pair=package_pair, score=score, evidence=evidence - ) - existing: SignalPairScore | None = best_per_pair.get(package_pair) - if existing is None or (-candidate.score, candidate.evidence) < ( - -existing.score, - existing.evidence, - ): - best_per_pair[package_pair] = candidate - - ordered: list[SignalPairScore] = sorted( - best_per_pair.values(), - key=lambda entry: (-entry.score, entry.package_pair, entry.evidence), - ) - return SignalRanking(signal_name=SIGNAL_NAME_COCHANGE, entries=tuple(ordered)) - - -def _read_commit_modules(*, repo_root: Path, revision: str) -> list[list[str]]: - completed: subprocess.CompletedProcess[str] = subprocess.run( - ["git", "log", f"--format={_COMMIT_SENTINEL}%H", "--name-only", revision], - cwd=repo_root, - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - raise DupscoreGitError(f"git log for {revision} failed: {completed.stderr.strip()}") - commits: list[list[str]] = [] - current: list[str] = [] - for line in completed.stdout.splitlines(): - if line.startswith(_COMMIT_SENTINEL): - if current: - commits.append(current) - current = [] - continue - stripped: str = line.strip() - if not stripped: - continue - module: str | None = module_name_for(stripped) - if module is not None: - current.append(module) - if current: - commits.append(current) - return commits - - -def _module_import_edges(facts: ProjectFacts) -> set[tuple[str, str]]: - live_modules: set[str] = {module_facts.module for module_facts in facts.modules} - edges: set[tuple[str, str]] = set() - for module_facts in facts.modules: - for target in module_facts.imported_modules: - if not target.startswith("sqlbuild"): - continue - candidate: str = target - while candidate and candidate not in live_modules: - candidate = candidate.rpartition(".")[0] - if candidate: - edges.add((module_facts.module, candidate)) - return edges diff --git a/scripts/dupscore/_helpers/package_pairs/signal_dataclasses.py b/scripts/dupscore/_helpers/package_pairs/signal_dataclasses.py deleted file mode 100644 index 08745f0496..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/signal_dataclasses.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Signal: public dataclass field-name overlap across modules.""" - -from __future__ import annotations - -from itertools import combinations - -from scripts.dupscore._helpers.inputs.source_provider import package_of, sorted_pair -from scripts.dupscore.constants import MIN_SHARED_DATACLASS_FIELDS, SIGNAL_NAME_DATACLASS_OVERLAP -from scripts.dupscore.models import ProjectFacts, SignalPairScore, SignalRanking - - -def score_dataclass_overlap(facts: ProjectFacts) -> SignalRanking: - """Rank package pairs whose public models share large field-name sets.""" - - models: list[tuple[str, str, frozenset[str]]] = [] - for module_facts in facts.modules: - for class_fact in module_facts.classes: - eligible: bool = ( - class_fact.dataclass_like - and class_fact.public - and len(class_fact.field_names) >= MIN_SHARED_DATACLASS_FIELDS - ) - if eligible: - models.append( - ( - class_fact.qualified_name, - module_facts.module, - frozenset(class_fact.field_names), - ) - ) - - best_per_pair: dict[tuple[str, str], SignalPairScore] = {} - for (left_qual, left_module, left_fields), ( - right_qual, - right_module, - right_fields, - ) in combinations(models, 2): - if left_module == right_module: - continue - shared: frozenset[str] = left_fields & right_fields - if len(shared) < MIN_SHARED_DATACLASS_FIELDS: - continue - jaccard: float = len(shared) / len(left_fields | right_fields) - score: float = jaccard * len(shared) - evidence: tuple[str, ...] = ( - f"models {left_qual} <-> {right_qual}", - f"{len(shared)} shared fields: " + ", ".join(sorted(shared)), - ) - package_pair: tuple[str, str] = sorted_pair( - left=package_of(left_module), - right=package_of(right_module), - ) - candidate: SignalPairScore = SignalPairScore( - package_pair=package_pair, score=score, evidence=evidence - ) - existing: SignalPairScore | None = best_per_pair.get(package_pair) - if existing is None or (-candidate.score, candidate.evidence) < ( - -existing.score, - existing.evidence, - ): - best_per_pair[package_pair] = candidate - - ordered: list[SignalPairScore] = sorted( - best_per_pair.values(), - key=lambda entry: (-entry.score, entry.package_pair, entry.evidence), - ) - return SignalRanking(signal_name=SIGNAL_NAME_DATACLASS_OVERLAP, entries=tuple(ordered)) diff --git a/scripts/dupscore/_helpers/package_pairs/signal_same_names.py b/scripts/dupscore/_helpers/package_pairs/signal_same_names.py deleted file mode 100644 index 6336db2356..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/signal_same_names.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Signal: identical public function names across modules, ranked by body similarity.""" - -from __future__ import annotations - -from itertools import combinations - -from scripts.dupscore._helpers.inputs.source_provider import package_of, sorted_pair -from scripts.dupscore.constants import ( - GENERIC_FUNCTION_NAMES, - HELPERS_ROLE_SEGMENT, - MAIN_ROLE_SEGMENT, - MAX_NAME_WEIGHT_WORDS, - SIGNAL_NAME_SAME_NAME, -) -from scripts.dupscore.models import FunctionFact, ProjectFacts, SignalPairScore, SignalRanking - -_MIN_DEFINITIONS_FOR_TWIN: int = 2 - - -def score_same_name_symbols(facts: ProjectFacts) -> SignalRanking: - """Rank package pairs defining identically named, similarly bodied public functions.""" - - functions_by_name: dict[str, list[FunctionFact]] = {} - for module_facts in facts.modules: - for function in module_facts.functions: - if function.public and function.name not in GENERIC_FUNCTION_NAMES: - functions_by_name.setdefault(function.name, []).append(function) - - best_per_pair: dict[tuple[str, str], SignalPairScore] = {} - for name in sorted(functions_by_name): - definitions: list[FunctionFact] = functions_by_name[name] - if len(definitions) < _MIN_DEFINITIONS_FOR_TWIN: - continue - for left, right in combinations(definitions, 2): - if left.module == right.module: - continue - if _is_wrapper_twin(left=left, right=right): - continue - union_tokens: frozenset[str] = left.body_tokens | right.body_tokens - if not union_tokens: - continue - similarity: float = len(left.body_tokens & right.body_tokens) / len(union_tokens) - name_weight: float = ( - min(len(name.split("_")), MAX_NAME_WEIGHT_WORDS) / MAX_NAME_WEIGHT_WORDS - ) - score: float = similarity * name_weight - if score <= 0: - continue - evidence: tuple[str, ...] = ( - f"twin {name}: {left.module}:{left.lineno} <-> {right.module}:{right.lineno}", - f"body token similarity {similarity:.2f}", - ) - package_pair: tuple[str, str] = sorted_pair( - left=package_of(left.module), - right=package_of(right.module), - ) - candidate: SignalPairScore = SignalPairScore( - package_pair=package_pair, score=score, evidence=evidence - ) - existing: SignalPairScore | None = best_per_pair.get(package_pair) - if existing is None or (-candidate.score, candidate.evidence) < ( - -existing.score, - existing.evidence, - ): - best_per_pair[package_pair] = candidate - - ordered: list[SignalPairScore] = sorted( - best_per_pair.values(), - key=lambda entry: (-entry.score, entry.package_pair, entry.evidence), - ) - return SignalRanking(signal_name=SIGNAL_NAME_SAME_NAME, entries=tuple(ordered)) - - -def _is_wrapper_twin(*, left: FunctionFact, right: FunctionFact) -> bool: - if package_of(left.module) != package_of(right.module): - return False - left_segments: frozenset[str] = frozenset(left.module.split(".")) - right_segments: frozenset[str] = frozenset(right.module.split(".")) - left_is_main: bool = MAIN_ROLE_SEGMENT in left_segments - right_is_main: bool = MAIN_ROLE_SEGMENT in right_segments - left_is_helper: bool = HELPERS_ROLE_SEGMENT in left_segments - right_is_helper: bool = HELPERS_ROLE_SEGMENT in right_segments - return (left_is_main and right_is_helper) or (left_is_helper and right_is_main) diff --git a/scripts/dupscore/_helpers/package_pairs/signal_state_fanin.py b/scripts/dupscore/_helpers/package_pairs/signal_state_fanin.py deleted file mode 100644 index 0ee78476c0..0000000000 --- a/scripts/dupscore/_helpers/package_pairs/signal_state_fanin.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Signal: distinct calling packages per configured persisted-state read method.""" - -from __future__ import annotations - -from itertools import combinations - -from scripts.dupscore._helpers.inputs.source_provider import package_of -from scripts.dupscore.constants import ( - MIN_STATE_FANIN_PACKAGES, - SIGNAL_NAME_STATE_FANIN, - STATE_READ_PREFIXES, -) -from scripts.dupscore.models import ( - DupscoreConfig, - FunctionFact, - ProjectFacts, - SignalPairScore, - SignalRanking, -) - - -def score_state_fanin(*, facts: ProjectFacts, config: DupscoreConfig) -> SignalRanking: - """Rank package pairs that independently read the same persisted-state surface.""" - - if not config.persisted_state_surfaces: - return SignalRanking(signal_name=SIGNAL_NAME_STATE_FANIN, entries=()) - read_owners: dict[str, str] = _collect_state_read_methods(facts=facts, config=config) - calling_packages: dict[str, set[str]] = {name: set() for name in read_owners} - for module_facts in facts.modules: - caller_package: str = package_of(module_facts.module) - all_functions: list[FunctionFact] = list(module_facts.functions) - for class_fact in module_facts.classes: - all_functions.extend(class_fact.methods) - for function in all_functions: - for bare_name in function.bare_attribute_calls: - if bare_name in calling_packages: - calling_packages[bare_name].add(caller_package) - for call in function.resolved_calls: - called_name: str = call.split("::")[-1] - if called_name in calling_packages: - calling_packages[called_name].add(caller_package) - - pair_scores: dict[tuple[str, str], list[str]] = {} - for method_name in sorted(read_owners): - owner: str = read_owners[method_name] - external: list[str] = sorted( - caller for caller in calling_packages[method_name] if not owner.startswith(caller) - ) - if len(external) < MIN_STATE_FANIN_PACKAGES: - continue - for left, right in combinations(external, 2): - pair_scores.setdefault((left, right), []).append(method_name) - - entries: list[SignalPairScore] = [] - for pair in sorted(pair_scores): - methods: list[str] = pair_scores[pair] - evidence: tuple[str, ...] = ("co-read state methods: " + ", ".join(methods),) - entries.append( - SignalPairScore(package_pair=pair, score=float(len(methods)), evidence=evidence) - ) - entries.sort(key=lambda entry: (-entry.score, entry.package_pair)) - return SignalRanking(signal_name=SIGNAL_NAME_STATE_FANIN, entries=tuple(entries)) - - -def _collect_state_read_methods(*, facts: ProjectFacts, config: DupscoreConfig) -> dict[str, str]: - owners: dict[str, str] = {} - for module_facts in facts.modules: - if not _is_state_surface(module=module_facts.module, config=config): - continue - for class_fact in module_facts.classes: - for method in class_fact.methods: - if method.name.startswith(STATE_READ_PREFIXES): - owners.setdefault(method.name, class_fact.qualified_name) - for function in module_facts.functions: - if function.public and function.name.startswith(STATE_READ_PREFIXES): - owners.setdefault(function.name, module_facts.module) - return owners - - -def _is_state_surface(*, module: str, config: DupscoreConfig) -> bool: - for surface in config.persisted_state_surfaces: - if module == surface or module.startswith(surface + "."): - return True - return False diff --git a/scripts/dupscore/constants.py b/scripts/dupscore/constants.py deleted file mode 100644 index 4c9b5f88ff..0000000000 --- a/scripts/dupscore/constants.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Tuned scoring constants for the dupscore duplication-risk advisory tool.""" - -from __future__ import annotations - -SOURCE_ROOT: str = "src" -PROJECT_PACKAGE: str = "sqlbuild" -EXCLUDED_MODULE_PREFIX: str = "sqlbuild.integrations.dbt" -PACKAGE_DEPTH: int = 3 - -MIN_REACHABLE_FUNCTIONS: int = 5 -MIN_SHARED_LEAVES: int = 3 -MAX_LEAF_USERS_FOR_PAIRING: int = 200 - -MIN_SHARED_DATACLASS_FIELDS: int = 4 - -MIN_SAME_NAME_WORDS: int = 1 -MAX_NAME_WEIGHT_WORDS: int = 6 - -STATE_READ_PREFIXES: tuple[str, ...] = ("get_", "read_", "fetch_", "list_", "iter_", "load_") -MIN_STATE_FANIN_PACKAGES: int = 2 - -MAX_FILES_PER_COMMIT: int = 30 -MIN_COCHANGES: int = 5 - -SIGNAL_NAME_CALLGRAPH: str = "callgraph_shape" -SIGNAL_NAME_STATE_FANIN: str = "state_fanin" -SIGNAL_NAME_DATACLASS_OVERLAP: str = "dataclass_overlap" -SIGNAL_NAME_SAME_NAME: str = "same_name_symbols" -SIGNAL_NAME_COCHANGE: str = "cochange" - -RRF_RANK_OFFSET: int = 10 -SIGNAL_WEIGHTS: dict[str, float] = { - SIGNAL_NAME_CALLGRAPH: 1.0, - SIGNAL_NAME_STATE_FANIN: 1.0, - SIGNAL_NAME_DATACLASS_OVERLAP: 1.0, - SIGNAL_NAME_SAME_NAME: 1.0, - SIGNAL_NAME_COCHANGE: 0.75, -} - -GENERIC_FUNCTION_NAMES: frozenset[str] = frozenset( - {"main", "run", "build", "execute", "register", "create", "apply", "render"} -) - -DEFAULT_TOP_RESULTS: int = 20 -CONFIG_FILENAME: str = "dupscore.toml" - -CLONES_MODE: str = "clones" -REPORT_MODE: str = "report" -PAIR_MODE: str = "pair" -PAIR_ARGUMENT_COUNT: int = 2 - -WORKTREE_LABEL: str = "worktree" -HEAD_REVISION: str = "HEAD" - -HELPERS_ROLE_SEGMENT: str = "_helpers" -MAIN_ROLE_SEGMENT: str = "main" - -LANGUAGE_PYTHON: str = "python" -LANGUAGE_RUST: str = "rust" -SUPPORTED_LANGUAGES: tuple[str, ...] = (LANGUAGE_PYTHON, LANGUAGE_RUST) -PYTHON_TEST_ROOT: str = "tests" -RUST_CRATES_ROOT: str = "crates" -RUST_SOURCE_DIRECTORY: str = "src" -RUST_TEST_DIRECTORIES: frozenset[str] = frozenset({"tests", "benches"}) - -CLONE_KGRAM_SIZE: int = 12 -CLONE_WINNOW_WINDOW: int = 6 -CLONE_MIN_FINGERPRINTS: int = 6 -CLONE_CANDIDATE_MIN_JACCARD: float = 0.2 -CLONE_MAX_FINGERPRINT_UNITS: int = 25 -CLONE_SMALL_NEAR_MISS_TOKENS: int = 80 -CLONE_SMALL_NEAR_MISS_MIN_SIMILARITY: float = 0.9 -DEFAULT_CLONE_MIN_TOKENS: int = 60 -DEFAULT_CLONE_MIN_SIMILARITY: float = 0.8 -DEFAULT_CLONE_TOP_RESULTS: int = 30 -CLONE_TEXT_MAX_MEMBERS: int = 12 - -CATEGORY_EXACT: str = "exact" -CATEGORY_RENAMED: str = "renamed" -CATEGORY_NEAR_MISS: str = "near-miss" -CATEGORY_ORDER: tuple[str, ...] = (CATEGORY_EXACT, CATEGORY_RENAMED, CATEGORY_NEAR_MISS) - -CHANGE_NEW: str = "new" -CHANGE_CHANGED: str = "changed" - -PLACEHOLDER_IDENTIFIER: str = "$id" -PLACEHOLDER_NUMBER: str = "$num" -PLACEHOLDER_STRING: str = "$str" -PLACEHOLDER_FSTRING: str = "$fstr" -PLACEHOLDER_BYTE_STRING: str = "$bstr" -PLACEHOLDER_CHAR: str = "$char" -PLACEHOLDER_BYTE: str = "$byte" -PLACEHOLDER_LIFETIME: str = "$life" - -RUST_KIND_IDENTIFIER: str = "identifier" -RUST_KIND_LIFETIME: str = "lifetime" -RUST_KIND_STRING: str = "string" -RUST_KIND_BYTE_STRING: str = "byte_string" -RUST_KIND_CHAR: str = "char" -RUST_KIND_BYTE: str = "byte" -RUST_KIND_NUMBER: str = "number" -RUST_KIND_PUNCTUATION: str = "punctuation" diff --git a/scripts/dupscore/dupscore.toml b/scripts/dupscore/dupscore.toml deleted file mode 100644 index e4e2061317..0000000000 --- a/scripts/dupscore/dupscore.toml +++ /dev/null @@ -1,63 +0,0 @@ -persisted_state_surfaces = [ - "sqlbuild.virtual.state", - "sqlbuild.adapter.state_sql", - "sqlbuild.compiler.source_freshness", - "sqlbuild.compiler.node_source_watermarks", -] - -[[allowlist]] -pair = ["sqlbuild.integrations.dagster", "sqlbuild.integrations.rivers"] -reason = "Deliberately mirrored orchestrator integrations with parallel external APIs." - -[[allowlist]] -pair = ["sqlbuild.virtual.state", "sqlbuild.virtual.state"] -reason = "DuckDB and Postgres state backends intentionally implement one storage contract." - -[[allowlist]] -pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.bigquery"] -reason = "Adapter backends intentionally conform to the shared adapter contract." - -[[allowlist]] -pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.databricks"] -reason = "Adapter backends intentionally conform to the shared adapter contract." - -[[allowlist]] -pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.duckdb"] -reason = "Adapter backends intentionally conform to the shared adapter contract." - -[[allowlist]] -pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.motherduck"] -reason = "Adapter backends intentionally conform to the shared adapter contract." - -[[allowlist]] -pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.postgres"] -reason = "Adapter backends intentionally conform to the shared adapter contract." - -[[allowlist]] -pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.snowflake"] -reason = "Adapter backends intentionally conform to the shared adapter contract." - -[[allowlist]] -pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.sqlserver"] -reason = "Adapter backends intentionally conform to the shared adapter contract." - -# Clone mode: suppress clone pairs whose two sides both match one entry's globs. -# `*` also crosses directory separators. Every entry needs a non-empty reason. -# -# [[clone_allowlist]] -# paths = ["src/sqlbuild/example/backends/*"] -# reason = "Backends intentionally mirror one protocol." - -# Clone mode: a method is a forced override when it is named after an abstract method of -# `contract` (own or inherited, parsed statically from the analysed source), is defined in a -# class under `paths` that derives from `contract`, and would otherwise be inherited from -# `contract`, a `forbidden_owners` class, or nowhere. Links between two forced overrides are -# hidden; links to any other unit are still reported. Every entry needs a non-empty reason. -[[contract_exemption]] -contract = "src/sqlbuild/adapter/contract/classes/strict_adapter.py:StrictAdapter" -paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/classes/*"] -forbidden_owners = ["src/sqlbuild/adapter/contract/classes/base_adapter.py:BaseAdapter"] -reason = """\ -tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires \ -every first-class adapter to define each StrictAdapter abstract method itself instead of \ -inheriting it from BaseAdapter or StrictAdapter, so identical overrides are intentional.""" diff --git a/scripts/dupscore/exceptions.py b/scripts/dupscore/exceptions.py deleted file mode 100644 index 741549e703..0000000000 --- a/scripts/dupscore/exceptions.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Structured errors for the dupscore duplication-risk advisory tool.""" - -from __future__ import annotations - - -class DupscoreConfigError(ValueError): - """Raised when the dupscore TOML configuration is missing or invalid.""" - - -class DupscoreGitError(RuntimeError): - """Raised when a git query required by dupscore fails.""" - - -class DupscoreUsageError(ValueError): - """Raised when CLI arguments cannot be interpreted.""" diff --git a/scripts/dupscore/main/__init__.py b/scripts/dupscore/main/__init__.py deleted file mode 100644 index ea5f482d61..0000000000 --- a/scripts/dupscore/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Dupscore CLI and report entrypoints.""" diff --git a/scripts/dupscore/main/build_clone_report.py b/scripts/dupscore/main/build_clone_report.py deleted file mode 100644 index 6a04ffa1ee..0000000000 --- a/scripts/dupscore/main/build_clone_report.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Build the function-level clone report for the worktree.""" - -from __future__ import annotations - -from collections import Counter -from pathlib import Path - -from scripts.dupscore._helpers.clones.clustering import cluster_clone_pairs -from scripts.dupscore._helpers.clones.filters import ( - allowlist_reason, - classify_unit_change, - collect_changes_since, - path_matches_any, -) -from scripts.dupscore._helpers.clones.fingerprints import fingerprint_tokens -from scripts.dupscore._helpers.clones.matching import find_clone_pairs -from scripts.dupscore._helpers.clones.sources import collect_clone_units -from scripts.dupscore._helpers.contracts.forced_overrides import find_forced_overrides -from scripts.dupscore.models import ( - CloneCluster, - CloneOptions, - ClonePair, - CloneReport, - CloneUnit, - DupscoreConfig, - FileChanges, -) - - -def build_clone_report( - *, - repo_root: Path, - options: CloneOptions, - config: DupscoreConfig, -) -> CloneReport: - """Detect exact, renamed, and near-miss function clones and group them into clusters.""" - - units: list[CloneUnit] = [ - unit - for unit in collect_clone_units( - repo_root=repo_root, - languages=options.languages, - include_tests=options.include_tests, - ) - if len(unit.normalized) >= options.min_tokens - ] - fingerprints: list[frozenset[int]] = [fingerprint_tokens(unit.normalized) for unit in units] - candidate_pairs: list[ClonePair] = find_clone_pairs( - units=units, - fingerprints=fingerprints, - min_similarity=options.min_similarity, - ) - paired: list[int] = sorted(_paired_units(candidate_pairs)) - flags: tuple[bool, ...] = find_forced_overrides( - repo_root=repo_root, - entries=config.contract_exemptions, - units=[units[unit_index] for unit_index in paired], - ) - forced: set[int] = {unit_index for unit_index, flag in zip(paired, flags, strict=True) if flag} - retained_pairs: list[ClonePair] = [ - pair for pair in candidate_pairs if pair.left not in forced or pair.right not in forced - ] - pairs: list[ClonePair] = [ - pair - for pair in retained_pairs - if allowlist_reason( - left_path=units[pair.left].path, - right_path=units[pair.right].path, - entries=config.clone_allowlist, - ) - is None - ] - changes: dict[str, FileChanges] | None = ( - collect_changes_since(repo_root=repo_root, revision=options.since) - if options.since is not None - else None - ) - - def change_of(unit: CloneUnit) -> str | None: - if changes is None: - return None - return classify_unit_change(unit=unit, changes=changes.get(unit.path)) - - clusters: list[CloneCluster] = [ - cluster - for cluster in cluster_clone_pairs( - units=units, pairs=pairs, change_of=change_of, forced=frozenset(forced) - ) - if _cluster_selected(cluster=cluster, options=options) - ] - unit_counts: Counter[str] = Counter(unit.language for unit in units) - return CloneReport( - since=options.since, - unit_counts={language: unit_counts[language] for language in options.languages}, - allowlisted_pairs=len(retained_pairs) - len(pairs), - contract_exempt_members=len(forced - _paired_units(retained_pairs)), - clusters=tuple(clusters), - ) - - -def _paired_units(pairs: list[ClonePair]) -> set[int]: - indexes: set[int] = set() - for pair in pairs: - indexes.update((pair.left, pair.right)) - return indexes - - -def _cluster_selected(*, cluster: CloneCluster, options: CloneOptions) -> bool: - if options.since is not None and all(member.change is None for member in cluster.members): - return False - if options.path_globs and not any( - path_matches_any(path=member.path, globs=options.path_globs) for member in cluster.members - ): - return False - return True diff --git a/scripts/dupscore/main/build_pair_evidence.py b/scripts/dupscore/main/build_pair_evidence.py deleted file mode 100644 index 52afcbe60e..0000000000 --- a/scripts/dupscore/main/build_pair_evidence.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Extract drill-down evidence for one package pair from a report.""" - -from __future__ import annotations - -from scripts.dupscore.models import DupscoreReport, PairEvidenceReport - - -def build_pair_evidence(*, report: DupscoreReport, left: str, right: str) -> PairEvidenceReport: - """Collect the combined rank and per-signal evidence for one package pair.""" - - requested: tuple[str, str] = (left, right) if left <= right else (right, left) - for position, entry in enumerate(report.entries, start=1): - if entry.package_pair == requested: - return PairEvidenceReport( - package_pair=requested, - combined_rank=position, - contributions=entry.contributions, - ) - return PairEvidenceReport(package_pair=requested, combined_rank=None, contributions=()) diff --git a/scripts/dupscore/main/build_report.py b/scripts/dupscore/main/build_report.py deleted file mode 100644 index 7232ba4fe5..0000000000 --- a/scripts/dupscore/main/build_report.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Build the fused duplication-risk report for one revision.""" - -from __future__ import annotations - -from pathlib import Path - -from scripts.dupscore._helpers.inputs.source_provider import ( - read_revision_sources, - read_worktree_sources, -) -from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts -from scripts.dupscore._helpers.package_pairs.fusion import fuse_rankings -from scripts.dupscore._helpers.package_pairs.signal_callgraph import score_callgraph_shape -from scripts.dupscore._helpers.package_pairs.signal_cochange import score_cochange -from scripts.dupscore._helpers.package_pairs.signal_dataclasses import score_dataclass_overlap -from scripts.dupscore._helpers.package_pairs.signal_same_names import score_same_name_symbols -from scripts.dupscore._helpers.package_pairs.signal_state_fanin import score_state_fanin -from scripts.dupscore.constants import HEAD_REVISION, WORKTREE_LABEL -from scripts.dupscore.models import DupscoreConfig, DupscoreReport, ProjectFacts, SignalRanking - - -def build_report( - *, - repo_root: Path, - revision: str | None, - config: DupscoreConfig, -) -> DupscoreReport: - """Score all signals for one revision and fuse them into a ranked report.""" - - sources: dict[str, str] = ( - read_worktree_sources(repo_root) - if revision is None - else read_revision_sources(repo_root=repo_root, revision=revision) - ) - facts: ProjectFacts = extract_project_facts(sources) - log_revision: str = revision if revision is not None else HEAD_REVISION - rankings: tuple[SignalRanking, ...] = ( - score_callgraph_shape(facts), - score_state_fanin(facts=facts, config=config), - score_dataclass_overlap(facts), - score_same_name_symbols(facts), - score_cochange(facts=facts, repo_root=repo_root, revision=log_revision), - ) - revision_label: str = revision if revision is not None else WORKTREE_LABEL - return fuse_rankings(rankings=rankings, config=config, revision_label=revision_label) diff --git a/scripts/dupscore/main/build_report_delta.py b/scripts/dupscore/main/build_report_delta.py deleted file mode 100644 index 4ac9ba9ff2..0000000000 --- a/scripts/dupscore/main/build_report_delta.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Compare two revision reports and describe the material differences.""" - -from __future__ import annotations - -from scripts.dupscore.constants import SIGNAL_NAME_SAME_NAME, SIGNAL_NAME_STATE_FANIN -from scripts.dupscore.models import CombinedPairEntry, DupscoreReport, ReportDelta - - -def build_report_delta(*, base: DupscoreReport, current: DupscoreReport, top: int) -> ReportDelta: - """Diff two reports: top-list membership and new high-precision evidence.""" - - base_top_pairs: set[tuple[str, str]] = {entry.package_pair for entry in base.entries[:top]} - current_top_pairs: set[tuple[str, str]] = { - entry.package_pair for entry in current.entries[:top] - } - entered: tuple[CombinedPairEntry, ...] = tuple( - entry for entry in current.entries[:top] if entry.package_pair not in base_top_pairs - ) - left_top: tuple[CombinedPairEntry, ...] = tuple( - entry for entry in base.entries[:top] if entry.package_pair not in current_top_pairs - ) - new_state_evidence: tuple[str, ...] = _new_signal_evidence( - base=base, - current=current, - signal_name=SIGNAL_NAME_STATE_FANIN, - ) - new_same_name_evidence: tuple[str, ...] = _new_signal_evidence( - base=base, - current=current, - signal_name=SIGNAL_NAME_SAME_NAME, - ) - return ReportDelta( - base_label=base.revision_label, - current_label=current.revision_label, - entered_top=entered, - left_top=left_top, - new_state_fanin_evidence=new_state_evidence, - new_same_name_evidence=new_same_name_evidence, - ) - - -def _new_signal_evidence( - *, - base: DupscoreReport, - current: DupscoreReport, - signal_name: str, -) -> tuple[str, ...]: - base_evidence: set[str] = _signal_evidence(report=base, signal_name=signal_name) - current_evidence: set[str] = _signal_evidence(report=current, signal_name=signal_name) - return tuple(sorted(current_evidence - base_evidence)) - - -def _signal_evidence(*, report: DupscoreReport, signal_name: str) -> set[str]: - evidence: set[str] = set() - for entry in report.entries: - for contribution in entry.contributions: - if contribution.signal_name == signal_name and contribution.evidence: - evidence.add(contribution.evidence[0]) - return evidence diff --git a/scripts/dupscore/main/dupscore.py b/scripts/dupscore/main/dupscore.py deleted file mode 100644 index 956c943c7a..0000000000 --- a/scripts/dupscore/main/dupscore.py +++ /dev/null @@ -1,107 +0,0 @@ -"""CLI entry for the dupscore duplication advisory tool (see scripts/dupscore/README.md).""" - -from __future__ import annotations - -import argparse -import sys -import time -from pathlib import Path - -from scripts.dupscore._helpers.clones.rendering import render_clone_json, render_clone_text -from scripts.dupscore._helpers.inputs.cli import clone_options_from, parse_arguments -from scripts.dupscore._helpers.inputs.config import load_config -from scripts.dupscore._helpers.package_pairs.fusion import filter_report_to_domain -from scripts.dupscore._helpers.package_pairs.rendering import ( - render_delta_json, - render_delta_text, - render_pair_json, - render_pair_text, - render_report_json, - render_report_text, -) -from scripts.dupscore.constants import ( - CLONES_MODE, - CONFIG_FILENAME, - DEFAULT_CLONE_TOP_RESULTS, - DEFAULT_TOP_RESULTS, - PAIR_ARGUMENT_COUNT, - PAIR_MODE, -) -from scripts.dupscore.exceptions import DupscoreUsageError -from scripts.dupscore.main.build_clone_report import build_clone_report -from scripts.dupscore.main.build_pair_evidence import build_pair_evidence -from scripts.dupscore.main.build_report import build_report -from scripts.dupscore.main.build_report_delta import build_report_delta -from scripts.dupscore.models import ( - CloneReport, - DupscoreConfig, - DupscoreReport, - PairEvidenceReport, - ReportDelta, -) - - -def dupscore(argv: list[str] | None = None) -> int: - """Run the dupscore CLI and print the requested advisory output.""" - - args: argparse.Namespace = parse_arguments(argv) - config_path: Path = Path(__file__).resolve().parents[1] / CONFIG_FILENAME - config: DupscoreConfig = load_config(config_path) - repo_root: Path = ( - args.repo_root.resolve() - if args.repo_root is not None - else Path(__file__).resolve().parents[3] - ) - output: str = ( - _execute_clones(args=args, config=config, repo_root=repo_root) - if args.mode == CLONES_MODE - else _execute(args=args, config=config, repo_root=repo_root) - ) - print(output) - return 0 - - -def _execute(*, args: argparse.Namespace, config: DupscoreConfig, repo_root: Path) -> str: - top: int = args.top if args.top is not None else DEFAULT_TOP_RESULTS - current: DupscoreReport = build_report(repo_root=repo_root, revision=None, config=config) - if args.mode == PAIR_MODE: - if len(args.packages) != PAIR_ARGUMENT_COUNT: - raise DupscoreUsageError("pair mode requires exactly two package names") - evidence: PairEvidenceReport = build_pair_evidence( - report=current, - left=args.packages[0], - right=args.packages[1], - ) - return render_pair_json(evidence) if args.as_json else render_pair_text(evidence) - if args.since is not None: - base: DupscoreReport = build_report(repo_root=repo_root, revision=args.since, config=config) - delta: ReportDelta = build_report_delta(base=base, current=current, top=top) - return render_delta_json(delta) if args.as_json else render_delta_text(delta) - filtered: DupscoreReport = filter_report_to_domain(report=current, domain=args.domain) - return ( - render_report_json(report=filtered, top=top) - if args.as_json - else render_report_text(report=filtered, top=top) - ) - - -def _execute_clones(*, args: argparse.Namespace, config: DupscoreConfig, repo_root: Path) -> str: - top: int = args.top if args.top is not None else DEFAULT_CLONE_TOP_RESULTS - print(f"dupscore: detecting function clones in {repo_root} ...", file=sys.stderr) - started: float = time.perf_counter() - try: - report: CloneReport = build_clone_report( - repo_root=repo_root, options=clone_options_from(args), config=config - ) - except Exception: - print("dupscore: clone detection failed", file=sys.stderr) - raise - unit_total: int = sum(report.unit_counts.values()) - print( - f"dupscore: found {len(report.clusters)} clone clusters across {unit_total} units " - f"in {time.perf_counter() - started:.1f}s", - file=sys.stderr, - ) - if args.as_json: - return render_clone_json(report=report, top=top) - return render_clone_text(report=report, top=top) diff --git a/scripts/dupscore/models.py b/scripts/dupscore/models.py deleted file mode 100644 index 4aabec8287..0000000000 --- a/scripts/dupscore/models.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Structured runtime models for the dupscore duplication-risk advisory tool.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - - -@dataclass(frozen=True, slots=True) -class FunctionFact: - """A top-level function or method extracted from one module.""" - - name: str - qualified_name: str - module: str - public: bool - lineno: int - resolved_calls: tuple[str, ...] - bare_attribute_calls: tuple[str, ...] - body_tokens: frozenset[str] - - -@dataclass(frozen=True, slots=True) -class ClassFact: - """A class extracted from one module, with dataclass-style field names.""" - - name: str - qualified_name: str - module: str - public: bool - lineno: int - dataclass_like: bool - field_names: tuple[str, ...] - methods: tuple[FunctionFact, ...] - - -@dataclass(frozen=True, slots=True) -class ModuleFacts: - """All extracted facts for one Python module.""" - - module: str - relative_path: str - functions: tuple[FunctionFact, ...] - classes: tuple[ClassFact, ...] - imported_modules: tuple[str, ...] - - -@dataclass(frozen=True, slots=True) -class ProjectFacts: - """Extracted facts for every analyzed module of one revision.""" - - modules: tuple[ModuleFacts, ...] - - -@dataclass(frozen=True, slots=True) -class SignalPairScore: - """One package pair scored by a single signal.""" - - package_pair: tuple[str, str] - score: float - evidence: tuple[str, ...] - - -@dataclass(frozen=True, slots=True) -class SignalRanking: - """A deterministic ranked list of package pairs for one signal.""" - - signal_name: str - entries: tuple[SignalPairScore, ...] - - -@dataclass(frozen=True, slots=True) -class SignalContribution: - """One signal's rank and fused points for a combined entry.""" - - signal_name: str - rank: int - points: float - evidence: tuple[str, ...] - - -@dataclass(frozen=True, slots=True) -class CombinedPairEntry: - """One package pair in the fused ranking.""" - - package_pair: tuple[str, str] - score: float - allowlisted: bool - allowlist_reason: str | None - contributions: tuple[SignalContribution, ...] - - -@dataclass(frozen=True, slots=True) -class DupscoreReport: - """The complete fused duplication-risk report for one revision.""" - - revision_label: str - total_pairs: int - entries: tuple[CombinedPairEntry, ...] - - -@dataclass(frozen=True, slots=True) -class PairEvidenceReport: - """Drill-down evidence for one requested package pair.""" - - package_pair: tuple[str, str] - combined_rank: int | None - contributions: tuple[SignalContribution, ...] - - -@dataclass(frozen=True, slots=True) -class ReportDelta: - """Differences between the reports of two revisions.""" - - base_label: str - current_label: str - entered_top: tuple[CombinedPairEntry, ...] - left_top: tuple[CombinedPairEntry, ...] - new_state_fanin_evidence: tuple[str, ...] - new_same_name_evidence: tuple[str, ...] - - -@dataclass(frozen=True, slots=True) -class CloneAllowlistEntry: - """Path globs whose mutual clones are intentional, with the recorded reason.""" - - paths: tuple[str, ...] - reason: str - - -@dataclass(frozen=True, slots=True) -class ContractExemptionEntry: - """Methods of a contract class that classes under ``paths`` must each define themselves.""" - - contract_path: str - contract_class: str - paths: tuple[str, ...] - reason: str - forbidden_owners: tuple[tuple[str, str], ...] = () - - -@dataclass(frozen=True, slots=True) -class DupscoreConfig: - """User configuration loaded from dupscore.toml.""" - - persisted_state_surfaces: tuple[str, ...] = () - allowlisted_pairs: dict[tuple[str, str], str] = field(default_factory=dict) - clone_allowlist: tuple[CloneAllowlistEntry, ...] = () - contract_exemptions: tuple[ContractExemptionEntry, ...] = () - - -@dataclass(frozen=True, slots=True) -class RustToken: - """One lexed Rust token with its kind, spelling, and 1-based line.""" - - kind: str - text: str - line: int - - -@dataclass(frozen=True, slots=True) -class CloneUnit: - """One function-level unit with its normalised token stream.""" - - language: str - path: str - name: str - start_line: int - end_line: int - normalized: tuple[str, ...] - concrete_key: str - normalized_key: str - - -@dataclass(frozen=True, slots=True) -class ClonePair: - """Two units whose fingerprints overlap above the similarity threshold.""" - - left: int - right: int - similarity: float - category: str - - -@dataclass(frozen=True, slots=True) -class CloneMember: - """One unit participating in a reported clone cluster.""" - - language: str - path: str - name: str - start_line: int - end_line: int - tokens: int - change: str | None - forced_override: bool - - -@dataclass(frozen=True, slots=True) -class ClonePairLink: - """One similarity edge between two members of a cluster, by member index.""" - - left: int - right: int - similarity: float - category: str - - -@dataclass(frozen=True, slots=True) -class CloneCluster: - """A transitive group of mutually similar units.""" - - category: str - similarity_min: float - similarity_max: float - duplicated_tokens: int - members: tuple[CloneMember, ...] - links: tuple[ClonePairLink, ...] - - -@dataclass(frozen=True, slots=True) -class CloneOptions: - """Detection and reporting options for the clones mode.""" - - languages: tuple[str, ...] - include_tests: bool - min_tokens: int - min_similarity: float - path_globs: tuple[str, ...] - since: str | None - - -@dataclass(frozen=True, slots=True) -class CloneReport: - """Every reported clone cluster for the worktree.""" - - since: str | None - unit_counts: dict[str, int] - allowlisted_pairs: int - contract_exempt_members: int - clusters: tuple[CloneCluster, ...] - - -@dataclass(frozen=True, slots=True) -class FileChanges: - """Lines of one file added or changed since a base revision.""" - - added_ranges: tuple[tuple[int, int], ...] = () - deletion_points: tuple[int, ...] = () - whole_file: bool = False - - -@dataclass(frozen=True, slots=True) -class RustFileUnits: - """Units found in one Rust file plus the test-only module files it declares.""" - - units: tuple[CloneUnit, ...] - test_module_prefixes: tuple[str, ...] diff --git a/tests/unit/scripts/dupscore/_helpers/clones/clustering/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/clustering/_test_types.py deleted file mode 100644 index d891d9263a..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/clustering/_test_types.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ClusterPairsTestCase: - description: str - pairs: tuple[tuple[int, int, float, str], ...] - expected_clusters: tuple[tuple[str, tuple[str, ...]], ...] - - -@dataclass(frozen=True) -class ClusterEstimateTestCase: - description: str - pairs: tuple[tuple[int, int, float, str], ...] - expected_links: tuple[tuple[int, int], ...] - expected_similarity_range: tuple[float, float] - expected_duplicated_tokens: int - - -@dataclass(frozen=True) -class ForcedClusterTestCase: - description: str - pairs: tuple[tuple[int, int, float, str], ...] - forced: frozenset[int] - expected_members: tuple[tuple[str, bool], ...] - expected_links: tuple[tuple[int, int], ...] - expected_duplicated_tokens: int diff --git a/tests/unit/scripts/dupscore/_helpers/clones/clustering/helpers.py b/tests/unit/scripts/dupscore/_helpers/clones/clustering/helpers.py deleted file mode 100644 index f07e07943f..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/clustering/helpers.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -from scripts.dupscore._helpers.clones.tokens import build_clone_unit -from scripts.dupscore.models import CloneCluster, ClonePair, CloneUnit - -_UNIT_SIZES: tuple[int, ...] = (100, 100, 90, 40, 40) - - -def build_units() -> list[CloneUnit]: - return [ - build_clone_unit( - language="python", - path=f"src/sqlbuild/module_{position}.py", - name=f"function_{position}", - start_line=1, - end_line=10, - normalized=["$id"] * size, - concrete=["name"] * size, - ) - for position, size in enumerate(_UNIT_SIZES) - ] - - -def build_pairs(specs: tuple[tuple[int, int, float, str], ...]) -> list[ClonePair]: - return [ - ClonePair(left=left, right=right, similarity=similarity, category=category) - for left, right, similarity, category in specs - ] - - -def no_change(unit: CloneUnit) -> str | None: - return None - - -def summarize_clusters(clusters: list[CloneCluster]) -> tuple[tuple[str, tuple[str, ...]], ...]: - summaries: list[tuple[str, tuple[str, ...]]] = [] - for cluster in clusters: - names: tuple[str, ...] = tuple(member.name for member in cluster.members) - summaries.append((cluster.category, names)) - return tuple(summaries) diff --git a/tests/unit/scripts/dupscore/_helpers/clones/clustering/test_cluster_clone_pairs.py b/tests/unit/scripts/dupscore/_helpers/clones/clustering/test_cluster_clone_pairs.py deleted file mode 100644 index a7f5170833..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/clustering/test_cluster_clone_pairs.py +++ /dev/null @@ -1,130 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.clones.clustering import cluster_clone_pairs -from scripts.dupscore.constants import CATEGORY_EXACT, CATEGORY_NEAR_MISS, CATEGORY_RENAMED -from scripts.dupscore.models import CloneCluster -from tests.unit.scripts.dupscore._helpers.clones.clustering._test_types import ( - ClusterEstimateTestCase, - ClusterPairsTestCase, - ForcedClusterTestCase, -) -from tests.unit.scripts.dupscore._helpers.clones.clustering.helpers import ( - build_pairs, - build_units, - no_change, - summarize_clusters, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ClusterPairsTestCase( - description="transitive pairs merge and rank by duplicated tokens", - pairs=( - (3, 4, 1.0, CATEGORY_EXACT), - (0, 1, 1.0, CATEGORY_RENAMED), - (1, 2, 0.9, CATEGORY_NEAR_MISS), - ), - expected_clusters=( - (CATEGORY_NEAR_MISS, ("function_0", "function_1", "function_2")), - (CATEGORY_EXACT, ("function_3", "function_4")), - ), - ), - ClusterPairsTestCase( - description="no pairs means no clusters", - pairs=(), - expected_clusters=(), - ), - ], - ids=lambda case: case.description, -) -def test_given_clone_pairs_when_clustering_then_groups_transitively( - test_case: ClusterPairsTestCase, -) -> None: - clusters: list[CloneCluster] = cluster_clone_pairs( - units=build_units(), - pairs=build_pairs(test_case.pairs), - change_of=no_change, - forced=frozenset(), - ) - - assert summarize_clusters(clusters) == test_case.expected_clusters - - -@pytest.mark.parametrize( - "test_case", - [ - ClusterEstimateTestCase( - description="chain reports member links and duplicated token estimate", - pairs=((0, 1, 1.0, CATEGORY_RENAMED), (1, 2, 0.9, CATEGORY_NEAR_MISS)), - expected_links=((0, 1), (1, 2)), - expected_similarity_range=(0.9, 1.0), - expected_duplicated_tokens=181, - ) - ], - ids=lambda case: case.description, -) -def test_given_cluster_when_clustering_then_reports_links_and_estimates( - test_case: ClusterEstimateTestCase, -) -> None: - cluster: CloneCluster = cluster_clone_pairs( - units=build_units(), - pairs=build_pairs(test_case.pairs), - change_of=no_change, - forced=frozenset(), - )[0] - - assert tuple((link.left, link.right) for link in cluster.links) == test_case.expected_links - assert (cluster.similarity_min, cluster.similarity_max) == test_case.expected_similarity_range - assert cluster.duplicated_tokens == test_case.expected_duplicated_tokens - - -@pytest.mark.parametrize( - "test_case", - [ - ForcedClusterTestCase( - description="forced members sort last and add no duplicated tokens", - pairs=((0, 3, 1.0, CATEGORY_EXACT), (1, 3, 1.0, CATEGORY_EXACT)), - forced=frozenset({0, 1}), - expected_members=( - ("function_3", False), - ("function_0", True), - ("function_1", True), - ), - expected_links=((0, 1), (0, 2)), - expected_duplicated_tokens=40, - ), - ForcedClusterTestCase( - description="without forced members one copy is kept out of the estimate", - pairs=((0, 3, 1.0, CATEGORY_EXACT), (1, 3, 1.0, CATEGORY_EXACT)), - forced=frozenset(), - expected_members=( - ("function_0", False), - ("function_1", False), - ("function_3", False), - ), - expected_links=((0, 2), (1, 2)), - expected_duplicated_tokens=140, - ), - ], - ids=lambda case: case.description, -) -def test_given_forced_members_when_clustering_then_orders_marks_and_estimates( - test_case: ForcedClusterTestCase, -) -> None: - cluster: CloneCluster = cluster_clone_pairs( - units=build_units(), - pairs=build_pairs(test_case.pairs), - change_of=no_change, - forced=test_case.forced, - )[0] - - assert ( - tuple((member.name, member.forced_override) for member in cluster.members) - == test_case.expected_members - ) - assert tuple((link.left, link.right) for link in cluster.links) == test_case.expected_links - assert cluster.duplicated_tokens == test_case.expected_duplicated_tokens diff --git a/tests/unit/scripts/dupscore/_helpers/clones/filters/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/filters/_test_types.py deleted file mode 100644 index 650c3bbf97..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/filters/_test_types.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from scripts.dupscore.models import FileChanges - - -@dataclass(frozen=True) -class ClassifyChangeTestCase: - description: str - changes: FileChanges | None - expected_change: str | None - - -@dataclass(frozen=True) -class ParseDiffTestCase: - description: str - diff_text: str - expected_changes: dict[str, FileChanges] diff --git a/tests/unit/scripts/dupscore/_helpers/clones/filters/test_clone_changes.py b/tests/unit/scripts/dupscore/_helpers/clones/filters/test_clone_changes.py deleted file mode 100644 index 2dd70eb62a..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/filters/test_clone_changes.py +++ /dev/null @@ -1,120 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.clones.filters import classify_unit_change, parse_unified_diff -from scripts.dupscore._helpers.clones.tokens import build_clone_unit -from scripts.dupscore.constants import CHANGE_CHANGED, CHANGE_NEW -from scripts.dupscore.models import CloneUnit, FileChanges -from tests.unit.scripts.dupscore._helpers.clones.filters._test_types import ( - ClassifyChangeTestCase, - ParseDiffTestCase, -) - -_DIFF_TEXT: str = """\ -diff --git a/src/sqlbuild/orders.py b/src/sqlbuild/orders.py -index 1111111..2222222 100644 ---- a/src/sqlbuild/orders.py -+++ b/src/sqlbuild/orders.py -@@ -10,0 +11,3 @@ def summarize(): -+ one -+ two -+ three -@@ -40 +43 @@ def other(): -- old -+ new -@@ -60,2 +62,0 @@ def removed(): -- gone -- gone -diff --git a/src/sqlbuild/deleted.py b/src/sqlbuild/deleted.py -deleted file mode 100644 ---- a/src/sqlbuild/deleted.py -+++ /dev/null -@@ -1,2 +0,0 @@ --x --y -""" - -_UNIT: CloneUnit = build_clone_unit( - language="python", - path="src/sqlbuild/orders.py", - name="summarize", - start_line=10, - end_line=20, - normalized=["$id"], - concrete=["name"], -) - - -@pytest.mark.parametrize( - "test_case", - [ - ParseDiffTestCase( - description="added ranges and pure deletions; deleted files are ignored", - diff_text=_DIFF_TEXT, - expected_changes={ - "src/sqlbuild/orders.py": FileChanges( - added_ranges=((11, 13), (43, 43)), - deletion_points=(62,), - ) - }, - ), - ParseDiffTestCase( - description="empty diff has no changes", - diff_text="", - expected_changes={}, - ), - ], - ids=lambda case: case.description, -) -def test_given_unified_diff_when_parsing_then_returns_added_ranges_and_deletions( - test_case: ParseDiffTestCase, -) -> None: - assert parse_unified_diff(test_case.diff_text) == test_case.expected_changes - - -@pytest.mark.parametrize( - "test_case", - [ - ClassifyChangeTestCase( - description="untouched file", - changes=None, - expected_change=None, - ), - ClassifyChangeTestCase( - description="untracked file is new", - changes=FileChanges(whole_file=True), - expected_change=CHANGE_NEW, - ), - ClassifyChangeTestCase( - description="fully added range is new", - changes=FileChanges(added_ranges=((5, 12), (13, 25))), - expected_change=CHANGE_NEW, - ), - ClassifyChangeTestCase( - description="partially added range is changed", - changes=FileChanges(added_ranges=((18, 30),)), - expected_change=CHANGE_CHANGED, - ), - ClassifyChangeTestCase( - description="deletion inside the unit is changed", - changes=FileChanges(deletion_points=(15,)), - expected_change=CHANGE_CHANGED, - ), - ClassifyChangeTestCase( - description="deletion after the last line is untouched", - changes=FileChanges(deletion_points=(20,)), - expected_change=None, - ), - ClassifyChangeTestCase( - description="edits outside the unit are untouched", - changes=FileChanges(added_ranges=((1, 9), (21, 40))), - expected_change=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_file_changes_when_classifying_unit_then_returns_expected_status( - test_case: ClassifyChangeTestCase, -) -> None: - assert classify_unit_change(unit=_UNIT, changes=test_case.changes) == test_case.expected_change diff --git a/tests/unit/scripts/dupscore/_helpers/clones/matching/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/matching/_test_types.py deleted file mode 100644 index 12f2929a7f..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/matching/_test_types.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class NearMissFloorTestCase: - description: str - shared_tokens: int - differing_tokens: int - min_similarity: float - expected_categories: list[str] diff --git a/tests/unit/scripts/dupscore/_helpers/clones/matching/helpers.py b/tests/unit/scripts/dupscore/_helpers/clones/matching/helpers.py deleted file mode 100644 index 7107a9233d..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/matching/helpers.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations - -from scripts.dupscore._helpers.clones.tokens import build_clone_unit -from scripts.dupscore.models import CloneUnit - - -def diverging_unit(*, name: str, shared_tokens: int, differing_tokens: int) -> CloneUnit: - """Build a unit whose stream starts with common tokens and ends with side-specific ones.""" - - tokens: list[str] = [f"shared{index}" for index in range(shared_tokens)] - tokens.extend(f"{name}{index}" for index in range(differing_tokens)) - return build_clone_unit( - language="python", - path=f"src/sqlbuild/{name}.py", - name=name, - start_line=1, - end_line=len(tokens), - normalized=tokens, - concrete=tokens, - ) diff --git a/tests/unit/scripts/dupscore/_helpers/clones/matching/test_find_clone_pairs.py b/tests/unit/scripts/dupscore/_helpers/clones/matching/test_find_clone_pairs.py deleted file mode 100644 index c10de2e878..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/matching/test_find_clone_pairs.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.clones.fingerprints import fingerprint_tokens -from scripts.dupscore._helpers.clones.matching import find_clone_pairs -from scripts.dupscore.constants import CATEGORY_NEAR_MISS -from scripts.dupscore.models import ClonePair, CloneUnit -from tests.unit.scripts.dupscore._helpers.clones.matching._test_types import ( - NearMissFloorTestCase, -) -from tests.unit.scripts.dupscore._helpers.clones.matching.helpers import diverging_unit - - -@pytest.mark.parametrize( - "test_case", - [ - NearMissFloorTestCase( - description="short pair below the small-unit similarity floor is dropped", - shared_tokens=60, - differing_tokens=10, - min_similarity=0.8, - expected_categories=[], - ), - NearMissFloorTestCase( - description="short pair above the small-unit similarity floor is kept", - shared_tokens=66, - differing_tokens=4, - min_similarity=0.8, - expected_categories=[CATEGORY_NEAR_MISS], - ), - NearMissFloorTestCase( - description="larger pair with the same ratio uses the configured threshold", - shared_tokens=86, - differing_tokens=14, - min_similarity=0.8, - expected_categories=[CATEGORY_NEAR_MISS], - ), - NearMissFloorTestCase( - description="a stricter configured threshold still applies to short pairs", - shared_tokens=66, - differing_tokens=4, - min_similarity=0.95, - expected_categories=[], - ), - ], - ids=lambda case: case.description, -) -def test_given_near_miss_pair_when_matching_then_applies_size_dependent_floor( - test_case: NearMissFloorTestCase, -) -> None: - units: list[CloneUnit] = [ - diverging_unit( - name=side, - shared_tokens=test_case.shared_tokens, - differing_tokens=test_case.differing_tokens, - ) - for side in ("left", "right") - ] - - pairs: list[ClonePair] = find_clone_pairs( - units=units, - fingerprints=[fingerprint_tokens(unit.normalized) for unit in units], - min_similarity=test_case.min_similarity, - ) - - assert [pair.category for pair in pairs] == test_case.expected_categories diff --git a/tests/unit/scripts/dupscore/_helpers/clones/python_units/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/python_units/_test_types.py deleted file mode 100644 index 1c69f17c63..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/python_units/_test_types.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PythonUnitsTestCase: - description: str - source: str - expected_units: tuple[tuple[str, int, int], ...] - - -@dataclass(frozen=True) -class PythonKeyTestCase: - description: str - left_source: str - right_source: str - expected_same_concrete: bool - expected_same_normalized: bool diff --git a/tests/unit/scripts/dupscore/_helpers/clones/python_units/test_extract_python_units.py b/tests/unit/scripts/dupscore/_helpers/clones/python_units/test_extract_python_units.py deleted file mode 100644 index 23178402ac..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/python_units/test_extract_python_units.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.clones.python_units import extract_python_units -from scripts.dupscore.models import CloneUnit -from tests.unit.scripts.dupscore._helpers.clones.python_units._test_types import ( - PythonKeyTestCase, - PythonUnitsTestCase, -) - -_LAYOUT_SOURCE: str = """\ -import functools - - -@functools.cache -def load_orders(path): - def parse(line): - return line.split(",") - return [parse(line) for line in open(path)] - - -class Warehouse: - class Shelf: - def label(self): - return "shelf" - - async def restock(self, count): - return count + 1 - - -if True: - def conditional_helper(): - return None -""" - - -@pytest.mark.parametrize( - "test_case", - [ - PythonUnitsTestCase( - description="functions, nested class methods, and conditional defs are units", - source=_LAYOUT_SOURCE, - expected_units=( - ("load_orders", 5, 8), - ("Warehouse.Shelf.label", 13, 14), - ("Warehouse.restock", 16, 17), - ("conditional_helper", 21, 22), - ), - ), - PythonUnitsTestCase( - description="unparseable source yields no units", - source="def broken(:\n pass\n", - expected_units=(), - ), - ], - ids=lambda case: case.description, -) -def test_given_python_source_when_extracting_units_then_returns_expected_units( - test_case: PythonUnitsTestCase, -) -> None: - units: tuple[CloneUnit, ...] = extract_python_units( - relative_path="src/sqlbuild/demo.py", source=test_case.source - ) - - assert ( - tuple((unit.name, unit.start_line, unit.end_line) for unit in units) - == test_case.expected_units - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PythonKeyTestCase( - description="keyword-only markers and defaults still change identity", - left_source='def f(a, b):\n """Doc."""\n total = a + b # sum\n return total\n', - right_source=( - "def f(a: int, *, b: int = 0) -> int:\n total: int = a + b\n return total\n" - ), - expected_same_concrete=False, - expected_same_normalized=False, - ), - PythonKeyTestCase( - description="annotations and docstring only differences are exact", - left_source='def f(a, b):\n """Doc."""\n total = a + b # sum\n return total\n', - right_source=( - "def f(a: int, b: list[int]) -> int:\n total: int = a + b\n return total\n" - ), - expected_same_concrete=True, - expected_same_normalized=True, - ), - PythonKeyTestCase( - description="renamed identifiers and literals share the normalised stream", - left_source='def f(a):\n return g(a, "x", 1)\n', - right_source='def other(value):\n return g(value, "y", 2)\n', - expected_same_concrete=False, - expected_same_normalized=True, - ), - PythonKeyTestCase( - description="different called functions keep distinct normalised streams", - left_source='def f(a):\n return g(a, "x", 1)\n', - right_source='def f(a):\n return h(a, "x", 1)\n', - expected_same_concrete=False, - expected_same_normalized=False, - ), - PythonKeyTestCase( - description="different called methods keep distinct normalised streams", - left_source="def f(orders):\n return orders.load(1)\n", - right_source="def f(orders):\n return orders.save(1)\n", - expected_same_concrete=False, - expected_same_normalized=False, - ), - PythonKeyTestCase( - description="call receivers and plain attribute reads stay abstracted", - left_source="def f(orders):\n return orders.total(orders.count)\n", - right_source="def f(items):\n return items.total(items.size)\n", - expected_same_concrete=False, - expected_same_normalized=True, - ), - PythonKeyTestCase( - description="method calls after non-ascii text keep their call target", - left_source='def f(a):\n label = "caf\u00e9"; return label.upper()\n', - right_source='def f(a):\n label = "caf\u00e9"; return label.lower()\n', - expected_same_concrete=False, - expected_same_normalized=False, - ), - PythonKeyTestCase( - description="non-ascii literals before the same method call stay renamed", - left_source='def f(a):\n label = "caf\u00e9"; return label.upper()\n', - right_source='def f(b):\n name = "cr\u00e8me"; return name.upper()\n', - expected_same_concrete=False, - expected_same_normalized=True, - ), - PythonKeyTestCase( - description="different literal kinds keep distinct placeholders", - left_source='def f(a):\n return g(a, "x")\n', - right_source="def f(a):\n return g(a, 1)\n", - expected_same_concrete=False, - expected_same_normalized=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_two_functions_when_extracting_then_identity_keys_reflect_normalisation( - test_case: PythonKeyTestCase, -) -> None: - left: CloneUnit = extract_python_units(relative_path="a.py", source=test_case.left_source)[0] - right: CloneUnit = extract_python_units(relative_path="b.py", source=test_case.right_source)[0] - - assert (left.concrete_key == right.concrete_key) is test_case.expected_same_concrete - assert (left.normalized_key == right.normalized_key) is test_case.expected_same_normalized diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rendering/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/rendering/_test_types.py deleted file mode 100644 index 0c2e6cfd3c..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/rendering/_test_types.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class HiddenCountsTestCase: - description: str - allowlisted_pairs: int - contract_exempt_members: int - expected_summary_fragment: str - - -@dataclass(frozen=True) -class ForcedMarkerTestCase: - description: str - forced_flags: tuple[bool, ...] - expected_member_lines: tuple[str, ...] - expected_json_flags: list[bool] diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rendering/helpers.py b/tests/unit/scripts/dupscore/_helpers/clones/rendering/helpers.py deleted file mode 100644 index fd923f5696..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/rendering/helpers.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from scripts.dupscore.constants import CATEGORY_EXACT -from scripts.dupscore.models import CloneCluster, CloneMember, ClonePairLink, CloneReport - -_MEMBER_NAMES: tuple[str, ...] = ("OrdersStore.write_orders", "BaseStore.write_orders") -_FIRST_MEMBER_LINE: int = 2 - - -def report_with_members(forced_flags: tuple[bool, ...]) -> CloneReport: - members: list[CloneMember] = [] - for position, (name, forced) in enumerate(zip(_MEMBER_NAMES, forced_flags, strict=True)): - members.append( - CloneMember( - language="python", - path="src/sqlbuild/demo/orders.py", - name=name, - start_line=1 + 10 * position, - end_line=9 + 10 * position, - tokens=80, - change=None, - forced_override=forced, - ) - ) - cluster: CloneCluster = CloneCluster( - category=CATEGORY_EXACT, - similarity_min=1.0, - similarity_max=1.0, - duplicated_tokens=80, - members=tuple(members), - links=(ClonePairLink(left=0, right=1, similarity=1.0, category=CATEGORY_EXACT),), - ) - return CloneReport( - since=None, - unit_counts={"python": 2}, - allowlisted_pairs=0, - contract_exempt_members=0, - clusters=(cluster,), - ) - - -def member_lines(text: str) -> tuple[str, ...]: - return tuple(line.strip() for line in text.splitlines()[_FIRST_MEMBER_LINE:]) diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rendering/test_render_clone_report.py b/tests/unit/scripts/dupscore/_helpers/clones/rendering/test_render_clone_report.py deleted file mode 100644 index b21f78d2ee..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/rendering/test_render_clone_report.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any - -import pytest - -from scripts.dupscore._helpers.clones.rendering import render_clone_json, render_clone_text -from scripts.dupscore.models import CloneReport -from tests.unit.scripts.dupscore._helpers.clones.rendering._test_types import ( - ForcedMarkerTestCase, - HiddenCountsTestCase, -) -from tests.unit.scripts.dupscore._helpers.clones.rendering.helpers import ( - member_lines, - report_with_members, -) - - -@pytest.mark.parametrize( - "test_case", - [ - HiddenCountsTestCase( - description="summary reports allowlisted pairs and contract-exempt members", - allowlisted_pairs=2, - contract_exempt_members=5, - expected_summary_fragment=( - "2 allowlisted pairs hidden; 5 contract-exempt members hidden" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_hidden_counts_when_rendering_then_text_and_json_report_them( - test_case: HiddenCountsTestCase, -) -> None: - report: CloneReport = CloneReport( - since=None, - unit_counts={"python": 3}, - allowlisted_pairs=test_case.allowlisted_pairs, - contract_exempt_members=test_case.contract_exempt_members, - clusters=(), - ) - - text: str = render_clone_text(report=report, top=5) - payload: dict[str, object] = json.loads(render_clone_json(report=report, top=5)) - - assert test_case.expected_summary_fragment in text.splitlines()[0] - assert payload["allowlisted_pairs"] == test_case.allowlisted_pairs - assert payload["contract_exempt_members"] == test_case.contract_exempt_members - - -@pytest.mark.parametrize( - "test_case", - [ - ForcedMarkerTestCase( - description="forced members are marked in text and flagged in json", - forced_flags=(False, True), - expected_member_lines=( - "src/sqlbuild/demo/orders.py:1-9 OrdersStore.write_orders (80 tokens)", - "src/sqlbuild/demo/orders.py:11-19 BaseStore.write_orders (80 tokens) [forced]", - ), - expected_json_flags=[False, True], - ) - ], - ids=lambda case: case.description, -) -def test_given_forced_members_when_rendering_then_marks_them( - test_case: ForcedMarkerTestCase, -) -> None: - report: CloneReport = report_with_members(test_case.forced_flags) - - text: str = render_clone_text(report=report, top=5) - payload: dict[str, Any] = json.loads(render_clone_json(report=report, top=5)) - - assert member_lines(text) == test_case.expected_member_lines - assert [ - member["forced_override"] for member in payload["clusters"][0]["members"] - ] == test_case.expected_json_flags diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/_test_types.py deleted file mode 100644 index bcdfac43da..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/_test_types.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class LexRustTestCase: - description: str - source: str - expected_tokens: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True) -class LexLineTestCase: - description: str - source: str - token_text: str - expected_line: int diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/test_lex_rust.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/test_lex_rust.py deleted file mode 100644 index 92c3b3dbbd..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/test_lex_rust.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.clones.rust_lexer import lex_rust -from scripts.dupscore.constants import ( - RUST_KIND_BYTE, - RUST_KIND_BYTE_STRING, - RUST_KIND_CHAR, - RUST_KIND_IDENTIFIER, - RUST_KIND_LIFETIME, - RUST_KIND_NUMBER, - RUST_KIND_PUNCTUATION, - RUST_KIND_STRING, -) -from scripts.dupscore.models import RustToken -from tests.unit.scripts.dupscore._helpers.clones.rust_lexer._test_types import ( - LexLineTestCase, - LexRustTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - LexRustTestCase( - description="raw string containing braces and quotes is one token", - source='let s = r#"{ "a" } }"#;', - expected_tokens=( - (RUST_KIND_IDENTIFIER, "let"), - (RUST_KIND_IDENTIFIER, "s"), - (RUST_KIND_PUNCTUATION, "="), - (RUST_KIND_STRING, 'r#"{ "a" } }"#'), - (RUST_KIND_PUNCTUATION, ";"), - ), - ), - LexRustTestCase( - description="lifetimes are distinguished from char literals", - source="fn f<'a>(x: &'a str) -> char { '{' }", - expected_tokens=( - (RUST_KIND_IDENTIFIER, "fn"), - (RUST_KIND_IDENTIFIER, "f"), - (RUST_KIND_PUNCTUATION, "<"), - (RUST_KIND_LIFETIME, "'a"), - (RUST_KIND_PUNCTUATION, ">"), - (RUST_KIND_PUNCTUATION, "("), - (RUST_KIND_IDENTIFIER, "x"), - (RUST_KIND_PUNCTUATION, ":"), - (RUST_KIND_PUNCTUATION, "&"), - (RUST_KIND_LIFETIME, "'a"), - (RUST_KIND_IDENTIFIER, "str"), - (RUST_KIND_PUNCTUATION, ")"), - (RUST_KIND_PUNCTUATION, "->"), - (RUST_KIND_IDENTIFIER, "char"), - (RUST_KIND_PUNCTUATION, "{"), - (RUST_KIND_CHAR, "'{'"), - (RUST_KIND_PUNCTUATION, "}"), - ), - ), - LexRustTestCase( - description="nested block comments and line comments are dropped", - source="/* outer /* inner { */ still } */ x // trailing {\ny", - expected_tokens=((RUST_KIND_IDENTIFIER, "x"), (RUST_KIND_IDENTIFIER, "y")), - ), - LexRustTestCase( - description="escaped chars and byte literals are single tokens", - source=r"""'\'' '\u{7B}' b'}' b"{\"}" br#"}"# 'static""", - expected_tokens=( - (RUST_KIND_CHAR, r"'\''"), - (RUST_KIND_CHAR, r"'\u{7B}'"), - (RUST_KIND_BYTE, "b'}'"), - (RUST_KIND_BYTE_STRING, r'b"{\"}"'), - (RUST_KIND_BYTE_STRING, 'br#"}"#'), - (RUST_KIND_LIFETIME, "'static"), - ), - ), - LexRustTestCase( - description="numbers keep suffixes and ranges stay separate", - source="0..10u8 1.5e-3 0xFF_u32 t.0", - expected_tokens=( - (RUST_KIND_NUMBER, "0"), - (RUST_KIND_PUNCTUATION, ".."), - (RUST_KIND_NUMBER, "10u8"), - (RUST_KIND_NUMBER, "1.5e-3"), - (RUST_KIND_NUMBER, "0xFF_u32"), - (RUST_KIND_IDENTIFIER, "t"), - (RUST_KIND_PUNCTUATION, "."), - (RUST_KIND_NUMBER, "0"), - ), - ), - LexRustTestCase( - description="raw identifiers and path separators", - source="r#type::new()", - expected_tokens=( - (RUST_KIND_IDENTIFIER, "r#type"), - (RUST_KIND_PUNCTUATION, "::"), - (RUST_KIND_IDENTIFIER, "new"), - (RUST_KIND_PUNCTUATION, "("), - (RUST_KIND_PUNCTUATION, ")"), - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_rust_source_when_lexing_then_returns_expected_tokens( - test_case: LexRustTestCase, -) -> None: - tokens: list[RustToken] = lex_rust(test_case.source) - - assert tuple((item.kind, item.text) for item in tokens) == test_case.expected_tokens - - -@pytest.mark.parametrize( - "test_case", - [ - LexLineTestCase( - description="multi-line string advances the line counter", - source='let a = "one\ntwo\nthree";\nlet b = 1;', - token_text="b", - expected_line=4, - ), - LexLineTestCase( - description="multi-line block comment advances the line counter", - source="/* one\n/* two\n*/ three\n*/\nafter", - token_text="after", - expected_line=5, - ), - ], - ids=lambda case: case.description, -) -def test_given_multiline_tokens_when_lexing_then_tracks_lines( - test_case: LexLineTestCase, -) -> None: - tokens: list[RustToken] = lex_rust(test_case.source) - - lines_by_text: dict[str, int] = {item.text: item.line for item in tokens} - assert lines_by_text[test_case.token_text] == test_case.expected_line diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_units/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_units/_test_types.py deleted file mode 100644 index 0179c682bc..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/rust_units/_test_types.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class RustUnitsTestCase: - description: str - source: str - include_tests: bool - expected_units: tuple[tuple[str, int, int], ...] - - -@dataclass(frozen=True) -class RustTestModuleTestCase: - description: str - relative_path: str - source: str - expected_prefixes: tuple[str, ...] - - -@dataclass(frozen=True) -class RustKeyTestCase: - description: str - left_source: str - right_source: str - expected_same_concrete: bool - expected_same_normalized: bool diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_units/test_extract_rust_units.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_units/test_extract_rust_units.py deleted file mode 100644 index 807e22dbf0..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/clones/rust_units/test_extract_rust_units.py +++ /dev/null @@ -1,238 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.clones.rust_units import extract_rust_units -from scripts.dupscore.models import RustFileUnits -from tests.unit.scripts.dupscore._helpers.clones.rust_units._test_types import ( - RustKeyTestCase, - RustTestModuleTestCase, - RustUnitsTestCase, -) - -_IMPL_AND_TRAIT_SOURCE: str = """\ -pub struct Inventory { items: Vec } - -impl Inventory where T: Default { - #[must_use] - pub fn first(&self) -> Option<&T> { - self.items.first() - } -} - -pub trait Fulfill { - fn ship(&self) -> bool; - fn describe(&self) -> String { - String::from("pending") - } -} - -impl<'a> Fulfill for &'a Inventory { - fn ship(&self) -> bool { true } -} - -fn helper(pointer: fn(u8) -> u8) -> u8 { - fn nested() -> u8 { 1 } - pointer(nested()) -} -""" - -_BRACE_TRAPS_SOURCE: str = """\ -fn braces() -> &'static str { - let open = '{'; - let text = r#"}}} "{" }"#; - /* } /* } */ } */ - let bytes = b"}"; - text -} - -fn after() {} -""" - -_TEST_ITEMS_SOURCE: str = """\ -fn kept() {} - -#[cfg(test)] -fn test_helper() {} - -#[cfg(not(test))] -fn production_only() {} - -#[cfg(test)] -mod tests { - #[test] - fn checks_orders() {} -} - -#[tokio::test] -async fn async_check() {} - -macro_rules! make_fn { - () => { fn generated() {} }; -} -""" - - -@pytest.mark.parametrize( - "test_case", - [ - RustUnitsTestCase( - description="impl and trait methods are qualified and bodiless decls skipped", - source=_IMPL_AND_TRAIT_SOURCE, - include_tests=False, - expected_units=( - ("Inventory::first", 5, 7), - ("Fulfill::describe", 12, 14), - ("Inventory::ship", 18, 18), - ("helper", 21, 24), - ), - ), - RustUnitsTestCase( - description="braces inside chars, raw strings, comments, and bytes are ignored", - source=_BRACE_TRAPS_SOURCE, - include_tests=False, - expected_units=(("braces", 1, 7), ("after", 9, 9)), - ), - RustUnitsTestCase( - description="cfg(test) items, test functions, and macro bodies are skipped", - source=_TEST_ITEMS_SOURCE, - include_tests=False, - expected_units=(("kept", 1, 1), ("production_only", 7, 7)), - ), - RustUnitsTestCase( - description="include-tests keeps test items", - source=_TEST_ITEMS_SOURCE, - include_tests=True, - expected_units=( - ("kept", 1, 1), - ("test_helper", 4, 4), - ("production_only", 7, 7), - ("checks_orders", 12, 12), - ("async_check", 16, 16), - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_rust_source_when_extracting_units_then_returns_fn_items( - test_case: RustUnitsTestCase, -) -> None: - extracted: RustFileUnits = extract_rust_units( - relative_path="crates/demo/src/lib.rs", - source=test_case.source, - include_tests=test_case.include_tests, - ) - - assert ( - tuple((unit.name, unit.start_line, unit.end_line) for unit in extracted.units) - == test_case.expected_units - ) - - -@pytest.mark.parametrize( - "test_case", - [ - RustTestModuleTestCase( - description="module root declares sibling test module", - relative_path="crates/demo/src/engine/mod.rs", - source="#[cfg(test)]\nmod tests;\nmod kept;\n", - expected_prefixes=( - "crates/demo/src/engine/tests.rs", - "crates/demo/src/engine/tests/", - ), - ), - RustTestModuleTestCase( - description="non-root file declares nested test module", - relative_path="crates/demo/src/engine.rs", - source="#[cfg(test)]\nmod checks;\n", - expected_prefixes=( - "crates/demo/src/engine/checks.rs", - "crates/demo/src/engine/checks/", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_cfg_test_module_declaration_when_extracting_then_reports_test_paths( - test_case: RustTestModuleTestCase, -) -> None: - extracted: RustFileUnits = extract_rust_units( - relative_path=test_case.relative_path, - source=test_case.source, - include_tests=False, - ) - - assert extracted.test_module_prefixes == test_case.expected_prefixes - - -@pytest.mark.parametrize( - "test_case", - [ - RustKeyTestCase( - description="renamed identifiers, literals, and visibility share the normalised stream", - left_source='fn count_orders(x: u8) -> u8 { let y = x + 1; println!("{y}"); y }', - right_source='pub fn tally(value: u16) -> u16 { let n = value + 9; println!("n"); n }', - expected_same_concrete=False, - expected_same_normalized=True, - ), - RustKeyTestCase( - description="different macros keep distinct normalised streams", - left_source='fn f(x: u8) { println!("{x}"); }', - right_source='fn f(x: u8) { print!("{x}"); }', - expected_same_concrete=False, - expected_same_normalized=False, - ), - RustKeyTestCase( - description="different called functions keep distinct normalised streams", - left_source="fn f(x: u8) -> u8 { load(x) }", - right_source="fn f(x: u8) -> u8 { save(x) }", - expected_same_concrete=False, - expected_same_normalized=False, - ), - RustKeyTestCase( - description="different called methods keep distinct normalised streams", - left_source="fn f(x: &str) -> usize { x.len() }", - right_source="fn f(x: &str) -> usize { x.count() }", - expected_same_concrete=False, - expected_same_normalized=False, - ), - RustKeyTestCase( - description="field reads and receivers stay abstracted", - left_source="fn f(order: &Order) -> u8 { order.total(order.count) }", - right_source="fn g(item: &Item) -> u8 { item.total(item.size) }", - expected_same_concrete=False, - expected_same_normalized=True, - ), - RustKeyTestCase( - description="comments and attributes do not change identity", - left_source="fn f(x: u8) -> u8 { x /* note */ }", - right_source="#[inline]\npub(crate) fn f(x: u8) -> u8 {\n // note\n x\n}", - expected_same_concrete=True, - expected_same_normalized=True, - ), - RustKeyTestCase( - description="char and string literals keep distinct placeholders", - left_source="fn f() -> char { 'x' }", - right_source='fn f() -> char { "x" }', - expected_same_concrete=False, - expected_same_normalized=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_two_rust_functions_when_extracting_then_identity_keys_reflect_normalisation( - test_case: RustKeyTestCase, -) -> None: - left: RustFileUnits = extract_rust_units( - relative_path="crates/demo/src/a.rs", source=test_case.left_source, include_tests=False - ) - right: RustFileUnits = extract_rust_units( - relative_path="crates/demo/src/b.rs", source=test_case.right_source, include_tests=False - ) - - assert ( - left.units[0].concrete_key == right.units[0].concrete_key - ) is test_case.expected_same_concrete - assert ( - left.units[0].normalized_key == right.units[0].normalized_key - ) is test_case.expected_same_normalized diff --git a/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/_test_types.py b/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/_test_types.py deleted file mode 100644 index ac19279dc4..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/_test_types.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from scripts.dupscore.models import ContractExemptionEntry - - -@dataclass(frozen=True) -class ForcedOverrideTestCase: - description: str - language: str - path: str - name: str - expected_forced: bool - - -@dataclass(frozen=True) -class ContractLookupErrorTestCase: - description: str - entry: ContractExemptionEntry - expected_error_fragment: str - - -@dataclass(frozen=True) -class InactiveContractTestCase: - description: str - entry: ContractExemptionEntry - path: str - name: str - expected_forced: bool diff --git a/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/helpers.py b/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/helpers.py deleted file mode 100644 index 906ae09173..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/helpers.py +++ /dev/null @@ -1,203 +0,0 @@ -from __future__ import annotations - -from scripts.dupscore._helpers.clones.tokens import build_clone_unit -from scripts.dupscore.models import CloneUnit, ContractExemptionEntry - -CONTRACT_PATH: str = "src/sqlbuild/demo/contract/strict_store.py" -BASE_PATH: str = "src/sqlbuild/demo/contract/base_store.py" -ALPHA_PATH: str = "src/sqlbuild/demo/stores/alpha/classes/alpha_store.py" -BETA_PATH: str = "src/sqlbuild/demo/stores/beta/classes/beta_store.py" -GAMMA_PATH: str = "src/sqlbuild/demo/stores/gamma/classes/gamma_store.py" -CONNECTION_PATH: str = "src/sqlbuild/demo/stores/beta/classes/orders_connection.py" -OUTSIDE_PATH: str = "src/sqlbuild/demo/elsewhere/outside_store.py" -DELTA_PATH: str = "src/sqlbuild/demo/stores/delta/classes/delta_store.py" -REEXPORT_PATH: str = "src/sqlbuild/demo/stores/epsilon/classes/epsilon_store.py" -QUALIFIED_PATH: str = "src/sqlbuild/demo/stores/zeta/classes/zeta_store.py" -STDLIB_BASE_PATH: str = "src/sqlbuild/demo/stores/eta/classes/eta_store.py" -GENERIC_BASE_PATH: str = "src/sqlbuild/demo/stores/theta/classes/theta_store.py" - -_MIXIN_SOURCE: str = """\ -from abc import abstractmethod - - -class ReaderMixin: - @abstractmethod - def read_orders(self, connection, keys): - ... - - @abstractmethod - def count_orders(self, connection): - ... -""" - -_CONTRACT_SOURCE: str = """\ -import abc -from abc import abstractmethod - -from sqlbuild.demo.contract.reader_mixin import ReaderMixin as Reader - - -class StrictStore(Reader): - @abstractmethod - def write_orders(self, connection, rows): - ... - - @abc.abstractmethod - def delete_orders(self, connection, keys): - ... - - @property - @abstractmethod - def store_name(self): - ... - - def count_orders(self, connection): - return 0 -""" - -WRITE_ORDERS: str = """ - def write_orders(self, connection, rows): - written = 0 - for row in rows: - if row.get("status") == "cancelled": - continue - values = [row["order_id"], row["customer"], row["amount"]] - connection.execute("INSERT INTO orders VALUES (?, ?, ?)", values) - written += 1 - connection.commit() - return written -""" - -DELETE_ORDERS: str = """ - def delete_orders(self, connection, keys): - pending = sorted(set(keys)) - while pending: - batch = pending[:50] - pending = pending[50:] - placeholders = ", ".join("?" for _ in batch) - try: - connection.execute(f"DELETE FROM orders WHERE id IN ({placeholders})", batch) - except KeyError as error: - raise ValueError(str(error)) from error - return len(keys) -""" - -READ_ORDERS: str = """ - def read_orders(self, connection, keys): - cursor = connection.cursor() - results = {} - with cursor: - for key in keys: - cursor.execute("SELECT * FROM orders WHERE id = ?", [key]) - fetched = cursor.fetchone() - if fetched is None: - results[key] = None - else: - results[key] = dict(zip(("id", "customer", "amount"), fetched)) - return results -""" - -RENDER_ORDERS_SQL: str = """ - def _render_orders_sql(self, table, columns, predicates): - selected = ", ".join(f'"{column}"' for column in columns) or "*" - clauses = [] - for name, value in predicates.items(): - if value is None: - clauses.append(f"{name} IS NULL") - elif isinstance(value, (list, tuple)): - clauses.append(f"{name} IN ({', '.join(repr(item) for item in value)})") - else: - clauses.append(f"{name} = {value!r}") - where = f" WHERE {' AND '.join(clauses)}" if clauses else "" - return f"SELECT {selected} FROM {table}{where}" -""" - -_BASE_IMPORT: str = "from sqlbuild.demo.contract.base_store import BaseStore\n\n\n" - -CONTRACT_FILES: dict[str, str] = { - "src/sqlbuild/demo/contract/reader_mixin.py": _MIXIN_SOURCE, - CONTRACT_PATH: _CONTRACT_SOURCE, - BASE_PATH: ( - "from .strict_store import StrictStore\n\n\nclass BaseStore(StrictStore):\n" + WRITE_ORDERS - ), - ALPHA_PATH: ( - _BASE_IMPORT - + "class AlphaStore(BaseStore):\n" - + WRITE_ORDERS - + DELETE_ORDERS - + RENDER_ORDERS_SQL - + "\n def helper(self):\n return 1\n" - ), - BETA_PATH: ( - _BASE_IMPORT - + "class BetaStore(BaseStore):\n" - + WRITE_ORDERS - + READ_ORDERS - + RENDER_ORDERS_SQL - ), - GAMMA_PATH: ( - "from sqlbuild.demo.stores.alpha.classes.alpha_store import AlphaStore\n\n\n" - "class GammaStore(AlphaStore):\n" + DELETE_ORDERS - ), - CONNECTION_PATH: "class OrdersConnection:\n" + READ_ORDERS, - OUTSIDE_PATH: ( - _BASE_IMPORT - + "class OutsideStore(BaseStore):\n" - + " def write_orders(self, connection, rows):\n return 0\n" - ), -} - -ANCESTRY_FILES: dict[str, str] = { - "src/sqlbuild/demo/shared/__init__.py": "from .writer import WriterMixin\n", - "src/sqlbuild/demo/shared/writer.py": "class WriterMixin:\n" + WRITE_ORDERS, - REEXPORT_PATH: ( - "from sqlbuild.demo.shared import WriterMixin\n" - + _BASE_IMPORT - + "class EpsilonStore(WriterMixin, BaseStore):\n" - + WRITE_ORDERS - ), - QUALIFIED_PATH: ( - "import sqlbuild.demo.shared.writer\n" - + _BASE_IMPORT - + "class ZetaStore(sqlbuild.demo.shared.writer.WriterMixin, BaseStore):\n" - + WRITE_ORDERS - ), - GENERIC_BASE_PATH: ( - "from typing import Generic, TypeVar\n" - + _BASE_IMPORT - + 'RowT = TypeVar("RowT")\n\n\n' - + "class WriterMixin(Generic[RowT]):\n" - + WRITE_ORDERS - + "\n\nclass ThetaStore(WriterMixin[int], BaseStore):\n" - + WRITE_ORDERS - ), - STDLIB_BASE_PATH: ( - "from abc import ABC\n" + _BASE_IMPORT + "class EtaStore(BaseStore, ABC):\n" + WRITE_ORDERS - ), -} - -DELTA_FILES: dict[str, str] = { - DELTA_PATH: ( - _BASE_IMPORT + "class DeltaStore(BaseStore):\n" + WRITE_ORDERS + RENDER_ORDERS_SQL - ), -} - -STORE_EXEMPTION: ContractExemptionEntry = ContractExemptionEntry( - contract_path=CONTRACT_PATH, - contract_class="StrictStore", - paths=("src/sqlbuild/demo/contract/*", "src/sqlbuild/demo/stores/*/classes/*"), - reason="Every store must define each StrictStore method itself.", - forbidden_owners=((BASE_PATH, "BaseStore"),), -) - - -def method_unit(*, language: str, path: str, name: str) -> CloneUnit: - return build_clone_unit( - language=language, - path=path, - name=name, - start_line=1, - end_line=2, - normalized=[], - concrete=[], - ) diff --git a/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/test_find_forced_overrides.py b/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/test_find_forced_overrides.py deleted file mode 100644 index 9d8d531eb9..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/contracts/forced_overrides/test_find_forced_overrides.py +++ /dev/null @@ -1,268 +0,0 @@ -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -import pytest - -from scripts.dupscore._helpers.contracts.forced_overrides import find_forced_overrides -from scripts.dupscore._helpers.inputs.config import load_config -from scripts.dupscore.exceptions import DupscoreConfigError -from scripts.dupscore.models import DupscoreConfig -from tests.unit.scripts.dupscore._helpers.contracts.forced_overrides._test_types import ( - ContractLookupErrorTestCase, - ForcedOverrideTestCase, - InactiveContractTestCase, -) -from tests.unit.scripts.dupscore._helpers.contracts.forced_overrides.helpers import ( - ALPHA_PATH, - ANCESTRY_FILES, - BASE_PATH, - BETA_PATH, - CONNECTION_PATH, - CONTRACT_FILES, - CONTRACT_PATH, - GAMMA_PATH, - GENERIC_BASE_PATH, - OUTSIDE_PATH, - QUALIFIED_PATH, - REEXPORT_PATH, - STDLIB_BASE_PATH, - STORE_EXEMPTION, - method_unit, -) -from tests.unit.scripts.dupscore.main.build_report.helpers import write_project_files - -_ADAPTER_CLASSES: str = "src/sqlbuild/adapter/contract/classes" -_ADAPTERS: str = "src/sqlbuild/adapters" - - -@pytest.mark.parametrize( - "test_case", - [ - ForcedOverrideTestCase( - description="base class override of an abstractmethod is forced", - language="python", - path=BASE_PATH, - name="BaseStore.write_orders", - expected_forced=True, - ), - ForcedOverrideTestCase( - description="subclass would otherwise inherit from a forbidden owner", - language="python", - path=ALPHA_PATH, - name="AlphaStore.write_orders", - expected_forced=True, - ), - ForcedOverrideTestCase( - description="abc.abstractmethod declarations are contract methods", - language="python", - path=ALPHA_PATH, - name="AlphaStore.delete_orders", - expected_forced=True, - ), - ForcedOverrideTestCase( - description="abstract methods inherited through an aliased mixin import count", - language="python", - path=BETA_PATH, - name="BetaStore.read_orders", - expected_forced=True, - ), - ForcedOverrideTestCase( - description="stacked property and abstractmethod decorators count", - language="python", - path=BETA_PATH, - name="BetaStore.store_name", - expected_forced=True, - ), - ForcedOverrideTestCase( - description="mixin abstract method made concrete by the contract is not a contract", - language="python", - path=ALPHA_PATH, - name="AlphaStore.count_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="private helper methods are not contract methods", - language="python", - path=ALPHA_PATH, - name="AlphaStore._render_orders_sql", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="override that could inherit from an allowed ancestor is voluntary", - language="python", - path=GAMMA_PATH, - name="GammaStore.delete_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="class that does not derive from the contract is not exempt", - language="python", - path=CONNECTION_PATH, - name="OrdersConnection.read_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="contract subclass outside the listed paths is not exempt", - language="python", - path=OUTSIDE_PATH, - name="OutsideStore.write_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="top-level functions named like contract methods are not exempt", - language="python", - path=ALPHA_PATH, - name="write_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="re-exported mixin that could supply the method blocks the exemption", - language="python", - path=REEXPORT_PATH, - name="EpsilonStore.write_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="module-qualified repository base blocks the exemption", - language="python", - path=QUALIFIED_PATH, - name="ZetaStore.write_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="same-file generic mixin that supplies the method keeps it voluntary", - language="python", - path=GENERIC_BASE_PATH, - name="ThetaStore.write_orders", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="standard-library bases do not block the exemption", - language="python", - path=STDLIB_BASE_PATH, - name="EtaStore.write_orders", - expected_forced=True, - ), - ForcedOverrideTestCase( - description="rust units are never exempt", - language="rust", - path=ALPHA_PATH, - name="AlphaStore.write_orders", - expected_forced=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_contract_exemption_when_checking_unit_then_reports_forced_override( - test_case: ForcedOverrideTestCase, - tmp_path: Path, -) -> None: - write_project_files(repo_root=tmp_path, files=CONTRACT_FILES | ANCESTRY_FILES) - - forced: tuple[bool, ...] = find_forced_overrides( - repo_root=tmp_path, - entries=(STORE_EXEMPTION,), - units=[method_unit(language=test_case.language, path=test_case.path, name=test_case.name)], - ) - - assert forced == (test_case.expected_forced,) - - -@pytest.mark.parametrize( - "test_case", - [ - ContractLookupErrorTestCase( - description="missing contract class in an existing file", - entry=replace(STORE_EXEMPTION, contract_class="MissingStore"), - expected_error_fragment=f"{CONTRACT_PATH}:MissingStore was not found", - ), - ContractLookupErrorTestCase( - description="missing forbidden owner class", - entry=replace(STORE_EXEMPTION, forbidden_owners=((BASE_PATH, "MissingBase"),)), - expected_error_fragment=f"{BASE_PATH}:MissingBase was not found", - ), - ], - ids=lambda case: case.description, -) -def test_given_unresolvable_contract_class_when_loading_then_raises_config_error( - test_case: ContractLookupErrorTestCase, - tmp_path: Path, -) -> None: - write_project_files(repo_root=tmp_path, files=CONTRACT_FILES) - - with pytest.raises(DupscoreConfigError) as raised: - _ = find_forced_overrides(repo_root=tmp_path, entries=(test_case.entry,), units=[]) - - assert test_case.expected_error_fragment in str(raised.value) - - -@pytest.mark.parametrize( - "test_case", - [ - InactiveContractTestCase( - description="entry whose contract file is absent exempts nothing", - entry=replace(STORE_EXEMPTION, contract_path="src/sqlbuild/demo/absent.py"), - path=ALPHA_PATH, - name="AlphaStore.write_orders", - expected_forced=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_absent_contract_file_when_checking_unit_then_nothing_is_exempt( - test_case: InactiveContractTestCase, - tmp_path: Path, -) -> None: - write_project_files(repo_root=tmp_path, files=CONTRACT_FILES) - - forced: tuple[bool, ...] = find_forced_overrides( - repo_root=tmp_path, - entries=(test_case.entry,), - units=[method_unit(language="python", path=test_case.path, name=test_case.name)], - ) - - assert forced == (test_case.expected_forced,) - - -@pytest.mark.parametrize( - "test_case", - [ - ForcedOverrideTestCase( - description="duckdb-backed contract override is forced", - language="python", - path=f"{_ADAPTER_CLASSES}/duckdb_backed_adapter.py", - name="DuckDbBackedAdapter.connect", - expected_forced=True, - ), - ForcedOverrideTestCase( - description="motherduck override could inherit the duckdb-backed method", - language="python", - path=f"{_ADAPTERS}/motherduck/classes/motherduck_adapter.py", - name="MotherDuckAdapter.connect", - expected_forced=False, - ), - ForcedOverrideTestCase( - description="base adapter contract method is forced", - language="python", - path=f"{_ADAPTER_CLASSES}/base_adapter.py", - name="BaseAdapter.render_merge", - expected_forced=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_shipped_contract_exemption_when_checking_repository_then_matches_contract( - test_case: ForcedOverrideTestCase, -) -> None: - repo_root: Path = Path(__file__).resolve().parents[7] - config: DupscoreConfig = load_config(repo_root / "scripts/dupscore/dupscore.toml") - - forced: tuple[bool, ...] = find_forced_overrides( - repo_root=repo_root, - entries=config.contract_exemptions, - units=[method_unit(language=test_case.language, path=test_case.path, name=test_case.name)], - ) - - assert forced == (test_case.expected_forced,) diff --git a/tests/unit/scripts/dupscore/_helpers/inputs/config/_test_types.py b/tests/unit/scripts/dupscore/_helpers/inputs/config/_test_types.py deleted file mode 100644 index d224d8ef76..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/inputs/config/_test_types.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from scripts.dupscore.models import CloneAllowlistEntry, ContractExemptionEntry - - -@dataclass(frozen=True) -class LoadConfigTestCase: - description: str - toml_text: str - expected_surfaces: tuple[str, ...] - expected_allowlisted_pair: tuple[str, str] - expected_reason: str - - -@dataclass(frozen=True) -class MissingConfigTestCase: - description: str - expected_surfaces: tuple[str, ...] - expected_allowlist_size: int - - -@dataclass(frozen=True) -class InvalidConfigTestCase: - description: str - toml_text: str - expected_error_fragment: str - - -@dataclass(frozen=True) -class CloneAllowlistTestCase: - description: str - toml_text: str - expected_entries: tuple[CloneAllowlistEntry, ...] - - -@dataclass(frozen=True) -class ShippedConfigTestCase: - description: str - relative_path: str - expected_pair_allowlist_size: int - expected_clone_allowlist_size: int - expected_contract_exemption_size: int - - -@dataclass(frozen=True) -class ContractExemptionConfigTestCase: - description: str - toml_text: str - expected_entries: tuple[ContractExemptionEntry, ...] diff --git a/tests/unit/scripts/dupscore/_helpers/inputs/config/test_load_config.py b/tests/unit/scripts/dupscore/_helpers/inputs/config/test_load_config.py deleted file mode 100644 index 7dbc1e1710..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/inputs/config/test_load_config.py +++ /dev/null @@ -1,346 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from scripts.dupscore._helpers.inputs.config import load_config -from scripts.dupscore.exceptions import DupscoreConfigError -from scripts.dupscore.models import CloneAllowlistEntry, ContractExemptionEntry, DupscoreConfig -from tests.unit.scripts.dupscore._helpers.inputs.config._test_types import ( - CloneAllowlistTestCase, - ContractExemptionConfigTestCase, - InvalidConfigTestCase, - LoadConfigTestCase, - MissingConfigTestCase, - ShippedConfigTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - LoadConfigTestCase( - description="loads surfaces and normalizes allowlist pair order", - toml_text=( - 'persisted_state_surfaces = ["sqlbuild.virtual.state"]\n' - "\n" - "[[allowlist]]\n" - 'pair = ["sqlbuild.b.right", "sqlbuild.a.left"]\n' - 'reason = "intentional mirror"\n' - ), - expected_surfaces=("sqlbuild.virtual.state",), - expected_allowlisted_pair=("sqlbuild.a.left", "sqlbuild.b.right"), - expected_reason="intentional mirror", - ) - ], - ids=lambda case: case.description, -) -def test_given_config_file_when_loading_then_returns_expected_config( - test_case: LoadConfigTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - config_path.write_text(test_case.toml_text, encoding="utf-8") - - config: DupscoreConfig = load_config(config_path) - - assert config.persisted_state_surfaces == test_case.expected_surfaces - assert ( - config.allowlisted_pairs[test_case.expected_allowlisted_pair] == test_case.expected_reason - ) - - -@pytest.mark.parametrize( - "test_case", - [ - MissingConfigTestCase( - description="returns empty config when the file is missing", - expected_surfaces=(), - expected_allowlist_size=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_config_file_when_loading_then_returns_empty_config( - test_case: MissingConfigTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - - config: DupscoreConfig = load_config(config_path) - - assert config.persisted_state_surfaces == test_case.expected_surfaces - assert len(config.allowlisted_pairs) == test_case.expected_allowlist_size - - -@pytest.mark.parametrize( - "test_case", - [ - InvalidConfigTestCase( - description="rejects allowlist entries without a reason", - toml_text='[[allowlist]]\npair = ["sqlbuild.a", "sqlbuild.b"]\n', - expected_error_fragment="reason", - ), - InvalidConfigTestCase( - description="rejects non-list surfaces", - toml_text='persisted_state_surfaces = "sqlbuild.virtual.state"\n', - expected_error_fragment="persisted_state_surfaces", - ), - InvalidConfigTestCase( - description="rejects clone allowlist entries without a reason", - toml_text='[[clone_allowlist]]\npaths = ["crates/*"]\nreason = " "\n', - expected_error_fragment="clone_allowlist entry 1 needs a non-empty reason", - ), - InvalidConfigTestCase( - description="rejects clone allowlist entries with empty paths", - toml_text='[[clone_allowlist]]\npaths = []\nreason = "mirror"\n', - expected_error_fragment="clone_allowlist entry 1 needs a non-empty paths list", - ), - InvalidConfigTestCase( - description="rejects clone allowlist entries with blank globs", - toml_text='[[clone_allowlist]]\npaths = ["src/*", ""]\nreason = "mirror"\n', - expected_error_fragment="non-empty glob strings", - ), - InvalidConfigTestCase( - description="rejects clone allowlist entries with unknown keys", - toml_text=( - '[[clone_allowlist]]\npaths = ["src/*"]\nreason = "mirror"\npair = ["a", "b"]\n' - ), - expected_error_fragment="unknown keys: pair", - ), - InvalidConfigTestCase( - description="rejects a non-table clone allowlist", - toml_text='clone_allowlist = ["src/*"]\n', - expected_error_fragment="clone_allowlist entry 1 must be a table", - ), - InvalidConfigTestCase( - description="rejects contract exemptions without a reason", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:StrictStore"\n' - 'paths = ["src/demo/*"]\n' - ), - expected_error_fragment="contract_exemption entry 1 needs a non-empty reason", - ), - InvalidConfigTestCase( - description="rejects contract specs without a class name", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py"\n' - 'paths = ["src/demo/*"]\n' - 'reason = "forced"\n' - ), - expected_error_fragment="contract_exemption entry 1 contract must be", - ), - InvalidConfigTestCase( - description="rejects contract specs that are not python files", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.rs:StrictStore"\n' - 'paths = ["src/demo/*"]\n' - 'reason = "forced"\n' - ), - expected_error_fragment="contract_exemption entry 1 contract must be", - ), - InvalidConfigTestCase( - description="rejects absolute contract paths", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "/src/demo/contract.py:StrictStore"\n' - 'paths = ["src/demo/*"]\n' - 'reason = "forced"\n' - ), - expected_error_fragment="contract_exemption entry 1 contract must be", - ), - InvalidConfigTestCase( - description="rejects contract class names that are not identifiers", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:Strict.Store"\n' - 'paths = ["src/demo/*"]\n' - 'reason = "forced"\n' - ), - expected_error_fragment="contract_exemption entry 1 contract must be", - ), - InvalidConfigTestCase( - description="rejects contract exemptions without a contract", - toml_text='[[contract_exemption]]\npaths = ["src/demo/*"]\nreason = "forced"\n', - expected_error_fragment="contract_exemption entry 1 contract must be", - ), - InvalidConfigTestCase( - description="rejects contract exemptions without paths", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:StrictStore"\n' - 'reason = "forced"\n' - ), - expected_error_fragment="contract_exemption entry 1 needs a non-empty paths list", - ), - InvalidConfigTestCase( - description="rejects malformed forbidden owners", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:StrictStore"\n' - 'forbidden_owners = ["BaseStore"]\n' - 'paths = ["src/demo/*"]\n' - 'reason = "forced"\n' - ), - expected_error_fragment="contract_exemption entry 1 forbidden_owners must be", - ), - InvalidConfigTestCase( - description="rejects a non-list forbidden owners value", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:StrictStore"\n' - 'forbidden_owners = "src/demo/base.py:BaseStore"\n' - 'paths = ["src/demo/*"]\n' - 'reason = "forced"\n' - ), - expected_error_fragment="contract_exemption entry 1 forbidden_owners must be a list", - ), - InvalidConfigTestCase( - description="rejects contract exemptions with unknown keys", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:StrictStore"\n' - 'paths = ["src/demo/*"]\n' - 'reason = "forced"\n' - 'methods = ["write_orders"]\n' - ), - expected_error_fragment="contract_exemption entry 1 has unknown keys: methods", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_config_when_loading_then_raises_config_error( - test_case: InvalidConfigTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - config_path.write_text(test_case.toml_text, encoding="utf-8") - - with pytest.raises(DupscoreConfigError) as raised: - _ = load_config(config_path) - - assert test_case.expected_error_fragment in str(raised.value) - - -@pytest.mark.parametrize( - "test_case", - [ - CloneAllowlistTestCase( - description="loads clone allowlist globs and strips the reason", - toml_text=( - '[[clone_allowlist]]\npaths = ["src/sqlbuild/example/*", "crates/*"]\n' - 'reason = " Backends mirror one protocol. "\n' - ), - expected_entries=( - CloneAllowlistEntry( - paths=("src/sqlbuild/example/*", "crates/*"), - reason="Backends mirror one protocol.", - ), - ), - ), - CloneAllowlistTestCase( - description="missing clone allowlist is empty", - toml_text="", - expected_entries=(), - ), - ], - ids=lambda case: case.description, -) -def test_given_clone_allowlist_when_loading_then_returns_entries( - test_case: CloneAllowlistTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - config_path.write_text(test_case.toml_text, encoding="utf-8") - - config: DupscoreConfig = load_config(config_path) - - assert config.clone_allowlist == test_case.expected_entries - - -@pytest.mark.parametrize( - "test_case", - [ - ShippedConfigTestCase( - description="shipped dupscore.toml is valid", - relative_path="scripts/dupscore/dupscore.toml", - expected_pair_allowlist_size=9, - expected_clone_allowlist_size=0, - expected_contract_exemption_size=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_shipped_config_when_loading_then_it_is_valid( - test_case: ShippedConfigTestCase, -) -> None: - config_path: Path = Path(__file__).resolve().parents[7] / test_case.relative_path - - config: DupscoreConfig = load_config(config_path) - - assert len(config.allowlisted_pairs) == test_case.expected_pair_allowlist_size - assert len(config.clone_allowlist) == test_case.expected_clone_allowlist_size - assert len(config.contract_exemptions) == test_case.expected_contract_exemption_size - - -@pytest.mark.parametrize( - "test_case", - [ - ContractExemptionConfigTestCase( - description="loads contract, forbidden owners, globs, and the stripped reason", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:StrictStore"\n' - 'forbidden_owners = ["src/demo/base.py:BaseStore"]\n' - 'paths = ["src/demo/stores/*"]\n' - 'reason = " Stores must define every contract method. "\n' - ), - expected_entries=( - ContractExemptionEntry( - contract_path="src/demo/contract.py", - contract_class="StrictStore", - paths=("src/demo/stores/*",), - reason="Stores must define every contract method.", - forbidden_owners=(("src/demo/base.py", "BaseStore"),), - ), - ), - ), - ContractExemptionConfigTestCase( - description="forbidden owners are optional", - toml_text=( - "[[contract_exemption]]\n" - 'contract = "src/demo/contract.py:StrictStore"\n' - 'paths = ["src/demo/stores/*"]\n' - 'reason = "forced"\n' - ), - expected_entries=( - ContractExemptionEntry( - contract_path="src/demo/contract.py", - contract_class="StrictStore", - paths=("src/demo/stores/*",), - reason="forced", - ), - ), - ), - ContractExemptionConfigTestCase( - description="missing contract exemptions are empty", - toml_text="", - expected_entries=(), - ), - ], - ids=lambda case: case.description, -) -def test_given_contract_exemption_when_loading_then_returns_entries( - test_case: ContractExemptionConfigTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - config_path.write_text(test_case.toml_text, encoding="utf-8") - - config: DupscoreConfig = load_config(config_path) - - assert config.contract_exemptions == test_case.expected_entries diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/_test_types.py deleted file mode 100644 index 736574550d..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/_test_types.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ExtractModulesTestCase: - description: str - sources: dict[str, str] - expected_modules: tuple[str, ...] - - -@dataclass(frozen=True) -class ExtractCallsTestCase: - description: str - sources: dict[str, str] - function_qualified_name: str - expected_resolved_calls: tuple[str, ...] - - -@dataclass(frozen=True) -class ExtractFieldsTestCase: - description: str - sources: dict[str, str] - class_qualified_name: str - expected_field_names: tuple[str, ...] diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/helpers.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/helpers.py deleted file mode 100644 index 7a4efdaeda..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/helpers.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from scripts.dupscore.models import ClassFact, FunctionFact, ProjectFacts - - -def find_function(*, facts: ProjectFacts, qualified_name: str) -> FunctionFact: - functions: dict[str, FunctionFact] = {} - for module in facts.modules: - for function in module.functions: - functions[function.qualified_name] = function - return functions[qualified_name] - - -def find_class(*, facts: ProjectFacts, qualified_name: str) -> ClassFact: - classes: dict[str, ClassFact] = {} - for module in facts.modules: - for class_fact in module.classes: - classes[class_fact.qualified_name] = class_fact - return classes[qualified_name] diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/test_extract_project_facts.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/test_extract_project_facts.py deleted file mode 100644 index 206731804a..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/test_extract_project_facts.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts -from scripts.dupscore.models import ClassFact, FunctionFact, ProjectFacts -from tests.unit.scripts.dupscore._helpers.package_pairs.facts._test_types import ( - ExtractCallsTestCase, - ExtractFieldsTestCase, - ExtractModulesTestCase, -) -from tests.unit.scripts.dupscore._helpers.package_pairs.facts.helpers import ( - find_class, - find_function, -) - -_PLANNER_SOURCE: str = ( - "from sqlbuild.alpha.shared_leaves.leaves import load_rows\n" - "\n" - "def run_plan() -> int:\n" - " return load_rows()\n" -) -_LEAVES_SOURCE: str = "def load_rows() -> int:\n return 1\n" -_MODELS_SOURCE: str = ( - "from dataclasses import dataclass\n" - "\n" - "@dataclass(frozen=True)\n" - "class PlanOptions:\n" - " select: str\n" - " exclude: str\n" -) - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractModulesTestCase( - description="extracts sorted analyzable modules", - sources={ - "src/sqlbuild/alpha/planner/plan.py": _PLANNER_SOURCE, - "src/sqlbuild/alpha/shared_leaves/leaves.py": _LEAVES_SOURCE, - }, - expected_modules=( - "sqlbuild.alpha.planner.plan", - "sqlbuild.alpha.shared_leaves.leaves", - ), - ), - ExtractModulesTestCase( - description="excludes dbt integration modules and non-source paths", - sources={ - "src/sqlbuild/integrations/dbt/adapter.py": "def hidden() -> None:\n return None\n", - "docs/example.py": "def ignored() -> None:\n return None\n", - "src/sqlbuild/beta/keep.py": "def kept() -> None:\n return None\n", - }, - expected_modules=("sqlbuild.beta.keep",), - ), - ], - ids=lambda case: case.description, -) -def test_given_sources_when_extracting_facts_then_returns_expected_modules( - test_case: ExtractModulesTestCase, -) -> None: - facts: ProjectFacts = extract_project_facts(test_case.sources) - - assert tuple(module.module for module in facts.modules) == test_case.expected_modules - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractCallsTestCase( - description="resolves imported project calls to their owning module", - sources={ - "src/sqlbuild/alpha/planner/plan.py": _PLANNER_SOURCE, - "src/sqlbuild/alpha/shared_leaves/leaves.py": _LEAVES_SOURCE, - }, - function_qualified_name="sqlbuild.alpha.planner.plan::run_plan", - expected_resolved_calls=("sqlbuild.alpha.shared_leaves.leaves::load_rows",), - ) - ], - ids=lambda case: case.description, -) -def test_given_imported_call_when_extracting_facts_then_resolves_call_target( - test_case: ExtractCallsTestCase, -) -> None: - facts: ProjectFacts = extract_project_facts(test_case.sources) - - function: FunctionFact = find_function( - facts=facts, - qualified_name=test_case.function_qualified_name, - ) - assert function.resolved_calls == test_case.expected_resolved_calls - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractFieldsTestCase( - description="extracts dataclass field names in declaration order", - sources={"src/sqlbuild/alpha/planner/models.py": _MODELS_SOURCE}, - class_qualified_name="sqlbuild.alpha.planner.models::PlanOptions", - expected_field_names=("select", "exclude"), - ) - ], - ids=lambda case: case.description, -) -def test_given_dataclass_when_extracting_facts_then_returns_field_names( - test_case: ExtractFieldsTestCase, -) -> None: - facts: ProjectFacts = extract_project_facts(test_case.sources) - - class_fact: ClassFact = find_class( - facts=facts, - qualified_name=test_case.class_qualified_name, - ) - assert class_fact.dataclass_like - assert class_fact.field_names == test_case.expected_field_names diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/_test_types.py deleted file mode 100644 index a341d9e795..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/_test_types.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from scripts.dupscore.models import DupscoreConfig, SignalRanking - - -@dataclass(frozen=True) -class FuseRankingsTestCase: - description: str - rankings: tuple[SignalRanking, ...] - config: DupscoreConfig - expected_order: tuple[tuple[str, str], ...] - expected_allowlisted_flags: tuple[bool, ...] - - -@dataclass(frozen=True) -class DomainFilterTestCase: - description: str - rankings: tuple[SignalRanking, ...] - domain: str | None - expected_pair_count: int diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/helpers.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/helpers.py deleted file mode 100644 index db9532715e..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/helpers.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -from scripts.dupscore.models import SignalPairScore, SignalRanking - - -def ranking_of(*, signal_name: str, pairs: tuple[tuple[str, str], ...]) -> SignalRanking: - entries: tuple[SignalPairScore, ...] = tuple( - SignalPairScore( - package_pair=pair, score=float(len(pairs) - index), evidence=(f"evidence {index}",) - ) - for index, pair in enumerate(pairs) - ) - return SignalRanking(signal_name=signal_name, entries=entries) diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/test_fuse_rankings.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/test_fuse_rankings.py deleted file mode 100644 index 11ede8cbe5..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/test_fuse_rankings.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.package_pairs.fusion import filter_report_to_domain, fuse_rankings -from scripts.dupscore.constants import ( - SIGNAL_NAME_CALLGRAPH, - SIGNAL_NAME_DATACLASS_OVERLAP, - SIGNAL_NAME_SAME_NAME, -) -from scripts.dupscore.models import DupscoreConfig, DupscoreReport -from tests.unit.scripts.dupscore._helpers.package_pairs.fusion._test_types import ( - DomainFilterTestCase, - FuseRankingsTestCase, -) -from tests.unit.scripts.dupscore._helpers.package_pairs.fusion.helpers import ranking_of - -_PAIR_A: tuple[str, str] = ("sqlbuild.alpha.executor", "sqlbuild.alpha.planner") -_PAIR_B: tuple[str, str] = ("sqlbuild.alpha.cli", "sqlbuild.alpha.executor") -_PAIR_C: tuple[str, str] = ("sqlbuild.beta.left", "sqlbuild.beta.right") - - -@pytest.mark.parametrize( - "test_case", - [ - FuseRankingsTestCase( - description="pair hit by two signals outranks single-signal pairs", - rankings=( - ranking_of(signal_name=SIGNAL_NAME_CALLGRAPH, pairs=(_PAIR_B, _PAIR_A)), - ranking_of(signal_name=SIGNAL_NAME_SAME_NAME, pairs=(_PAIR_A,)), - ranking_of(signal_name=SIGNAL_NAME_DATACLASS_OVERLAP, pairs=(_PAIR_A, _PAIR_C)), - ), - config=DupscoreConfig(), - expected_order=(_PAIR_A, _PAIR_B, _PAIR_C), - expected_allowlisted_flags=(False, False, False), - ), - FuseRankingsTestCase( - description="allowlisted pairs keep their reason and marker", - rankings=(ranking_of(signal_name=SIGNAL_NAME_CALLGRAPH, pairs=(_PAIR_A,)),), - config=DupscoreConfig(allowlisted_pairs={_PAIR_A: "intentional mirror"}), - expected_order=(_PAIR_A,), - expected_allowlisted_flags=(True,), - ), - ], - ids=lambda case: case.description, -) -def test_given_rankings_when_fusing_then_returns_expected_order( - test_case: FuseRankingsTestCase, -) -> None: - report: DupscoreReport = fuse_rankings( - rankings=test_case.rankings, - config=test_case.config, - revision_label="worktree", - ) - - assert tuple(entry.package_pair for entry in report.entries) == test_case.expected_order - allowlisted_flags: tuple[bool, ...] = tuple(entry.allowlisted for entry in report.entries) - assert allowlisted_flags == test_case.expected_allowlisted_flags - - -@pytest.mark.parametrize( - "test_case", - [ - DomainFilterTestCase( - description="keeps only pairs mentioning the requested domain", - rankings=(ranking_of(signal_name=SIGNAL_NAME_CALLGRAPH, pairs=(_PAIR_A, _PAIR_C)),), - domain="alpha", - expected_pair_count=1, - ), - DomainFilterTestCase( - description="returns everything when no domain is requested", - rankings=(ranking_of(signal_name=SIGNAL_NAME_CALLGRAPH, pairs=(_PAIR_A, _PAIR_C)),), - domain=None, - expected_pair_count=2, - ), - ], - ids=lambda case: case.description, -) -def test_given_report_when_filtering_domain_then_returns_expected_pair_count( - test_case: DomainFilterTestCase, -) -> None: - report: DupscoreReport = fuse_rankings( - rankings=test_case.rankings, - config=DupscoreConfig(), - revision_label="worktree", - ) - - filtered: DupscoreReport = filter_report_to_domain(report=report, domain=test_case.domain) - - assert filtered.total_pairs == test_case.expected_pair_count diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/_test_types.py deleted file mode 100644 index 91a1d411f1..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/_test_types.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class DataclassOverlapPairTestCase: - description: str - sources: dict[str, str] - expected_top_pair: tuple[str, str] - - -@dataclass(frozen=True) -class DataclassOverlapSuppressionTestCase: - description: str - sources: dict[str, str] - expected_entry_count: int diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/test_dataclass_overlap_signal.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/test_dataclass_overlap_signal.py deleted file mode 100644 index 9c51f75e73..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/test_dataclass_overlap_signal.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts -from scripts.dupscore._helpers.package_pairs.signal_dataclasses import score_dataclass_overlap -from scripts.dupscore.models import SignalRanking -from tests.unit.scripts.dupscore._helpers.package_pairs.signal_dataclasses._test_types import ( - DataclassOverlapPairTestCase, - DataclassOverlapSuppressionTestCase, -) - -_PLANNER_MODEL: str = ( - "from dataclasses import dataclass\n" - "\n" - "@dataclass(frozen=True)\n" - "class PlanOptions:\n" - " select: str\n" - " exclude: str\n" - " full_refresh: bool\n" - " changes_only: bool\n" - " cli_vars: dict\n" -) -_EXECUTOR_MODEL: str = ( - "from dataclasses import dataclass\n" - "\n" - "@dataclass(frozen=True)\n" - "class BuildOptions:\n" - " select: str\n" - " exclude: str\n" - " full_refresh: bool\n" - " changes_only: bool\n" - " concurrency: int\n" -) -_SMALL_MODEL: str = ( - "from dataclasses import dataclass\n" - "\n" - "@dataclass(frozen=True)\n" - "class Tiny:\n" - " left: str\n" - " right: str\n" -) - - -@pytest.mark.parametrize( - "test_case", - [ - DataclassOverlapPairTestCase( - description="ranks models sharing at least four field names", - sources={ - "src/sqlbuild/alpha/planner/models.py": _PLANNER_MODEL, - "src/sqlbuild/alpha/executor/models.py": _EXECUTOR_MODEL, - }, - expected_top_pair=("sqlbuild.alpha.executor", "sqlbuild.alpha.planner"), - ) - ], - ids=lambda case: case.description, -) -def test_given_overlapping_models_when_scoring_then_ranks_pair( - test_case: DataclassOverlapPairTestCase, -) -> None: - ranking: SignalRanking = score_dataclass_overlap(extract_project_facts(test_case.sources)) - - assert ranking.entries[0].package_pair == test_case.expected_top_pair - - -@pytest.mark.parametrize( - "test_case", - [ - DataclassOverlapSuppressionTestCase( - description="ignores models below the shared-field threshold", - sources={ - "src/sqlbuild/alpha/planner/models.py": _SMALL_MODEL, - "src/sqlbuild/alpha/executor/models.py": _SMALL_MODEL, - }, - expected_entry_count=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_small_models_when_scoring_then_returns_no_pairs( - test_case: DataclassOverlapSuppressionTestCase, -) -> None: - ranking: SignalRanking = score_dataclass_overlap(extract_project_facts(test_case.sources)) - - assert len(ranking.entries) == test_case.expected_entry_count diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/_test_types.py deleted file mode 100644 index 628c8fc4ab..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/_test_types.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class SameNamePairTestCase: - description: str - sources: dict[str, str] - expected_top_pair: tuple[str, str] - - -@dataclass(frozen=True) -class SameNameSuppressionTestCase: - description: str - sources: dict[str, str] - expected_entry_count: int diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/test_same_name_signal.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/test_same_name_signal.py deleted file mode 100644 index b174c6c6c1..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/test_same_name_signal.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts -from scripts.dupscore._helpers.package_pairs.signal_same_names import score_same_name_symbols -from scripts.dupscore.models import SignalRanking -from tests.unit.scripts.dupscore._helpers.package_pairs.signal_same_names._test_types import ( - SameNamePairTestCase, - SameNameSuppressionTestCase, -) - -_TWIN_BODY: str = ( - "def resolve_target_relation(name: str) -> str:\n return name.strip().lower()\n" -) -_MIXED_BODY: str = "def _hidden() -> int:\n return 1\n\ndef run() -> int:\n return 2\n" - - -@pytest.mark.parametrize( - "test_case", - [ - SameNamePairTestCase( - description="ranks a cross-package public twin with similar bodies", - sources={ - "src/sqlbuild/alpha/planner/targets.py": _TWIN_BODY, - "src/sqlbuild/alpha/executor/rewrite.py": _TWIN_BODY, - }, - expected_top_pair=("sqlbuild.alpha.executor", "sqlbuild.alpha.planner"), - ) - ], - ids=lambda case: case.description, -) -def test_given_cross_package_twin_when_scoring_same_names_then_ranks_pair( - test_case: SameNamePairTestCase, -) -> None: - ranking: SignalRanking = score_same_name_symbols(extract_project_facts(test_case.sources)) - - assert ranking.entries[0].package_pair == test_case.expected_top_pair - - -@pytest.mark.parametrize( - "test_case", - [ - SameNameSuppressionTestCase( - description="filters same-package main and helpers wrapper twins", - sources={ - "src/sqlbuild/alpha/planner/main/resolve.py": _TWIN_BODY, - "src/sqlbuild/alpha/planner/_helpers/resolve.py": _TWIN_BODY, - }, - expected_entry_count=0, - ), - SameNameSuppressionTestCase( - description="ignores private and generic function names", - sources={ - "src/sqlbuild/alpha/planner/one.py": _MIXED_BODY, - "src/sqlbuild/alpha/executor/two.py": _MIXED_BODY, - }, - expected_entry_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_wrapper_or_generic_names_when_scoring_same_names_then_suppresses_pairs( - test_case: SameNameSuppressionTestCase, -) -> None: - ranking: SignalRanking = score_same_name_symbols(extract_project_facts(test_case.sources)) - - assert len(ranking.entries) == test_case.expected_entry_count diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/_test_types.py deleted file mode 100644 index 24dda5687b..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/_test_types.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class StateFaninPairTestCase: - description: str - sources: dict[str, str] - persisted_state_surfaces: tuple[str, ...] - expected_top_pair: tuple[str, str] - expected_top_score: float - - -@dataclass(frozen=True) -class StateFaninSuppressionTestCase: - description: str - sources: dict[str, str] - persisted_state_surfaces: tuple[str, ...] - expected_entry_count: int diff --git a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/test_state_fanin_signal.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/test_state_fanin_signal.py deleted file mode 100644 index 7380eadf57..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/test_state_fanin_signal.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts -from scripts.dupscore._helpers.package_pairs.signal_state_fanin import score_state_fanin -from scripts.dupscore.models import DupscoreConfig, SignalRanking -from tests.unit.scripts.dupscore._helpers.package_pairs.signal_state_fanin._test_types import ( - StateFaninPairTestCase, - StateFaninSuppressionTestCase, -) - -_STATE_CLASS: str = ( - "class StateBackend:\n" - " def get_model_refs(self) -> list[str]:\n" - " return []\n" - "\n" - " def get_seed_refs(self) -> list[str]:\n" - " return []\n" -) -_PLANNER_READER: str = "def plan(backend: object) -> None:\n backend.get_model_refs()\n backend.get_seed_refs()\n" -_EXECUTOR_READER: str = "def build(backend: object) -> None:\n backend.get_model_refs()\n backend.get_seed_refs()\n" - - -@pytest.mark.parametrize( - "test_case", - [ - StateFaninPairTestCase( - description="scores two packages independently reading two state methods", - sources={ - "src/sqlbuild/alpha/state/backend.py": _STATE_CLASS, - "src/sqlbuild/alpha/planner/plan.py": _PLANNER_READER, - "src/sqlbuild/alpha/executor/build.py": _EXECUTOR_READER, - }, - persisted_state_surfaces=("sqlbuild.alpha.state",), - expected_top_pair=("sqlbuild.alpha.executor", "sqlbuild.alpha.planner"), - expected_top_score=2.0, - ) - ], - ids=lambda case: case.description, -) -def test_given_two_reading_packages_when_scoring_fanin_then_scores_pair( - test_case: StateFaninPairTestCase, -) -> None: - config: DupscoreConfig = DupscoreConfig( - persisted_state_surfaces=test_case.persisted_state_surfaces - ) - - ranking: SignalRanking = score_state_fanin( - facts=extract_project_facts(test_case.sources), - config=config, - ) - - assert ranking.entries[0].package_pair == test_case.expected_top_pair - assert ranking.entries[0].score == test_case.expected_top_score - - -@pytest.mark.parametrize( - "test_case", - [ - StateFaninSuppressionTestCase( - description="returns nothing when no surfaces are configured", - sources={ - "src/sqlbuild/alpha/state/backend.py": _STATE_CLASS, - "src/sqlbuild/alpha/planner/plan.py": _PLANNER_READER, - "src/sqlbuild/alpha/executor/build.py": _EXECUTOR_READER, - }, - persisted_state_surfaces=(), - expected_entry_count=0, - ), - StateFaninSuppressionTestCase( - description="ignores reads from a single external package", - sources={ - "src/sqlbuild/alpha/state/backend.py": _STATE_CLASS, - "src/sqlbuild/alpha/planner/plan.py": _PLANNER_READER, - }, - persisted_state_surfaces=("sqlbuild.alpha.state",), - expected_entry_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_insufficient_readers_when_scoring_fanin_then_returns_no_pairs( - test_case: StateFaninSuppressionTestCase, -) -> None: - config: DupscoreConfig = DupscoreConfig( - persisted_state_surfaces=test_case.persisted_state_surfaces - ) - - ranking: SignalRanking = score_state_fanin( - facts=extract_project_facts(test_case.sources), - config=config, - ) - - assert len(ranking.entries) == test_case.expected_entry_count diff --git a/tests/unit/scripts/dupscore/main/build_clone_report/_test_types.py b/tests/unit/scripts/dupscore/main/build_clone_report/_test_types.py deleted file mode 100644 index 3374deaf85..0000000000 --- a/tests/unit/scripts/dupscore/main/build_clone_report/_test_types.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from scripts.dupscore.models import CloneAllowlistEntry, ContractExemptionEntry - - -@dataclass(frozen=True) -class SeededCloneTestCase: - description: str - left_path: str - right_path: str - expected_category: str - - -@dataclass(frozen=True) -class ClusterShapeTestCase: - description: str - expected_languages: list[tuple[str, ...]] - expected_sizes: list[int] - - -@dataclass(frozen=True) -class ExcludedUnitTestCase: - description: str - include_tests: bool - name: str - expected_reported: bool - - -@dataclass(frozen=True) -class ReportedPathTestCase: - description: str - include_tests: bool - path: str - expected_reported: bool - - -@dataclass(frozen=True) -class AllowlistTestCase: - description: str - entries: tuple[CloneAllowlistEntry, ...] - expected_languages: list[tuple[str, ...]] - expected_allowlisted_pairs: int - - -@dataclass(frozen=True) -class PathFilterTestCase: - description: str - path_globs: tuple[str, ...] - expected_languages: list[tuple[str, ...]] - - -@dataclass(frozen=True) -class SinceFilterTestCase: - description: str - changed_files: dict[str, str] - expected_changes: dict[str, str | None] - - -@dataclass(frozen=True) -class SinceUnchangedTestCase: - description: str - changed_files: dict[str, str] - expected_cluster_count: int - - -@dataclass(frozen=True) -class ContractExemptionReportTestCase: - description: str - entries: tuple[ContractExemptionEntry, ...] - expected_clusters: list[list[str]] - expected_contract_exempt_members: int - - -@dataclass(frozen=True) -class ContractExemptionSinceTestCase: - description: str - changed_files: dict[str, str] - expected_clusters: list[list[str]] - expected_member_changes: dict[str, str | None] - - -@dataclass(frozen=True) -class ForcedMemberOrderTestCase: - description: str - expected_member_flags: list[list[tuple[str, bool]]] diff --git a/tests/unit/scripts/dupscore/main/build_clone_report/helpers.py b/tests/unit/scripts/dupscore/main/build_clone_report/helpers.py deleted file mode 100644 index 963e58eddc..0000000000 --- a/tests/unit/scripts/dupscore/main/build_clone_report/helpers.py +++ /dev/null @@ -1,287 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from scripts.dupscore.constants import ( - DEFAULT_CLONE_MIN_SIMILARITY, - DEFAULT_CLONE_MIN_TOKENS, - SUPPORTED_LANGUAGES, -) -from scripts.dupscore.models import CloneCluster, CloneMember, CloneOptions, CloneReport -from tests.unit.scripts.dupscore.main.build_report.helpers import write_project_files - -PYTHON_BASE: str = ''' -def summarize_orders(orders, threshold): - """Rank customers by order totals.""" - totals = {} - for order in orders: - customer = order["customer"] - amount = order["amount"] - if amount < threshold: - continue - totals[customer] = totals.get(customer, 0) + amount - ranked = sorted(totals.items(), key=lambda item: item[1], reverse=True) - result = [] - for customer, total in ranked: - result.append(f"{customer}: {total}") - return result -''' - -PYTHON_EXACT_COPY: str = """ -import logging - - -@staticmethod -def summarize_orders(orders: list[dict[str, int]], threshold: int) -> list[str]: - # Copied without the docstring, with annotations and a comment. - totals: dict[str, int] = {} - for order in orders: - customer = order["customer"] - amount = order["amount"] - if amount < threshold: - continue - totals[customer] = totals.get(customer, 0) + amount - ranked = sorted(totals.items(), key=lambda item: item[1], reverse=True) - result = [] - for customer, total in ranked: - result.append(f"{customer}: {total}") - return result -""" - -PYTHON_RENAMED_COPY: str = """ -class InventoryReport: - @staticmethod - def rank_products(products, minimum): - per_product = {} - for product in products: - sku = product["sku"] - quantity = product["quantity"] - if quantity < minimum: - continue - per_product[sku] = per_product.get(sku, 1) + quantity - ordered = sorted(per_product.items(), key=lambda entry: entry[0], reverse=True) - lines = [] - for sku, count in ordered: - lines.append(f"{sku} = {count}") - return lines -""" - -PYTHON_NEAR_MISS_COPY: str = """ -def summarize_tickets(tickets, threshold): - totals = {} - for ticket in tickets: - queue = ticket["queue"] - minutes = ticket["minutes"] - if not queue: - continue - if minutes < threshold: - continue - totals[queue] = totals.get(queue, 0) + minutes - ranked = sorted(totals.items(), key=lambda item: item[1], reverse=True) - result = ["support queues"] - for queue, total in ranked: - result.append(f"{queue}: {total}") - return result -""" - -PYTHON_UNRELATED: str = """ -def parse_fulfillment_window(text): - parts = text.split("-") - if len(parts) != 2: - raise ValueError(text) - start, end = (int(value) for value in parts) - while start > end: - end += 24 - with open("windows.log", "a") as handle: - handle.write(str((start, end))) - try: - return {"start": start, "end": end, "span": end - start} - except KeyError: - return None -""" - -PYTHON_SHORT_HELPER: str = """ -def add_quantities(left, right): - return left + right -""" - -RUST_BASE: str = """ -use std::collections::HashMap; - -/// Rank inventory by quantity. -pub fn total_inventory(items: &[Item], minimum: u32) -> Vec { - let mut totals: HashMap = HashMap::new(); - for item in items { - if item.quantity < minimum { - continue; - } - *totals.entry(item.sku.clone()).or_insert(0) += item.quantity; - } - let mut ranked: Vec<(String, u32)> = totals.into_iter().collect(); - ranked.sort_by(|left, right| right.1.cmp(&left.1)); - ranked - .iter() - .map(|(sku, total)| format!("{sku}: {total}")) - .collect() -} -""" - -RUST_RENAMED_COPY: str = """ -pub struct OrderBook; - -impl OrderBook { - #[inline] - pub(crate) fn rank_orders(orders: &[Order], floor: u64) -> Vec { - let mut per_customer: HashMap = HashMap::new(); - for order in orders { - if order.amount < floor { - continue; - } - *per_customer.entry(order.customer.clone()).or_insert(7) += order.amount; - } - let mut sorted: Vec<(String, u64)> = per_customer.into_iter().collect(); - sorted.sort_by(|a, b| b.1.cmp(&a.1)); - sorted - .iter() - .map(|(customer, amount)| format!("{customer} -> {amount}")) - .collect() - } -} -""" - -RUST_NEAR_MISS_COPY: str = """ -/* A block comment /* with a nested { brace */ that must not confuse matching. */ -fn tally_products(products: &[Product], minimum: u32) -> Vec { - let mut totals: HashMap = HashMap::new(); - for product in products { - if product.sku.is_empty() { - continue; - } - if product.quantity < minimum { - continue; - } - *totals.entry(product.sku.clone()).or_insert(0) += product.quantity; - } - let mut ranked: Vec<(String, u32)> = totals.into_iter().collect(); - ranked.sort_by(|left, right| right.1.cmp(&left.1)); - ranked - .iter() - .map(|(sku, total)| format!(r#"{sku}: "{total}""#)) - .collect() -} -""" - -RUST_TEST_ONLY_COPIES: str = ( - """ -#[cfg(test)] -mod checks; - -#[cfg(test)] -mod tests { - use super::*; -""" - + RUST_BASE.replace("total_inventory", "inline_test_copy") - + """ -} -""" -) - -RUST_UNRELATED: str = """ -pub fn parse_window<'a>(text: &'a str) -> Option<(u32, u32)> { - let mut parts = text.split('-'); - let start: u32 = parts.next()?.trim().parse().ok()?; - let end: u32 = parts.next()?.trim().parse().ok()?; - match (start, end) { - (s, e) if s <= e => Some((s, e)), - (s, e) => Some((s, e + 24)), - } -} -""" - -RUST_EXACT_COPY: str = ( - RUST_BASE.replace("/// Rank inventory by quantity.", "// Kept in sync by hand.") - .replace("pub fn", "pub(crate) fn") - .replace(" .collect()", " .collect() // done") -) - -SEEDED_FILES: dict[str, str] = { - "src/sqlbuild/alpha/orders.py": PYTHON_BASE + PYTHON_SHORT_HELPER, - "src/sqlbuild/beta/orders_copy.py": PYTHON_EXACT_COPY, - "src/sqlbuild/beta/inventory.py": PYTHON_RENAMED_COPY + PYTHON_SHORT_HELPER, - "src/sqlbuild/gamma/support.py": PYTHON_NEAR_MISS_COPY, - "src/sqlbuild/gamma/windows.py": PYTHON_UNRELATED, - "crates/demo-rules/src/lib.rs": RUST_BASE + RUST_TEST_ONLY_COPIES, - "crates/demo-rules/src/checks.rs": RUST_BASE.replace("total_inventory", "external_test_copy"), - "crates/demo-rules/src/copy.rs": RUST_EXACT_COPY, - "crates/demo-rules/src/orders.rs": RUST_RENAMED_COPY, - "crates/demo-rules/src/products.rs": RUST_NEAR_MISS_COPY, - "crates/demo-rules/src/windows.rs": RUST_UNRELATED, - "crates/demo-rules/tests/integration.rs": RUST_BASE, -} - - -def clone_options( - *, - include_tests: bool = False, - since: str | None = None, - path_globs: tuple[str, ...] = (), -) -> CloneOptions: - return CloneOptions( - languages=SUPPORTED_LANGUAGES, - include_tests=include_tests, - min_tokens=DEFAULT_CLONE_MIN_TOKENS, - min_similarity=DEFAULT_CLONE_MIN_SIMILARITY, - path_globs=path_globs, - since=since, - ) - - -def seed_repository(tmp_path: Path) -> Path: - repo_root: Path = tmp_path / "repo" - write_project_files(repo_root=repo_root, files=SEEDED_FILES) - return repo_root - - -def link_categories(report: CloneReport) -> dict[frozenset[str], str]: - categories: dict[frozenset[str], str] = {} - for cluster in report.clusters: - paths: list[str] = [member.path for member in cluster.members] - for link in cluster.links: - categories[frozenset((paths[link.left], paths[link.right]))] = link.category - return categories - - -def reported_members(report: CloneReport) -> list[CloneMember]: - members: list[CloneMember] = [] - for cluster in report.clusters: - members.extend(cluster.members) - return members - - -def cluster_languages(report: CloneReport) -> list[tuple[str, ...]]: - languages: list[tuple[str, ...]] = [] - for cluster in report.clusters: - languages.append(tuple(sorted({member.language for member in cluster.members}))) - return sorted(languages) - - -def member_changes(cluster: CloneCluster) -> dict[str, str | None]: - return {member.path: member.change for member in cluster.members} - - -def cluster_member_names(report: CloneReport) -> list[list[str]]: - names: list[list[str]] = [] - for cluster in report.clusters: - names.append(sorted(member.name for member in cluster.members)) - return sorted(names) - - -def member_changes_by_name(report: CloneReport) -> dict[str, str | None]: - return {member.name: member.change for member in reported_members(report)} - - -def cluster_member_flags(report: CloneReport) -> list[list[tuple[str, bool]]]: - flags: list[list[tuple[str, bool]]] = [] - for cluster in report.clusters: - flags.append([(member.name, member.forced_override) for member in cluster.members]) - return sorted(flags) diff --git a/tests/unit/scripts/dupscore/main/build_clone_report/test_build_clone_report.py b/tests/unit/scripts/dupscore/main/build_clone_report/test_build_clone_report.py deleted file mode 100644 index 61536a7a31..0000000000 --- a/tests/unit/scripts/dupscore/main/build_clone_report/test_build_clone_report.py +++ /dev/null @@ -1,505 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from scripts.dupscore.constants import ( - CATEGORY_EXACT, - CATEGORY_NEAR_MISS, - CATEGORY_RENAMED, - CHANGE_CHANGED, - CHANGE_NEW, -) -from scripts.dupscore.main.build_clone_report import build_clone_report -from scripts.dupscore.models import CloneAllowlistEntry, CloneReport, DupscoreConfig -from tests.unit.scripts.dupscore._helpers.contracts.forced_overrides.helpers import ( - CONTRACT_FILES, - DELTA_PATH, - RENDER_ORDERS_SQL, - STORE_EXEMPTION, - WRITE_ORDERS, -) -from tests.unit.scripts.dupscore.main.build_clone_report._test_types import ( - AllowlistTestCase, - ClusterShapeTestCase, - ContractExemptionReportTestCase, - ContractExemptionSinceTestCase, - ExcludedUnitTestCase, - ForcedMemberOrderTestCase, - PathFilterTestCase, - ReportedPathTestCase, - SeededCloneTestCase, - SinceFilterTestCase, - SinceUnchangedTestCase, -) -from tests.unit.scripts.dupscore.main.build_clone_report.helpers import ( - PYTHON_BASE, - PYTHON_NEAR_MISS_COPY, - clone_options, - cluster_languages, - cluster_member_flags, - cluster_member_names, - link_categories, - member_changes, - member_changes_by_name, - reported_members, - seed_repository, -) -from tests.unit.scripts.dupscore.main.build_report.helpers import ( - commit_all, - initialize_repo, - write_project_files, -) - -_DELTA_HEADER: str = ( - "from sqlbuild.demo.contract.base_store import BaseStore\n\n\nclass DeltaStore(BaseStore):\n" -) -_VOLUNTARY_CLUSTERS: list[list[str]] = [ - ["AlphaStore._render_orders_sql", "BetaStore._render_orders_sql"], - ["AlphaStore.delete_orders", "GammaStore.delete_orders"], - ["BetaStore.read_orders", "OrdersConnection.read_orders"], -] - - -@pytest.mark.parametrize( - "test_case", - [ - SeededCloneTestCase( - description="python copy with annotations and comments is exact", - left_path="src/sqlbuild/alpha/orders.py", - right_path="src/sqlbuild/beta/orders_copy.py", - expected_category=CATEGORY_EXACT, - ), - SeededCloneTestCase( - description="python method with renamed identifiers and literals is renamed", - left_path="src/sqlbuild/alpha/orders.py", - right_path="src/sqlbuild/beta/inventory.py", - expected_category=CATEGORY_RENAMED, - ), - SeededCloneTestCase( - description="python copy with extra statements is a near miss", - left_path="src/sqlbuild/alpha/orders.py", - right_path="src/sqlbuild/gamma/support.py", - expected_category=CATEGORY_NEAR_MISS, - ), - SeededCloneTestCase( - description="rust copy with different comments and visibility is exact", - left_path="crates/demo-rules/src/lib.rs", - right_path="crates/demo-rules/src/copy.rs", - expected_category=CATEGORY_EXACT, - ), - SeededCloneTestCase( - description="rust impl method with renamed identifiers is renamed", - left_path="crates/demo-rules/src/lib.rs", - right_path="crates/demo-rules/src/orders.rs", - expected_category=CATEGORY_RENAMED, - ), - SeededCloneTestCase( - description="rust copy with an extra guard and raw string is a near miss", - left_path="crates/demo-rules/src/lib.rs", - right_path="crates/demo-rules/src/products.rs", - expected_category=CATEGORY_NEAR_MISS, - ), - ], - ids=lambda case: case.description, -) -def test_given_seeded_clones_when_building_report_then_links_expected_category( - test_case: SeededCloneTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - - report: CloneReport = build_clone_report( - repo_root=repo_root, options=clone_options(), config=DupscoreConfig() - ) - - categories: dict[frozenset[str], str] = link_categories(report) - assert categories[frozenset((test_case.left_path, test_case.right_path))] == ( - test_case.expected_category - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ClusterShapeTestCase( - description="each language forms one four-member cluster", - expected_languages=[("python",), ("rust",)], - expected_sizes=[4, 4], - ) - ], - ids=lambda case: case.description, -) -def test_given_seeded_clones_when_building_report_then_groups_transitive_clusters( - test_case: ClusterShapeTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - - report: CloneReport = build_clone_report( - repo_root=repo_root, options=clone_options(), config=DupscoreConfig() - ) - - assert cluster_languages(report) == test_case.expected_languages - assert sorted(len(cluster.members) for cluster in report.clusters) == test_case.expected_sizes - - -@pytest.mark.parametrize( - "test_case", - [ - ExcludedUnitTestCase( - description="unrelated python function is not reported", - include_tests=False, - name="parse_fulfillment_window", - expected_reported=False, - ), - ExcludedUnitTestCase( - description="short duplicated helper is below the size threshold", - include_tests=False, - name="add_quantities", - expected_reported=False, - ), - ExcludedUnitTestCase( - description="unrelated rust function is not reported", - include_tests=False, - name="parse_window", - expected_reported=False, - ), - ExcludedUnitTestCase( - description="copy inside a cfg(test) module is skipped by default", - include_tests=False, - name="inline_test_copy", - expected_reported=False, - ), - ExcludedUnitTestCase( - description="copy in a cfg(test) external module file is skipped by default", - include_tests=False, - name="external_test_copy", - expected_reported=False, - ), - ExcludedUnitTestCase( - description="copy inside a cfg(test) module is reported with include-tests", - include_tests=True, - name="inline_test_copy", - expected_reported=True, - ), - ExcludedUnitTestCase( - description="copy in a cfg(test) external module is reported with include-tests", - include_tests=True, - name="external_test_copy", - expected_reported=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_seeded_tree_when_building_report_then_applies_unit_exclusions( - test_case: ExcludedUnitTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - - report: CloneReport = build_clone_report( - repo_root=repo_root, - options=clone_options(include_tests=test_case.include_tests), - config=DupscoreConfig(), - ) - - names: set[str] = {member.name for member in reported_members(report)} - assert (test_case.name in names) is test_case.expected_reported - - -@pytest.mark.parametrize( - "test_case", - [ - ReportedPathTestCase( - description="rust tests directory is skipped by default", - include_tests=False, - path="crates/demo-rules/tests/integration.rs", - expected_reported=False, - ), - ReportedPathTestCase( - description="rust tests directory is analyzed with include-tests", - include_tests=True, - path="crates/demo-rules/tests/integration.rs", - expected_reported=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_rust_test_directory_when_building_report_then_respects_include_tests( - test_case: ReportedPathTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - - report: CloneReport = build_clone_report( - repo_root=repo_root, - options=clone_options(include_tests=test_case.include_tests), - config=DupscoreConfig(), - ) - - paths: set[str] = {member.path for member in reported_members(report)} - assert (test_case.path in paths) is test_case.expected_reported - - -@pytest.mark.parametrize( - "test_case", - [ - AllowlistTestCase( - description="entry covering both sides suppresses the rust clones", - entries=(CloneAllowlistEntry(paths=("crates/*",), reason="Mirrored rule helpers."),), - expected_languages=[("python",)], - expected_allowlisted_pairs=6, - ), - AllowlistTestCase( - description="entry covering only one side keeps the pair", - entries=( - CloneAllowlistEntry( - paths=("crates/demo-rules/src/lib.rs",), reason="Only one side." - ), - ), - expected_languages=[("python",), ("rust",)], - expected_allowlisted_pairs=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_clone_allowlist_when_building_report_then_suppresses_covered_pairs( - test_case: AllowlistTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - - report: CloneReport = build_clone_report( - repo_root=repo_root, - options=clone_options(), - config=DupscoreConfig(clone_allowlist=test_case.entries), - ) - - assert report.allowlisted_pairs == test_case.expected_allowlisted_pairs - assert cluster_languages(report) == test_case.expected_languages - - -@pytest.mark.parametrize( - "test_case", - [ - PathFilterTestCase( - description="glob keeps only clusters with a matching member", - path_globs=("*/gamma/*",), - expected_languages=[("python",)], - ), - PathFilterTestCase( - description="glob without matches keeps nothing", - path_globs=("docs/*",), - expected_languages=[], - ), - ], - ids=lambda case: case.description, -) -def test_given_path_filter_when_building_report_then_keeps_matching_clusters( - test_case: PathFilterTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - - report: CloneReport = build_clone_report( - repo_root=repo_root, - options=clone_options(path_globs=test_case.path_globs), - config=DupscoreConfig(), - ) - - assert cluster_languages(report) == test_case.expected_languages - - -@pytest.mark.parametrize( - "test_case", - [ - SinceUnchangedTestCase( - description="unchanged worktree reports nothing", - changed_files={}, - expected_cluster_count=0, - ), - SinceUnchangedTestCase( - description="edits outside clone members report nothing", - changed_files={"src/sqlbuild/gamma/notes.py": "VALUE = 1\n"}, - expected_cluster_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_since_revision_without_clone_edits_when_building_report_then_reports_nothing( - test_case: SinceUnchangedTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - initialize_repo(repo_root) - base_revision: str = commit_all(repo_root=repo_root, message="seed clones") - write_project_files(repo_root=repo_root, files=test_case.changed_files) - - report: CloneReport = build_clone_report( - repo_root=repo_root, options=clone_options(since=base_revision), config=DupscoreConfig() - ) - - assert len(report.clusters) == test_case.expected_cluster_count - - -@pytest.mark.parametrize( - "test_case", - [ - SinceFilterTestCase( - description="untracked copy is new and keeps its cluster", - changed_files={"src/sqlbuild/delta/branch_copy.py": PYTHON_BASE}, - expected_changes={ - "src/sqlbuild/delta/branch_copy.py": CHANGE_NEW, - "src/sqlbuild/alpha/orders.py": None, - }, - ), - SinceFilterTestCase( - description="edited clone member is marked changed", - changed_files={ - "src/sqlbuild/gamma/support.py": PYTHON_NEAR_MISS_COPY.replace( - 'result = ["support queues"]', 'result = ["support queue totals"]' - ) - }, - expected_changes={ - "src/sqlbuild/gamma/support.py": CHANGE_CHANGED, - "src/sqlbuild/alpha/orders.py": None, - }, - ), - ], - ids=lambda case: case.description, -) -def test_given_since_revision_when_building_report_then_marks_touched_members( - test_case: SinceFilterTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = seed_repository(tmp_path) - initialize_repo(repo_root) - base_revision: str = commit_all(repo_root=repo_root, message="seed clones") - write_project_files(repo_root=repo_root, files=test_case.changed_files) - - report: CloneReport = build_clone_report( - repo_root=repo_root, options=clone_options(since=base_revision), config=DupscoreConfig() - ) - - assert len(report.clusters) == 1 - changes: dict[str, str | None] = member_changes(report.clusters[0]) - assert {path: changes[path] for path in test_case.expected_changes} == ( - test_case.expected_changes - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ContractExemptionReportTestCase( - description="links between forced overrides are hidden, voluntary copies stay", - entries=(STORE_EXEMPTION,), - expected_clusters=_VOLUNTARY_CLUSTERS, - expected_contract_exempt_members=3, - ), - ContractExemptionReportTestCase( - description="without an exemption every override cluster is reported", - entries=(), - expected_clusters=sorted( - [ - *_VOLUNTARY_CLUSTERS, - ["AlphaStore.write_orders", "BaseStore.write_orders", "BetaStore.write_orders"], - ] - ), - expected_contract_exempt_members=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_contract_exemption_when_building_report_then_hides_forced_overrides( - test_case: ContractExemptionReportTestCase, - tmp_path: Path, -) -> None: - write_project_files(repo_root=tmp_path, files=CONTRACT_FILES) - - report: CloneReport = build_clone_report( - repo_root=tmp_path, - options=clone_options(), - config=DupscoreConfig(contract_exemptions=test_case.entries), - ) - - assert cluster_member_names(report) == test_case.expected_clusters - assert report.contract_exempt_members == test_case.expected_contract_exempt_members - - -@pytest.mark.parametrize( - "test_case", - [ - ContractExemptionSinceTestCase( - description="new private helper copy is reported beside a hidden forced override", - changed_files={DELTA_PATH: _DELTA_HEADER + WRITE_ORDERS + RENDER_ORDERS_SQL}, - expected_clusters=[ - [ - "AlphaStore._render_orders_sql", - "BetaStore._render_orders_sql", - "DeltaStore._render_orders_sql", - ] - ], - expected_member_changes={ - "AlphaStore._render_orders_sql": None, - "BetaStore._render_orders_sql": None, - "DeltaStore._render_orders_sql": CHANGE_NEW, - }, - ), - ContractExemptionSinceTestCase( - description="new forced override alone reports nothing", - changed_files={DELTA_PATH: _DELTA_HEADER + WRITE_ORDERS}, - expected_clusters=[], - expected_member_changes={}, - ), - ], - ids=lambda case: case.description, -) -def test_given_since_revision_with_contract_exemption_when_building_report_then_hides_forced( - test_case: ContractExemptionSinceTestCase, - tmp_path: Path, -) -> None: - write_project_files(repo_root=tmp_path, files=CONTRACT_FILES) - initialize_repo(tmp_path) - base_revision: str = commit_all(repo_root=tmp_path, message="seed stores") - write_project_files(repo_root=tmp_path, files=test_case.changed_files) - - report: CloneReport = build_clone_report( - repo_root=tmp_path, - options=clone_options(since=base_revision), - config=DupscoreConfig(contract_exemptions=(STORE_EXEMPTION,)), - ) - - assert cluster_member_names(report) == test_case.expected_clusters - assert member_changes_by_name(report) == test_case.expected_member_changes - - -@pytest.mark.parametrize( - "test_case", - [ - ForcedMemberOrderTestCase( - description="voluntary copies are listed before flagged forced overrides", - expected_member_flags=[ - [("AlphaStore._render_orders_sql", False), ("BetaStore._render_orders_sql", False)], - [("GammaStore.delete_orders", False), ("AlphaStore.delete_orders", True)], - [("OrdersConnection.read_orders", False), ("BetaStore.read_orders", True)], - ], - ) - ], - ids=lambda case: case.description, -) -def test_given_contract_exemption_when_building_report_then_lists_forced_members_last( - test_case: ForcedMemberOrderTestCase, - tmp_path: Path, -) -> None: - write_project_files(repo_root=tmp_path, files=CONTRACT_FILES) - - report: CloneReport = build_clone_report( - repo_root=tmp_path, - options=clone_options(), - config=DupscoreConfig(contract_exemptions=(STORE_EXEMPTION,)), - ) - - assert cluster_member_flags(report) == test_case.expected_member_flags diff --git a/tests/unit/scripts/dupscore/main/build_report/_test_types.py b/tests/unit/scripts/dupscore/main/build_report/_test_types.py deleted file mode 100644 index b452c9713a..0000000000 --- a/tests/unit/scripts/dupscore/main/build_report/_test_types.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class BuildReportTestCase: - description: str - expected_top_pair: tuple[str, str] - expected_signal_names: tuple[str, ...] - - -@dataclass(frozen=True) -class ReportDeltaTestCase: - description: str - expected_new_twin_fragment: str diff --git a/tests/unit/scripts/dupscore/main/build_report/helpers.py b/tests/unit/scripts/dupscore/main/build_report/helpers.py deleted file mode 100644 index d2bd940cf1..0000000000 --- a/tests/unit/scripts/dupscore/main/build_report/helpers.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -_TWIN_BODY: str = ( - "def resolve_target_relation(name: str) -> str:\n return name.strip().lower()\n" -) -_LEAVES: str = ( - "def leaf_one() -> int:\n return 1\n\n" - "def leaf_two() -> int:\n return 2\n\n" - "def leaf_three() -> int:\n return 3\n\n" - "def leaf_four() -> int:\n return 4\n\n" - "def leaf_five() -> int:\n return 5\n" -) -_LEAF_IMPORT: str = ( - "from sqlbuild.alpha.primitives.leaves import (" - "leaf_one, leaf_two, leaf_three, leaf_four, leaf_five)\n" -) -_LEAF_SUM: str = "leaf_one() + leaf_two() + leaf_three() + leaf_four() + leaf_five()" -_PLANNER_ENTRY: str = ( - _LEAF_IMPORT - + "from sqlbuild.alpha.planner.middle import plan_middle\n" - + "\n" - + "def run_plan() -> int:\n" - + f" return {_LEAF_SUM} + plan_middle()\n" -) -_EXECUTOR_ENTRY: str = ( - _LEAF_IMPORT - + "from sqlbuild.alpha.executor.middle import build_middle\n" - + "\n" - + "def run_build() -> int:\n" - + f" return {_LEAF_SUM} + build_middle()\n" -) -_PLANNER_MIDDLE: str = ( - "from sqlbuild.alpha.primitives.leaves import leaf_one\n" - "\n" - "def plan_middle() -> int:\n" - " return leaf_one()\n" -) -_EXECUTOR_MIDDLE: str = ( - "from sqlbuild.alpha.primitives.leaves import leaf_two\n" - "\n" - "def build_middle() -> int:\n" - " return leaf_two()\n" -) -_STATE_CLASS: str = ( - "class StateBackend:\n" - " def get_model_refs(self) -> list[str]:\n" - " return []\n" - "\n" - " def get_seed_refs(self) -> list[str]:\n" - " return []\n" -) -_PLANNER_STATE_READER: str = ( - "def read_plan_state(backend: object) -> None:\n" - " backend.get_model_refs()\n" - " backend.get_seed_refs()\n" -) -_EXECUTOR_STATE_READER: str = ( - "def read_build_state(backend: object) -> None:\n" - " backend.get_model_refs()\n" - " backend.get_seed_refs()\n" -) - -BASE_PROJECT_FILES: dict[str, str] = { - "src/sqlbuild/alpha/primitives/leaves.py": _LEAVES, - "src/sqlbuild/alpha/planner/plan.py": _PLANNER_ENTRY, - "src/sqlbuild/alpha/planner/middle.py": _PLANNER_MIDDLE, - "src/sqlbuild/alpha/executor/build.py": _EXECUTOR_ENTRY, - "src/sqlbuild/alpha/executor/middle.py": _EXECUTOR_MIDDLE, - "src/sqlbuild/alpha/state/backend.py": _STATE_CLASS, - "src/sqlbuild/alpha/planner/bound_state.py": _PLANNER_STATE_READER, - "src/sqlbuild/alpha/executor/bound_state.py": _EXECUTOR_STATE_READER, -} - -TWIN_PROJECT_FILES: dict[str, str] = { - "src/sqlbuild/alpha/planner/targets.py": _TWIN_BODY, - "src/sqlbuild/alpha/executor/rewrite.py": _TWIN_BODY, -} - - -def write_project_files(*, repo_root: Path, files: dict[str, str]) -> None: - for relative, contents in files.items(): - target: Path = repo_root / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(contents, encoding="utf-8") - - -def run_git(*, repo_root: Path, arguments: list[str]) -> str: - completed: subprocess.CompletedProcess[str] = subprocess.run( - ["git", *arguments], - cwd=repo_root, - capture_output=True, - text=True, - check=True, - ) - return completed.stdout.strip() - - -def initialize_repo(repo_root: Path) -> None: - _ = run_git(repo_root=repo_root, arguments=["init", "--quiet"]) - _ = run_git(repo_root=repo_root, arguments=["config", "user.email", "dupscore@test.local"]) - _ = run_git(repo_root=repo_root, arguments=["config", "user.name", "Dupscore Test"]) - - -def commit_all(*, repo_root: Path, message: str) -> str: - _ = run_git(repo_root=repo_root, arguments=["add", "--all"]) - _ = run_git(repo_root=repo_root, arguments=["commit", "--quiet", "--message", message]) - return run_git(repo_root=repo_root, arguments=["rev-parse", "HEAD"]) diff --git a/tests/unit/scripts/dupscore/main/build_report/test_build_report.py b/tests/unit/scripts/dupscore/main/build_report/test_build_report.py deleted file mode 100644 index 025680ef5d..0000000000 --- a/tests/unit/scripts/dupscore/main/build_report/test_build_report.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from scripts.dupscore.main.build_pair_evidence import build_pair_evidence -from scripts.dupscore.main.build_report import build_report -from scripts.dupscore.main.build_report_delta import build_report_delta -from scripts.dupscore.models import ( - DupscoreConfig, - DupscoreReport, - PairEvidenceReport, - ReportDelta, -) -from tests.unit.scripts.dupscore.main.build_report._test_types import ( - BuildReportTestCase, - ReportDeltaTestCase, -) -from tests.unit.scripts.dupscore.main.build_report.helpers import ( - BASE_PROJECT_FILES, - TWIN_PROJECT_FILES, - commit_all, - initialize_repo, - write_project_files, -) - -_CONFIG: DupscoreConfig = DupscoreConfig(persisted_state_surfaces=("sqlbuild.alpha.state",)) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildReportTestCase( - description="ranks the planner and executor seam first with multi-signal evidence", - expected_top_pair=("sqlbuild.alpha.executor", "sqlbuild.alpha.planner"), - expected_signal_names=("callgraph_shape", "same_name_symbols", "state_fanin"), - ) - ], - ids=lambda case: case.description, -) -def test_given_seeded_repo_when_building_report_then_ranks_seam_first( - test_case: BuildReportTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = tmp_path / "repo" - repo_root.mkdir() - initialize_repo(repo_root) - write_project_files(repo_root=repo_root, files=BASE_PROJECT_FILES | TWIN_PROJECT_FILES) - _ = commit_all(repo_root=repo_root, message="seed project") - - report: DupscoreReport = build_report(repo_root=repo_root, revision=None, config=_CONFIG) - - assert report.entries - assert report.entries[0].package_pair == test_case.expected_top_pair - top_signals: tuple[str, ...] = tuple( - contribution.signal_name for contribution in report.entries[0].contributions - ) - for expected_signal in test_case.expected_signal_names: - assert expected_signal in top_signals - - evidence: PairEvidenceReport = build_pair_evidence( - report=report, - left=test_case.expected_top_pair[1], - right=test_case.expected_top_pair[0], - ) - assert evidence.combined_rank == 1 - assert evidence.contributions - - -@pytest.mark.parametrize( - "test_case", - [ - ReportDeltaTestCase( - description="reports a newly introduced twin against a base revision", - expected_new_twin_fragment="resolve_target_relation", - ) - ], - ids=lambda case: case.description, -) -def test_given_new_twin_when_comparing_revisions_then_reports_new_same_name_evidence( - test_case: ReportDeltaTestCase, - tmp_path: Path, -) -> None: - repo_root: Path = tmp_path / "repo" - repo_root.mkdir() - initialize_repo(repo_root) - write_project_files(repo_root=repo_root, files=BASE_PROJECT_FILES) - base_revision: str = commit_all(repo_root=repo_root, message="seed project without twin") - write_project_files(repo_root=repo_root, files=TWIN_PROJECT_FILES) - _ = commit_all(repo_root=repo_root, message="introduce twin") - - base_report: DupscoreReport = build_report( - repo_root=repo_root, revision=base_revision, config=_CONFIG - ) - current_report: DupscoreReport = build_report( - repo_root=repo_root, revision=None, config=_CONFIG - ) - delta: ReportDelta = build_report_delta(base=base_report, current=current_report, top=10) - - assert any( - test_case.expected_new_twin_fragment in evidence - for evidence in delta.new_same_name_evidence - ) diff --git a/tests/unit/scripts/dupscore/main/dupscore/_test_types.py b/tests/unit/scripts/dupscore/main/dupscore/_test_types.py deleted file mode 100644 index 707a8253c8..0000000000 --- a/tests/unit/scripts/dupscore/main/dupscore/_test_types.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class CliJsonTestCase: - description: str - arguments: tuple[str, ...] - expected_total_clusters: int - expected_unit_counts: dict[str, int] - expected_cluster_keys: frozenset[str] - expected_member_keys: frozenset[str] - expected_progress_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class CliTextTestCase: - description: str - arguments: tuple[str, ...] - expected_prefix: str - expected_fragments: tuple[str, ...] - absent_fragment: str - - -@dataclass(frozen=True) -class CliSinceTestCase: - description: str - new_files: dict[str, str] - expected_fragments: tuple[str, ...] - expected_new_markers: int - - -@dataclass(frozen=True) -class CliUsageErrorTestCase: - description: str - arguments: tuple[str, ...] - expected_error_fragment: str - - -@dataclass(frozen=True) -class CliHelpTestCase: - description: str - expected_fragments: tuple[str, ...] diff --git a/tests/unit/scripts/dupscore/main/dupscore/test_dupscore_cli.py b/tests/unit/scripts/dupscore/main/dupscore/test_dupscore_cli.py deleted file mode 100644 index f02556a67e..0000000000 --- a/tests/unit/scripts/dupscore/main/dupscore/test_dupscore_cli.py +++ /dev/null @@ -1,226 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import pytest - -from scripts.dupscore.main.dupscore import dupscore -from tests.unit.scripts.dupscore.main.build_clone_report.helpers import ( - PYTHON_BASE, - seed_repository, -) -from tests.unit.scripts.dupscore.main.build_report.helpers import ( - commit_all, - initialize_repo, - write_project_files, -) -from tests.unit.scripts.dupscore.main.dupscore._test_types import ( - CliHelpTestCase, - CliJsonTestCase, - CliSinceTestCase, - CliTextTestCase, - CliUsageErrorTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - CliJsonTestCase( - description="json output lists ranked clusters with member locations", - arguments=("clones", "--json", "--top", "1"), - expected_total_clusters=2, - expected_unit_counts={"python": 5, "rust": 5}, - expected_cluster_keys=frozenset( - { - "category", - "duplicated_tokens", - "links", - "members", - "rank", - "similarity_max", - "similarity_min", - } - ), - expected_member_keys=frozenset( - { - "change", - "end_line", - "forced_override", - "language", - "name", - "path", - "start_line", - "tokens", - } - ), - expected_progress_fragments=( - "detecting function clones", - "found 2 clone clusters across 10 units", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_seeded_tree_when_running_json_cli_then_prints_cluster_document( - test_case: CliJsonTestCase, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], -) -> None: - repo_root: Path = seed_repository(tmp_path) - - exit_code: int = dupscore([*test_case.arguments, "--repo-root", str(repo_root)]) - - output: str - errors: str - output, errors = capsys.readouterr() - payload: dict[str, Any] = json.loads(output) - clusters: list[dict[str, Any]] = payload["clusters"] - members: list[dict[str, Any]] = clusters[0]["members"] - assert exit_code == 0 - assert payload["total_clusters"] == test_case.expected_total_clusters - assert payload["unit_counts"] == test_case.expected_unit_counts - assert payload["since"] is None - assert payload["contract_exempt_members"] == 0 - assert [set(cluster) for cluster in clusters] == [set(test_case.expected_cluster_keys)] - assert all(set(member) == test_case.expected_member_keys for member in members) - assert all(fragment in errors for fragment in test_case.expected_progress_fragments) - - -@pytest.mark.parametrize( - "test_case", - [ - CliTextTestCase( - description="default mode prints python clone locations", - arguments=("--lang", "python"), - expected_prefix="dupscore clones: 1 clusters (python 5 units;", - expected_fragments=( - "src/sqlbuild/alpha/orders.py:2-15 summarize_orders", - "src/sqlbuild/beta/inventory.py:4-16 InventoryReport.rank_products", - ), - absent_fragment="crates/", - ), - CliTextTestCase( - description="rust mode prints impl-qualified names", - arguments=("clones", "--lang", "rust"), - expected_prefix="dupscore clones: 1 clusters (rust 5 units;", - expected_fragments=("crates/demo-rules/src/orders.rs:6-20 OrderBook::rank_orders",), - absent_fragment="src/sqlbuild/", - ), - ], - ids=lambda case: case.description, -) -def test_given_seeded_tree_when_running_text_cli_then_prints_locations( - test_case: CliTextTestCase, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], -) -> None: - repo_root: Path = seed_repository(tmp_path) - - exit_code: int = dupscore([*test_case.arguments, "--repo-root", str(repo_root)]) - - output: str = capsys.readouterr().out - assert exit_code == 0 - assert output.startswith(test_case.expected_prefix) - assert all(fragment in output for fragment in test_case.expected_fragments) - assert test_case.absent_fragment not in output - - -@pytest.mark.parametrize( - "test_case", - [ - CliSinceTestCase( - description="branch copy is the only new member", - new_files={"src/sqlbuild/delta/branch_copy.py": PYTHON_BASE}, - expected_fragments=( - ": 1 clusters (", - "src/sqlbuild/delta/branch_copy.py:2-15 summarize_orders (118 tokens) [new]", - ), - expected_new_markers=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_branch_copy_when_running_cli_since_base_then_marks_new_member( - test_case: CliSinceTestCase, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], -) -> None: - repo_root: Path = seed_repository(tmp_path) - initialize_repo(repo_root) - base_revision: str = commit_all(repo_root=repo_root, message="seed clones") - write_project_files(repo_root=repo_root, files=test_case.new_files) - - exit_code: int = dupscore(["--repo-root", str(repo_root), "--since", base_revision]) - - output: str = capsys.readouterr().out - assert exit_code == 0 - assert output.startswith(f"dupscore clones changed since {base_revision}") - assert all(fragment in output for fragment in test_case.expected_fragments) - assert output.count("[new]") == test_case.expected_new_markers - - -@pytest.mark.parametrize( - "test_case", - [ - CliUsageErrorTestCase( - description="clone options are rejected in report mode", - arguments=("report", "--min-tokens", "10"), - expected_error_fragment="--min-tokens only apply to the clones mode", - ), - CliUsageErrorTestCase( - description="similarity outside the unit interval is rejected", - arguments=("clones", "--min-similarity", "1.5"), - expected_error_fragment="--min-similarity must be in (0, 1]", - ), - CliUsageErrorTestCase( - description="unknown languages are rejected", - arguments=("--lang", "go"), - expected_error_fragment="invalid choice: 'go'", - ), - CliUsageErrorTestCase( - description="domain filter is rejected in clones mode", - arguments=("--domain", "sqlbuild.compiler"), - expected_error_fragment="--domain only applies to the report mode", - ), - CliUsageErrorTestCase( - description="clones mode takes no package names", - arguments=("clones", "sqlbuild.alpha"), - expected_error_fragment="takes no positional package names", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_arguments_when_running_cli_then_exits_with_usage_error( - test_case: CliUsageErrorTestCase, - capsys: pytest.CaptureFixture[str], -) -> None: - with pytest.raises(SystemExit) as raised: - _ = dupscore(list(test_case.arguments)) - - assert raised.value.code == 2 - assert test_case.expected_error_fragment in capsys.readouterr().err - - -@pytest.mark.parametrize( - "test_case", - [ - CliHelpTestCase( - description="help lists the clone mode and its options", - expected_fragments=("clones", "report", "--include-tests", "--since", "--path"), - ) - ], - ids=lambda case: case.description, -) -def test_given_help_flag_when_running_cli_then_lists_clone_mode( - test_case: CliHelpTestCase, - capsys: pytest.CaptureFixture[str], -) -> None: - with pytest.raises(SystemExit) as raised: - _ = dupscore(["--help"]) - - help_text: str = capsys.readouterr().out - assert raised.value.code == 0 - assert all(fragment in help_text for fragment in test_case.expected_fragments) diff --git a/uv.lock b/uv.lock index e8ab530c9d..5fcd19f89a 100644 --- a/uv.lock +++ b/uv.lock @@ -1241,31 +1241,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.22.0" +version = "0.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/57/2e/095268d6d43654271a1a86b18732de7275245a554fbfcd6ca72ce6e9781a/fensu-0.22.0.tar.gz", hash = "sha256:8426e67fb1fd8950cf4e49e7bebed628315f8b5b9f321bcdfe588b49437b9922", size = 507043, upload-time = "2026-09-21T02:47:27.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/f4/d037850c050343d2548b901a994a77783e7a02d352fe35aa2477644a185c/fensu-0.23.0.tar.gz", hash = "sha256:997c31c26e9b6ea88b2797898a5644a95ae543ba568fb5f3482efd2e17b54d61", size = 508009, upload-time = "2026-09-24T15:29:54.691Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/0d/9ae677ce03d3c6e6213abbd1419fa7c68d33df1b87231a89843e78e589ff/fensu-0.22.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3eff0ea2825fef24029958ef3d172b516ae1b8b8cf1b8e5b91cf82a0af178da6", size = 4026447, upload-time = "2026-09-21T02:47:08Z" }, - { url = "https://files.pythonhosted.org/packages/72/b3/ed2700076f00d802db8423b8ba97113a78159e0cedc7b1745a499567f018/fensu-0.22.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa26a38115bc9bec6b9b18929e371bc0e4974b2d3c5b86136bd46b3f13d562a9", size = 3892007, upload-time = "2026-09-21T02:47:10.072Z" }, - { url = "https://files.pythonhosted.org/packages/22/1f/ba8aa656faae52ce39f5ace623133155a14a280e542a155b0c57451316df/fensu-0.22.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23b246e846ca29db069b8258c0c65ea04a15dbf692547d1a369d9c34090f2eb6", size = 4118931, upload-time = "2026-09-21T02:47:11.991Z" }, - { url = "https://files.pythonhosted.org/packages/a5/e2/4d47d41b84d2f1d0245a78aee8d5bd40c32cc0ee9c51bec79ba899497958/fensu-0.22.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16f61c8cc1f2872310f91b1f60cc6cac8e88d68e63351c59d96a983a076035f5", size = 4184390, upload-time = "2026-09-21T02:47:13.66Z" }, - { url = "https://files.pythonhosted.org/packages/23/71/f2ea9ddfc85ca55c2fb04e255b64db680406d467896b4b16a760e45fccca/fensu-0.22.0-cp312-abi3-win_amd64.whl", hash = "sha256:3a7b704372c7c179635ebc56804754c3eedd531018d80450f405a90ec3a68b5c", size = 3864265, upload-time = "2026-09-21T02:47:15.467Z" }, + { url = "https://files.pythonhosted.org/packages/1a/67/597b68a104ae906705e8fc27ec53f57eaad36d869a5323b71518aa77592d/fensu-0.23.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ee20effc98db0d4e682c3f3a55b72ffca81386228c9092cab82b4bd044d1ddc", size = 4026956, upload-time = "2026-09-24T15:29:34.609Z" }, + { url = "https://files.pythonhosted.org/packages/13/0e/1e4583e15a49bc066530ea88faa7257caa131a41bf084c89eb9b34a944a7/fensu-0.23.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:39f85e019e2a66a1cfa280efc02b47740fee8270a6e4befef4885f46f8e03753", size = 3892516, upload-time = "2026-09-24T15:29:36.742Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d6/a10db3b1c7f5fb667bd6e634f494d527c29637b2fb9bc8f68d13f0300482/fensu-0.23.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3002c67ba494386e1c6e94f5f2abbb4b355a174dce0ecf8ef71756c203b7d26a", size = 4119441, upload-time = "2026-09-24T15:29:38.636Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a6/0ac54e58f3d0aec31b6ad6fa46bb8d3454540e7f6b066d6832880cf81fbb/fensu-0.23.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed8a57574977971b2f3c599b5beaac9bd905612aba89971b7b1697f790310e5", size = 4184904, upload-time = "2026-09-24T15:29:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/99/57/7bb9c5c2b971c26bcbf1a1c10a54097925cbaff07f3c8bcb9aefad3d3865/fensu-0.23.0-cp312-abi3-win_amd64.whl", hash = "sha256:98d78a6582ea21fc456c18a31b7d01f3ff43b8ed687de9e51ef8497e54ee2514", size = 3864765, upload-time = "2026-09-24T15:29:42.311Z" }, ] [[package]] name = "fensu-cli" -version = "0.22.0" +version = "0.23.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ea/62665e018e4217e5e0baf33abc86820a5b47e42baa5418c4662579e37ada/fensu_cli-0.22.0.tar.gz", hash = "sha256:5c33ddc4e51269f17158643fe6779bb1afa49436c92a3a4cf910cf29188dc819", size = 667725, upload-time = "2026-09-21T02:47:28.564Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/6f/48493231455bdaceeb637594765c15c46b59252b2526b5ee9478b0c0841c/fensu_cli-0.23.0.tar.gz", hash = "sha256:0a598d0013c662b4bc4e7785dc91f8d480665a06750c2114acbafe8de9ef3dba", size = 715137, upload-time = "2026-09-24T15:29:56.142Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/ee/2405bdfc90f042c62cd582d5eacc78b115a74cbc0933fa1578192955d1dd/fensu_cli-0.22.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9e4fb1601327ca4f35bfbfc512721020c150539cc1727fe96e95680c46242dcd", size = 7356781, upload-time = "2026-09-21T02:47:17.913Z" }, - { url = "https://files.pythonhosted.org/packages/5d/56/7a53d8516583e8df639c90b0578eb10663b4e95d30fcd5299fd88293415d/fensu_cli-0.22.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:771988acd1623f370bba36be148be207efa76c546dfe96c54aa13a286f1ea5b4", size = 6936782, upload-time = "2026-09-21T02:47:19.921Z" }, - { url = "https://files.pythonhosted.org/packages/1a/97/f3d088e639ba05a56afe5131e87ed4a5604079bb998954335c6b0f397561/fensu_cli-0.22.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9db206e95d6f5d21767d7f383666200b7e11148e13c567e788cbdc49b56373ee", size = 7014215, upload-time = "2026-09-21T02:47:21.843Z" }, - { url = "https://files.pythonhosted.org/packages/6e/d4/0588650ec0fbe26ba55705731e99607ab148149508ab4d67af9c1935b744/fensu_cli-0.22.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:104e1001329d9ba337c97617bb9f0e3fb820ee097a15514be7501223f19317f1", size = 7420715, upload-time = "2026-09-21T02:47:23.665Z" }, - { url = "https://files.pythonhosted.org/packages/17/50/db6c723b0672734d0c67d2339cb74a93eaaca225cdef52089cb0f8b63ea4/fensu_cli-0.22.0-py3-none-win_amd64.whl", hash = "sha256:3ef61f8fc2237dcca8aeb5b0a700c8f6ee1db061c342775893a95edc9ed3631b", size = 7115273, upload-time = "2026-09-21T02:47:25.459Z" }, + { url = "https://files.pythonhosted.org/packages/34/ce/87ca84074f6ba2f278d2739505c6f6d1cde53b8b66085b373666873607a3/fensu_cli-0.23.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:4ccbf9ed3182344ef9e6fefe1d6d62f0770b825d18059bce1954347dc1d3615e", size = 7745194, upload-time = "2026-09-24T15:29:44.933Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b1/21d23c39b2c8dd51e544b6546fada9bd317214a1e4a6f44b3351cc73d4be/fensu_cli-0.23.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8f8d1851c1711359ad2688a3aaa1717af8e48a2c51700a54c647082c3fe51e15", size = 7271582, upload-time = "2026-09-24T15:29:46.978Z" }, + { url = "https://files.pythonhosted.org/packages/aa/56/c190cdedf6e2afa67e58bd6cd6188853983f160e14b73494a08906e84869/fensu_cli-0.23.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d21e87ba99a9c600783c7107e2d06cce15cdce379921a8c996884c16b4c8a407", size = 7362814, upload-time = "2026-09-24T15:29:49.078Z" }, + { url = "https://files.pythonhosted.org/packages/74/13/54147ab05067512127c99df415b73667f878693bf4bd50eb1e03fcb84f76/fensu_cli-0.23.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07ab456f0f8eb90a611e842e3ad1ddf44053b5eb5a5c69d4499f248bd21d01b9", size = 7795295, upload-time = "2026-09-24T15:29:50.828Z" }, + { url = "https://files.pythonhosted.org/packages/70/53/2c0a86d35cb37123dffd5644020cb5284169f83bd0571281e341896b8946/fensu_cli-0.23.0-py3-none-win_amd64.whl", hash = "sha256:8fa9948fd8ee64319a105f70085ddb790b4131bb5ff88636ae6d27ef51b14f57", size = 7490965, upload-time = "2026-09-24T15:29:53.097Z" }, ] [[package]] @@ -4240,7 +4240,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.22.0" }, + { name = "fensu", specifier = ">=0.23.0" }, { name = "ingestr", specifier = ">=1.0.0" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" },