diff --git a/fensu.toml b/fensu.toml index e4b8adc906..80fe3fe0d1 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1536,8 +1536,15 @@ roots = ["crates"] tests = [] tooling = [] rule_packs = ["rust"] -rule_modules = ["scripts.fensu_policy.rules.native_boundary"] +rule_modules = [ + "scripts.fensu_policy.rules.native_boundary", + "scripts.fensu_policy.rules.unused_state", +] select = ["FPRS", "XSB072", "XSB073", "XSB075"] +warn = ["XSB076", "XSB077"] + +[targets.rust.rule_options.XSB076] +schema_paths = ["crates/sqlbuild-orchestration/src/store"] [[targets.rust.rule_exceptions]] rule = "FPRSR307" diff --git a/pyproject.toml b/pyproject.toml index bef1f027d9..74496f9d4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.28.0", + "fensu>=0.28.3", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/scripts/fensu_policy/_helpers/unused_state.py b/scripts/fensu_policy/_helpers/unused_state.py new file mode 100644 index 0000000000..f4fb97c5a6 --- /dev/null +++ b/scripts/fensu_policy/_helpers/unused_state.py @@ -0,0 +1,434 @@ +"""Find control-store state nothing reads and Rust enum variants nothing names.""" + +from __future__ import annotations + +import itertools +import re +from collections.abc import Iterable +from pathlib import Path + +from scripts.fensu_policy.constants import ( + BRACE_CLOSE, + BRACE_OPEN, + BRACKET_CLOSE, + BRACKET_OPEN, + LIST_SEPARATOR, + PARENTHESIS_CLOSE, + PARENTHESIS_OPEN, + REMOVED_CODE, + RUST_ATTRIBUTE_PATTERN, + RUST_BARE_WORD_PATTERN, + RUST_ENUM_PATTERN, + RUST_FROM_ATTRIBUTE_PATTERN, + RUST_IMPL_HEADER_PATTERN, + RUST_LINE_CONTINUATION, + RUST_LITERAL_PATTERN, + RUST_MACRO_INVOCATION_PATTERN, + RUST_PATH_PAIR_PATTERN, + RUST_SELF_PATH_PATTERN, + RUST_SIMPLE_ESCAPES, + RUST_TEST_FILE_NAME, + RUST_TEST_MODULE_PATTERN, + RUST_TEST_PATH_SEGMENTS, + RUST_VARIANT_IMPORT_PATTERN, + SQL_ADD_COLUMN_PATTERN, + SQL_ASSIGNMENT, + SQL_COLUMN_CONSTRAINT_WORDS, + SQL_COMMENT_PATTERN, + SQL_CONFLICT_TARGET_WORDS, + SQL_CREATE_TABLE_PATTERN, + SQL_DDL_PREFIX_PATTERN, + SQL_DELETE_WORD, + SQL_EXCLUDED_QUALIFIER, + SQL_FILE_PATTERN, + SQL_FRAGMENT_MARKERS, + SQL_FROM_WORD, + SQL_NEUTRAL_WORDS, + SQL_QUALIFIER, + SQL_READ_WORDS, + SQL_SET_WORD, + SQL_STAR, + SQL_STAR_PREDECESSORS, + SQL_STATEMENT_SEPARATOR, + SQL_TOKEN_PATTERN, + SQL_VIEW_PREFIX_PATTERN, + SQL_WORD_PATTERN, + SQL_WRITE_WORDS, +) +from scripts.fensu_policy.models import EnumVariant, StateDefinition, StateReads + +_LEXEMES: re.Pattern[str] = re.compile(RUST_LITERAL_PATTERN, re.DOTALL) +_CONTINUATION: re.Pattern[str] = re.compile(RUST_LINE_CONTINUATION) +_TEST_MODULE: re.Pattern[str] = re.compile(RUST_TEST_MODULE_PATTERN) +_ENUM: re.Pattern[str] = re.compile(RUST_ENUM_PATTERN) +_ATTRIBUTE: re.Pattern[str] = re.compile(RUST_ATTRIBUTE_PATTERN) +_FROM_ATTRIBUTE: re.Pattern[str] = re.compile(RUST_FROM_ATTRIBUTE_PATTERN) +_MACRO_INVOCATION: re.Pattern[str] = re.compile(RUST_MACRO_INVOCATION_PATTERN) +_PATH_PAIR: re.Pattern[str] = re.compile(RUST_PATH_PAIR_PATTERN) +_IMPL_HEADER: re.Pattern[str] = re.compile(RUST_IMPL_HEADER_PATTERN) +_SELF_PATH: re.Pattern[str] = re.compile(RUST_SELF_PATH_PATTERN) +_VARIANT_IMPORT: re.Pattern[str] = re.compile(RUST_VARIANT_IMPORT_PATTERN) +_BARE_WORD: re.Pattern[str] = re.compile(RUST_BARE_WORD_PATTERN) +_SQL_COMMENTS: re.Pattern[str] = re.compile(SQL_COMMENT_PATTERN, re.DOTALL) +_SQL_TOKENS: re.Pattern[str] = re.compile(SQL_TOKEN_PATTERN) +_SQL_WORD: re.Pattern[str] = re.compile(SQL_WORD_PATTERN) +_DDL_PREFIX: re.Pattern[str] = re.compile(SQL_DDL_PREFIX_PATTERN, re.IGNORECASE) +_VIEW_PREFIX: re.Pattern[str] = re.compile(SQL_VIEW_PREFIX_PATTERN, re.IGNORECASE) +_CREATE_TABLE: re.Pattern[str] = re.compile(SQL_CREATE_TABLE_PATTERN, re.IGNORECASE) +_ADD_COLUMN: re.Pattern[str] = re.compile(SQL_ADD_COLUMN_PATTERN, re.IGNORECASE) + + +def is_rust_test_path(*, path: str) -> bool: + """Whether a repository-relative Rust path belongs to tests, benches or examples.""" + + parts: list[str] = path.split("/") + return parts[-1] == RUST_TEST_FILE_NAME or any( + part in RUST_TEST_PATH_SEGMENTS for part in parts[:-1] + ) + + +def code_view(*, source: str) -> str: + """Source with comments and literals blanked and `#[cfg(test)]` modules removed.""" + + pieces: list[str] = [] + end: int = 0 + match: re.Match[str] + for match in _LEXEMES.finditer(source): + pieces.append(source[end : match.start()]) + pieces.append(_blank(text=match.group(0), fill=" ")) + end = match.end() + pieces.append(source[end:]) + code: str = "".join(pieces) + module: re.Match[str] | None = _TEST_MODULE.search(code) + while module is not None: + closing: int = _closing(text=code, opening=module.end() - 1, pair=(BRACE_OPEN, BRACE_CLOSE)) + removed: str = _blank(text=code[module.start() : closing + 1], fill=REMOVED_CODE) + code = code[: module.start()] + removed + code[closing + 1 :] + module = _TEST_MODULE.search(code, module.start() + len(removed)) + return code + + +def string_literals(*, source: str) -> list[str]: + """Values of the string literals in production code, skipping test modules.""" + + code: str = code_view(source=source) + literals: list[str] = [] + match: re.Match[str] + for match in _LEXEMES.finditer(source): + if code[match.start()] == REMOVED_CODE: + continue + if match.group("raw") is not None: + literals.append(match.group("raw_body")) + elif match.group("plain") is not None: + literals.append(_unescape(text=match.group("plain_body"))) + return literals + + +def schema_files(*, root: Path, directories: tuple[str, ...]) -> dict[str, str]: + """Control-store SQL files under the configured directories, keyed by relative path.""" + + files: dict[str, str] = {} + directory: str + for directory in directories: + path: Path + for path in sorted((root / directory).glob(SQL_FILE_PATTERN)): + files[path.relative_to(root).as_posix()] = path.read_text(encoding="utf-8") + return files + + +def schema_definitions(*, path: str, sql: str) -> list[StateDefinition]: + """Tables and columns that one control-store SQL file creates, in source order.""" + + text: str = _blank_sql_comments(sql=sql) + definitions: list[StateDefinition] = [] + table: re.Match[str] + for table in _CREATE_TABLE.finditer(text): + name: str = table.group(1).lower() + definitions.append( + StateDefinition(path=path, line=_line(text=text, offset=table.start()), table=name) + ) + closing: int = _closing( + text=text, opening=table.end() - 1, pair=(PARENTHESIS_OPEN, PARENTHESIS_CLOSE) + ) + offset: int + piece: str + for offset, piece in _top_level_pieces(text=text, start=table.end(), end=closing): + column: str | None = _column_name(piece=piece) + if column is not None: + definitions.append( + StateDefinition( + path=path, line=_line(text=text, offset=offset), table=name, column=column + ) + ) + added: re.Match[str] + for added in _ADD_COLUMN.finditer(text): + definitions.append( + StateDefinition( + path=path, + line=_line(text=text, offset=added.start()), + table=added.group(1).lower(), + column=added.group(2).lower(), + ) + ) + return sorted(definitions, key=lambda definition: definition.line) + + +def sql_read_fragments(*, sql: str) -> list[str]: + """Statements of a SQL file that can read state: everything except plain DDL.""" + + return [ + statement + for statement in _blank_sql_comments(sql=sql).split(SQL_STATEMENT_SEPARATOR) + if statement.strip() and not _plain_ddl(text=statement) + ] + + +def collect_reads(*, fragments: Iterable[str], definitions: list[StateDefinition]) -> StateReads: + """Tables and columns some fragment reads, attributing columns to mentioned tables.""" + + columns_by_table: dict[str, set[str]] = {} + definition: StateDefinition + for definition in definitions: + columns: set[str] = columns_by_table.setdefault(definition.table, set()) + if definition.column is not None: + columns.add(definition.column) + reads: StateReads = StateReads(tables=set(), columns=set()) + fragment: str + for fragment in fragments: + if not _plain_ddl(text=fragment): + reads = _read_fragment( + fragment=fragment, columns_by_table=columns_by_table, reads=reads + ) + return reads + + +def unread_definitions( + *, definitions: list[StateDefinition], reads: StateReads +) -> list[StateDefinition]: + """Unread tables, and unread columns of tables that are themselves read.""" + + return [ + definition + for definition in definitions + if (definition.column is None and definition.table not in reads.tables) + or ( + definition.column is not None + and definition.table in reads.tables + and (definition.table, definition.column) not in reads.columns + ) + ] + + +def enum_variants(*, path: str, code: str) -> list[EnumVariant]: + """Variants of every enum declared in one file's production code view.""" + + variants: list[EnumVariant] = [] + generated: list[tuple[int, int]] = [ + ( + invocation.start(), + _closing(text=code, opening=invocation.end() - 1, pair=(BRACE_OPEN, BRACE_CLOSE)), + ) + for invocation in _MACRO_INVOCATION.finditer(code) + ] + declaration: re.Match[str] + for declaration in _ENUM.finditer(code): + if any(start <= declaration.start() <= end for start, end in generated): + continue + opening: int = declaration.end() - 1 + closing: int = _closing(text=code, opening=opening, pair=(BRACE_OPEN, BRACE_CLOSE)) + offset: int + piece: str + for offset, piece in _variant_pieces(code=code, start=opening + 1, end=closing): + name: re.Match[str] | None = _SQL_WORD.match(piece) + if ( + name is not None + and _FROM_ATTRIBUTE.search(code, offset, offset + len(piece)) is None + ): + variants.append( + EnumVariant( + path=path, + line=_line(text=code, offset=offset), + enum=declaration.group("name"), + variant=name.group(0), + body=(opening, closing), + ) + ) + return variants + + +def named_variants(*, variants: list[EnumVariant], codes: dict[str, str]) -> set[EnumVariant]: + """Variants named by a path, by `Self` in an impl's file, or by an imported variant.""" + + bodies: dict[str, list[tuple[int, int]]] = {} + variant: EnumVariant + for variant in variants: + bodies.setdefault(variant.path, []).append(variant.body) + names: set[tuple[str, str]] = set() + path: str + code: str + for path, code in codes.items(): + names.update(_file_names(code=_without_spans(text=code, spans=bodies.get(path, [])))) + return {variant for variant in variants if (variant.enum, variant.variant) in names} + + +def _file_names(*, code: str) -> set[tuple[str, str]]: + names: set[tuple[str, str]] = { + (match.group(1), match.group(2)) for match in _PATH_PAIR.finditer(code) + } + implemented: set[str] = set() + header: re.Match[str] + for header in _IMPL_HEADER.finditer(code): + implemented.update(_SQL_WORD.findall(header.group(0))) + self_names: set[str] = {match.group(1) for match in _SELF_PATH.finditer(code)} + imported: set[str] = {match.group(1) for match in _VARIANT_IMPORT.finditer(code)} + bare: set[str] = set(_BARE_WORD.findall(code)) if imported else set() + enum: str + for enum in implemented: + names.update(itertools.product((enum,), self_names)) + for enum in imported: + names.update(itertools.product((enum,), bare)) + return names + + +def _without_spans(*, text: str, spans: list[tuple[int, int]]) -> str: + start: int + end: int + for start, end in spans: + text = text[:start] + _blank(text=text[start:end], fill=" ") + text[end:] + return text + + +def _read_fragment( + *, fragment: str, columns_by_table: dict[str, set[str]], reads: StateReads +) -> StateReads: + tokens: list[str] = [token.strip('"').lower() for token in _SQL_TOKENS.findall(fragment)] + known_columns: set[str] = set().union(*columns_by_table.values()) + mentioned: list[str] = [token for token in tokens if token in columns_by_table] + keywords: bool = any(token in SQL_READ_WORDS | SQL_WRITE_WORDS for token in tokens) + words: set[str] = { + token for token in tokens if _SQL_WORD.fullmatch(token) and token not in SQL_NEUTRAL_WORDS + } + shaped: bool = any(token in SQL_FRAGMENT_MARKERS for token in tokens) + if not keywords and not mentioned and not (shaped and words and words <= known_columns): + return reads + read_words: set[str] = set() + star: bool = False + reading: bool = not keywords + deleting: bool = False + set_depth: int | None = None + depth: int = 0 + position: int + token: str + for position, token in enumerate(tokens): + before: str = tokens[position - 1] if position else "" + qualifier: str = tokens[position - 2] if position > 1 and before == SQL_QUALIFIER else "" + depth += (token == PARENTHESIS_OPEN) - (token == PARENTHESIS_CLOSE) + deleting = deleting or token == SQL_DELETE_WORD + if token in SQL_WRITE_WORDS or (before, token) in SQL_CONFLICT_TARGET_WORDS: + reading = False + set_depth = depth if token == SQL_SET_WORD else None + elif token == SQL_FROM_WORD and deleting: + deleting = False + elif token in SQL_READ_WORDS: + reading = True + set_depth = None + elif set_depth == depth: + reading = token == SQL_ASSIGNMENT or (reading and token != LIST_SEPARATOR) + if not reading or token in SQL_READ_WORDS or qualifier == SQL_EXCLUDED_QUALIFIER: + continue + star = star or (token == SQL_STAR and before in SQL_STAR_PREDECESSORS) + read_words.add(token) + if token in columns_by_table: + reads.tables.add(token) + owner: str + for owner in dict.fromkeys(mentioned or list(columns_by_table)): + owned: set[str] = columns_by_table[owner] + credited: set[str] = owned if star and mentioned else owned & read_words + reads.columns.update((owner, column) for column in credited) + return reads + + +def _plain_ddl(*, text: str) -> bool: + return _DDL_PREFIX.match(text) is not None and _VIEW_PREFIX.match(text) is None + + +def _blank(*, text: str, fill: str) -> str: + return re.sub(r"[^\n]", fill, text) + + +def _blank_sql_comments(*, sql: str) -> str: + return _SQL_COMMENTS.sub(lambda comment: _blank(text=comment.group(0), fill=" "), sql) + + +def _unescape(*, text: str) -> str: + return re.sub( + r"\\(.)", + lambda escaped: RUST_SIMPLE_ESCAPES.get(escaped.group(1), ""), + _CONTINUATION.sub("", text), + ) + + +def _line(*, text: str, offset: int) -> int: + return text.count("\n", 0, offset) + 1 + + +def _closing(*, text: str, opening: int, pair: tuple[str, str]) -> int: + depth: int = 0 + offset: int + for offset in range(opening, len(text)): + depth += (text[offset] == pair[0]) - (text[offset] == pair[1]) + if depth == 0: + return offset + return len(text) + + +def _top_level_pieces(*, text: str, start: int, end: int) -> list[tuple[int, str]]: + pieces: list[tuple[int, str]] = [] + depth: int = 0 + piece_start: int = start + offset: int + for offset in range(start, end): + character: str = text[offset] + depth += (character in (PARENTHESIS_OPEN, BRACKET_OPEN, BRACE_OPEN)) - ( + character in (PARENTHESIS_CLOSE, BRACKET_CLOSE, BRACE_CLOSE) + ) + if character == LIST_SEPARATOR and depth == 0: + pieces.append((piece_start, text[piece_start:offset])) + piece_start = offset + 1 + pieces.append((piece_start, text[piece_start:end])) + stripped: list[tuple[int, str]] = [] + piece: str + for piece_start, piece in pieces: + if piece.strip(): + stripped.append((piece_start + len(piece) - len(piece.lstrip()), piece.strip())) + return stripped + + +def _variant_pieces(*, code: str, start: int, end: int) -> list[tuple[int, str]]: + body: str = _without_attributes(text=code[start:end]) + return [ + (start + offset, piece) + for offset, piece in _top_level_pieces(text=body, start=0, end=len(body)) + ] + + +def _without_attributes(*, text: str) -> str: + match: re.Match[str] | None = _ATTRIBUTE.search(text) + while match is not None: + closing: int = _closing( + text=text, opening=match.end() - 1, pair=(BRACKET_OPEN, BRACKET_CLOSE) + ) + text = ( + text[: match.start()] + + _blank(text=text[match.start() : closing + 1], fill=" ") + + text[closing + 1 :] + ) + match = _ATTRIBUTE.search(text, match.start() + 1) + return text + + +def _column_name(*, piece: str) -> str | None: + first: str = piece.split(None, 1)[0].strip('"').lower() + if first in SQL_COLUMN_CONSTRAINT_WORDS or _SQL_WORD.fullmatch(first) is None: + return None + return first diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index c2b4068e93..67e54a61b5 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -364,3 +364,83 @@ default=(), description="`path=count` budgets for test files that still patch the native module.", ) +BRACE_CLOSE: str = "}" +BRACE_OPEN: str = "{" +BRACKET_CLOSE: str = "]" +BRACKET_OPEN: str = "[" +CONTROL_STORE_SCHEMA_PATHS: RuleOption[tuple[str, ...]] = RuleOption.string_list( + name="schema_paths", + default=(), + description="Repository-relative directories whose *.sql files create control-store state.", +) +LIST_SEPARATOR: str = "," +PARENTHESIS_CLOSE: str = ")" +PARENTHESIS_OPEN: str = "(" +REMOVED_CODE: str = "\0" +RUST_ATTRIBUTE_PATTERN: str = r"#\s*\[" +RUST_BARE_WORD_PATTERN: str = r"(?[A-Za-z_]\w*)\s*(?:<[^{;]*>)?\s*(?:where\b[^{;]*)?\{" +RUST_FROM_ATTRIBUTE_PATTERN: str = r"#\s*\[\s*from\s*\]" +RUST_IMPL_HEADER_PATTERN: str = r"\bimpl\b[^{;]*\{" +RUST_LINE_CONTINUATION: str = r"\\\n\s*" +RUST_LITERAL_PATTERN: str = ( + r"//[^\n]*" + r"|/\*.*?\*/" + r"|(?Pb?r(?P#*)\"(?P.*?)\"(?P=hashes))" + r"|(?Pb?\"(?P(?:\\.|[^\"\\])*)\")" + r"|'(?:\\u\{[0-9a-fA-F]+\}|\\.|[^\\'\n])'" +) +RUST_MACRO_INVOCATION_PATTERN: str = r"\b[A-Za-z_]\w*!\s*\{" +RUST_PATH_PAIR_PATTERN: str = r"(?=\b([A-Za-z_]\w*)\s*::\s*([A-Za-z_]\w*)\b)" +RUST_SELF_PATH_PATTERN: str = r"\bSelf\s*::\s*([A-Za-z_]\w*)\b" +RUST_SIMPLE_ESCAPES: dict[str, str] = { + "n": "\n", + "t": "\t", + "r": "\r", + "0": "\0", + "\\": "\\", + '"': '"', + "'": "'", +} +RUST_TEST_FILE_NAME: str = "tests.rs" +RUST_TEST_MODULE_PATTERN: str = ( + r"#\s*\[\s*cfg\s*\(\s*test\s*\)\s*\]\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*\{" +) +RUST_TEST_PATH_SEGMENTS: frozenset[str] = frozenset({"tests", "benches", "examples"}) +RUST_VARIANT_IMPORT_PATTERN: str = r"\b([A-Za-z_]\w*)\s*::\s*(?:\*|\{)" +SQL_ADD_COLUMN_PATTERN: str = ( + r"\bALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?\"?(\w+)\"?\s+ADD\s+(?:COLUMN\s+)?" + r"(?:IF\s+NOT\s+EXISTS\s+)?\"?(\w+)\"?" +) +SQL_ASSIGNMENT: str = "=" +SQL_COLUMN_CONSTRAINT_WORDS: frozenset[str] = frozenset( + {"constraint", "primary", "unique", "foreign", "check", "exclude", "like"} +) +SQL_COMMENT_PATTERN: str = r"--[^\n]*|/\*.*?\*/" +SQL_CONFLICT_TARGET_WORDS: frozenset[tuple[str, str]] = frozenset({("conflict", "on")}) +SQL_CREATE_TABLE_PATTERN: str = ( + r"\bCREATE\s+(?:UNLOGGED\s+)?TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?\"?(\w+)\"?\s*\(" +) +SQL_DDL_PREFIX_PATTERN: str = r"\s*(?:CREATE|ALTER|DROP|COMMENT|GRANT|REVOKE)\b" +SQL_DELETE_WORD: str = "delete" +SQL_EXCLUDED_QUALIFIER: str = "excluded" +SQL_FILE_PATTERN: str = "*.sql" +SQL_FRAGMENT_MARKERS: frozenset[str] = frozenset({"=", ",", "is", "in"}) +SQL_FROM_WORD: str = "from" +SQL_NEUTRAL_WORDS: frozenset[str] = frozenset( + {"and", "or", "not", "is", "null", "in", "as", "asc", "desc", "true", "false"} +) +SQL_QUALIFIER: str = "." +SQL_READ_WORDS: frozenset[str] = frozenset( + {"select", "from", "join", "on", "where", "group", "order", "having", "returning", "using"} +) +SQL_SET_WORD: str = "set" +SQL_STAR: str = "*" +SQL_STAR_PREDECESSORS: frozenset[str] = frozenset({"select", ",", "."}) +SQL_STATEMENT_SEPARATOR: str = ";" +SQL_TOKEN_PATTERN: str = r"\"[^\"]*\"|'(?:[^']|'')*'|[A-Za-z_]\w*|[(),.*=]" +SQL_VIEW_PREFIX_PATTERN: str = r"\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:MATERIALIZED\s+)?VIEW\b" +SQL_WORD_PATTERN: str = r"[A-Za-z_]\w*" +SQL_WRITE_WORDS: frozenset[str] = frozenset( + {"insert", "into", "values", "update", "set", "delete", "conflict", "do"} +) diff --git a/scripts/fensu_policy/models.py b/scripts/fensu_policy/models.py index 3f66ea27fd..86d3ef49dc 100644 --- a/scripts/fensu_policy/models.py +++ b/scripts/fensu_policy/models.py @@ -46,3 +46,32 @@ def points_upward(self, *, source: str, target: str) -> bool: if source == target: return False return self.ranks[target] >= self.ranks[source] + + +@dataclass(frozen=True) +class StateDefinition: + """One control-store table, or one of its columns when `column` is set.""" + + path: str + line: int + table: str + column: str | None = None + + +@dataclass(frozen=True) +class StateReads: + """Tables and `(table, column)` pairs that production SQL reads.""" + + tables: set[str] + columns: set[tuple[str, str]] + + +@dataclass(frozen=True) +class EnumVariant: + """One Rust enum variant and the offsets of its enum's body in the declaring file.""" + + path: str + line: int + enum: str + variant: str + body: tuple[int, int] diff --git a/scripts/fensu_policy/rules/unused_state.py b/scripts/fensu_policy/rules/unused_state.py new file mode 100644 index 0000000000..af09c0f9e9 --- /dev/null +++ b/scripts/fensu_policy/rules/unused_state.py @@ -0,0 +1,103 @@ +"""Rules flagging control-store state and Rust enum variants that production code never uses.""" + +from __future__ import annotations + +from fensu import AnalyzerId, Family, Fault, Project, RuleContext, rule + +from scripts.fensu_policy._helpers.unused_state import ( + code_view, + collect_reads, + enum_variants, + is_rust_test_path, + named_variants, + schema_definitions, + schema_files, + sql_read_fragments, + string_literals, + unread_definitions, +) +from scripts.fensu_policy.constants import CONTROL_STORE_SCHEMA_PATHS +from scripts.fensu_policy.models import EnumVariant, StateDefinition, StateReads + + +@rule( + code="XSB076", + family=Family.CUSTOM, + slug="control-store-unread-state", + message="control-store state is written but never read by production code", + remediation=( + "Delete the table or column, or add it together with its first production reader; " + "writes, indexes and tests do not count as reads." + ), + cacheable=False, + analyzers=(AnalyzerId.RUST,), + options=(CONTROL_STORE_SCHEMA_PATHS,), +) +def control_store_unread_state(*, project: Project, ctx: RuleContext) -> list[Fault]: + del project + schema: dict[str, str] = schema_files( + root=ctx.repo_root, directories=ctx.option(CONTROL_STORE_SCHEMA_PATHS) + ) + definitions: list[StateDefinition] = [] + fragments: list[str] = [] + path: str + sql: str + for path, sql in schema.items(): + definitions.extend(schema_definitions(path=path, sql=sql)) + fragments.extend(sql_read_fragments(sql=sql)) + if not definitions: + return [] + for facts in ctx.rust.files: + if not facts.test and not is_rust_test_path(path=facts.file.path.value): + fragments.extend(string_literals(source=facts.source)) + reads: StateReads = collect_reads(fragments=fragments, definitions=definitions) + return [ + ctx.fault_for( + path=ctx.repo_root / definition.path, + line=definition.line, + column=0, + message=( + f"table {definition.table} is never read by production code" + if definition.column is None + else f"column {definition.table}.{definition.column} is never read by production " + "code" + ), + ) + for definition in unread_definitions(definitions=definitions, reads=reads) + ] + + +@rule( + code="XSB077", + family=Family.CUSTOM, + slug="unconstructed-enum-variant", + message="enum variant is never constructed or matched by production code", + remediation=( + "Delete the variant, or add it together with the production code that produces it; " + "tests and serde input alone do not keep a variant alive." + ), + analyzers=(AnalyzerId.RUST,), +) +def unconstructed_enum_variant(*, project: Project, ctx: RuleContext) -> list[Fault]: + del project + codes: dict[str, str] = { + facts.file.path.value: code_view(source=facts.source) + for facts in ctx.rust.files + if not facts.test and not is_rust_test_path(path=facts.file.path.value) + } + variants: list[EnumVariant] = [] + path: str + code: str + for path, code in codes.items(): + variants.extend(enum_variants(path=path, code=code)) + named: set[EnumVariant] = named_variants(variants=variants, codes=codes) + return [ + ctx.fault_for( + path=ctx.repo_root / variant.path, + line=variant.line, + column=0, + message=f"variant {variant.enum}::{variant.variant} is never constructed or matched", + ) + for variant in variants + if variant not in named + ] diff --git a/tests/unit/scripts/fensu_policy/rules/_test_types.py b/tests/unit/scripts/fensu_policy/rules/_test_types.py index 318a8b279e..ebfe7938f9 100644 --- a/tests/unit/scripts/fensu_policy/rules/_test_types.py +++ b/tests/unit/scripts/fensu_policy/rules/_test_types.py @@ -57,3 +57,18 @@ class NativeLayeringTestCase: path: str source: str expected_lines: list[int] | None + + +@dataclass(frozen=True) +class UnreadStateTestCase: + description: str + schema: str + rust: str + expected_unread: tuple[tuple[str, str | None], ...] + + +@dataclass(frozen=True) +class EnumVariantTestCase: + description: str + files: tuple[tuple[str, str], ...] + expected_unnamed: tuple[tuple[str, str], ...] diff --git a/tests/unit/scripts/fensu_policy/rules/test_unused_state.py b/tests/unit/scripts/fensu_policy/rules/test_unused_state.py new file mode 100644 index 0000000000..4320209d15 --- /dev/null +++ b/tests/unit/scripts/fensu_policy/rules/test_unused_state.py @@ -0,0 +1,220 @@ +import itertools +from textwrap import dedent + +import pytest + +from scripts.fensu_policy._helpers.unused_state import ( + code_view, + collect_reads, + enum_variants, + named_variants, + schema_definitions, + sql_read_fragments, + string_literals, + unread_definitions, +) +from scripts.fensu_policy.models import EnumVariant, StateDefinition +from tests.unit.scripts.fensu_policy.rules._test_types import ( + EnumVariantTestCase, + UnreadStateTestCase, +) + +ORDERS_SCHEMA: str = dedent( + """ + -- Orders and their fulfillment attempts. + CREATE TABLE orders ( + order_id BIGINT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('open', 'shipped')), + retries BIGINT NOT NULL DEFAULT 0, + note_json TEXT NOT NULL, + PRIMARY KEY (order_id) + ); + CREATE INDEX orders_open ON orders (status) WHERE retries > 0; + ALTER TABLE orders ADD COLUMN shipped_at TIMESTAMPTZ; + CREATE TABLE order_events (event_id BIGINT NOT NULL, order_id BIGINT NOT NULL); + """ +) + + +@pytest.mark.parametrize( + "test_case", + [ + UnreadStateTestCase( + description="written columns, indexes, DELETE targets and comments are not reads", + schema=ORDERS_SCHEMA, + rust=dedent( + """ + fn load() { query("SELECT order_id, status FROM orders WHERE shipped_at IS NULL"); } + fn save() { query("UPDATE orders SET retries = 0, note_json = $1 WHERE order_id = $2"); } + fn prune() { query("DELETE FROM order_events WHERE event_id < $1"); } + // query("SELECT retries FROM orders") + """ + ), + expected_unread=( + ("orders", "retries"), + ("orders", "note_json"), + ("order_events", None), + ), + ), + UnreadStateTestCase( + description="reads inside a test module do not count", + schema=ORDERS_SCHEMA, + rust=dedent( + """ + fn load() { query("SELECT * FROM orders"); } + #[cfg(test)] + mod tests { + fn check() { query("SELECT event_id, order_id FROM order_events"); } + } + """ + ), + expected_unread=(("order_events", None),), + ), + UnreadStateTestCase( + description="an assignment's right side reads but an upsert's excluded value does not", + schema=ORDERS_SCHEMA, + rust=dedent( + """ + fn bump() { + query("UPDATE orders SET retries = retries + 1 WHERE order_id = $1 RETURNING status"); + query("INSERT INTO orders (order_id, note_json) VALUES ($1, $2) \\ + ON CONFLICT (order_id) DO UPDATE SET note_json = excluded.note_json"); + query("SELECT o.order_id FROM orders o JOIN order_events e ON e.order_id = o.order_id"); + } + """ + ), + expected_unread=( + ("orders", "note_json"), + ("orders", "shipped_at"), + ("order_events", "event_id"), + ), + ), + UnreadStateTestCase( + description="a predicate assembled from fragments reads its columns", + schema=ORDERS_SCHEMA, + rust=dedent( + """ + fn filter(shipped: bool) { + let predicate = if shipped { "shipped_at IS NOT NULL" } else { "retries = $1" }; + query(&format!("SELECT order_id, status, note_json FROM orders WHERE {predicate}")); + query("SELECT event_id, order_id FROM order_events"); + let key = "status"; + } + """ + ), + expected_unread=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_control_store_sql_when_collecting_reads_then_reports_unread_state( + test_case: UnreadStateTestCase, +) -> None: + definitions: list[StateDefinition] = schema_definitions( + path="store/schema.sql", sql=test_case.schema + ) + fragments: list[str] = [ + *sql_read_fragments(sql=test_case.schema), + *string_literals(source=test_case.rust), + ] + unread: list[StateDefinition] = unread_definitions( + definitions=definitions, + reads=collect_reads(fragments=fragments, definitions=definitions), + ) + + assert tuple((definition.table, definition.column) for definition in unread) == ( + test_case.expected_unread + ) + + +@pytest.mark.parametrize( + "test_case", + [ + EnumVariantTestCase( + description="paths, Self in an impl and glob imports name variants; text does not", + files=( + ( + "crates/shop/src/models.rs", + dedent( + """ + /// Mentions Outcome::Cancelled only in a comment. + #[derive(Deserialize)] + #[serde(rename_all = "snake_case")] + pub enum Outcome { + /// Waiting for stock. + Pending, + #[serde(rename = "a,b")] + Shipped { carrier: String, parcels: (u8, u8) }, + Returned(Vec<(u8, u8)>), + Cancelled = 4, + } + + impl Outcome { + fn label(self) -> &'static str { + match self { Self::Shipped { .. } => "shipped", _ => "Outcome::Cancelled" } + } + } + + fn start() -> Outcome { Outcome::Pending } + """ + ), + ), + ( + "crates/shop/src/returns.rs", + dedent( + """ + use crate::models::Outcome::*; + + fn returned(outcome: &Outcome) -> bool { matches!(outcome, Returned(_)) } + """ + ), + ), + ), + expected_unnamed=(("Outcome", "Cancelled"),), + ), + EnumVariantTestCase( + description="test-only use, macro-declared enums and #[from] variants", + files=( + ( + "crates/shop/src/errors.rs", + dedent( + """ + pub enum StoreError { + Io(#[from] std::io::Error), + Closed, + Locked, + } + + fn fail() -> StoreError { StoreError::Locked } + + ids! { + pub enum Generated { Never => "never" } + } + + #[cfg(test)] + mod tests { + fn closed() -> super::StoreError { super::StoreError::Closed } + } + """ + ), + ), + ), + expected_unnamed=(("StoreError", "Closed"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_rust_enums_when_finding_named_variants_then_reports_unnamed_variants( + test_case: EnumVariantTestCase, +) -> None: + codes: dict[str, str] = {path: code_view(source=source) for path, source in test_case.files} + variants: list[EnumVariant] = list( + itertools.chain.from_iterable( + enum_variants(path=path, code=code) for path, code in codes.items() + ) + ) + unnamed: set[EnumVariant] = set(variants) - named_variants(variants=variants, codes=codes) + + assert {(variant.enum, variant.variant) for variant in unnamed} == set( + test_case.expected_unnamed + ) diff --git a/uv.lock b/uv.lock index eda041b554..95b2e27b10 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.28.0" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/69/a132ffc407be97a0fdcfa052cb8b49408091db748e6afe0f7a6c77e6fa7d/fensu-0.28.0.tar.gz", hash = "sha256:11bcea3207f7fd92babc200f40d074e3da789ee11e7fc7024575a415de7c1176", size = 538990, upload-time = "2026-10-27T00:39:16.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/3d/d5b4fb0ad09426dae015af2817ba8416d2ea974a0c4a6ebea63112f90b1b/fensu-0.28.3.tar.gz", hash = "sha256:a7fc87cb60eaef964b4e980d19bd6de9c95cecc75c019d8b0058e02b6751f4b5", size = 539031, upload-time = "2026-11-01T09:41:23.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/e9/7434bd0565d54ff58796105728b3ccc8b838c7200997184978e6ebfdec4d/fensu-0.28.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:29df31aff900d20b0c845940af0cd91eead55e158a2511e682e3e887a22dada8", size = 4154207, upload-time = "2026-10-27T00:38:52.311Z" }, - { url = "https://files.pythonhosted.org/packages/7f/d8/e0875b269515d9a75643d5488bdc6f70a4b6c346995f12ce07304a3610f7/fensu-0.28.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:33a6d8b7d2f734c48f48087462c0e5a1ff94535c0e27e6a4820bc10303ff3653", size = 4004243, upload-time = "2026-10-27T00:38:54.994Z" }, - { url = "https://files.pythonhosted.org/packages/e8/05/b87c0ca5cf3f724517d6958f1ebd68948f9399306bd6d6814783635bad9a/fensu-0.28.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de07b3fee5d908d2a766ca136753c8040d6fe25a1346d9b3b5a98483fc23d874", size = 4230192, upload-time = "2026-10-27T00:38:57.321Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d2/fdfc7fd322c3497c53c22754a33433e33678a6de84832fd41add09481a21/fensu-0.28.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f44293a2da6bf4558d00eb4f859b28f3fb69d396dad1c25ac466a92a7d1dd734", size = 4307267, upload-time = "2026-10-27T00:38:59.679Z" }, - { url = "https://files.pythonhosted.org/packages/56/c0/2ab2c2c1af4ddd1b45c750679bd9f0b777d7eca3c0e00a233c991aaa13e7/fensu-0.28.0-cp312-abi3-win_amd64.whl", hash = "sha256:ffa88f1d259a1830d513ae930ff856d5b5e4941ba433889dea04d3db5753f281", size = 3977828, upload-time = "2026-10-27T00:39:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a7/9493fbcf3158ae009651d19c7289855e8ef571dfaf5c7c77154a8a6a4a21/fensu-0.28.3-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e3896f173d2bb676fe04defdd4ebdf4db38aee3ee6e57d7a5a297df32d7c5c", size = 4154274, upload-time = "2026-11-01T09:41:03.614Z" }, + { url = "https://files.pythonhosted.org/packages/a2/71/20fdb7147e268c5dbd9eae0fc9dd99641e416f9e75e8fd067ac089f6db78/fensu-0.28.3-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d74db7466b2ddd533413d2daf4c704aa4acbcae61e2a0edba2564a4910666707", size = 4004309, upload-time = "2026-11-01T09:41:05.719Z" }, + { url = "https://files.pythonhosted.org/packages/d0/4c/40b01d8e46ff3464eb503529e00ad23eddd2ffbd605282618672d0aff45f/fensu-0.28.3-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da5843aa40c7d6398a95b340b3c60d4360b5feddac34960663ee84aec09df8ef", size = 4230261, upload-time = "2026-11-01T09:41:07.481Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7a/243cccd83531c3c86c660b993e77a7014c65d1d3f20c07e45b067dba81cf/fensu-0.28.3-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52e1623e614f101c5ada2eda59548ff441b7efe4228813e69bc0f0ef9b3fbca6", size = 4307334, upload-time = "2026-11-01T09:41:09.938Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/38dbb50d5079a1a5fcaf62bcdd582c12a53609c575a747cb2461e83ddfcf/fensu-0.28.3-cp312-abi3-win_amd64.whl", hash = "sha256:a39c407725ea4d92123414bdbe9f80d5db0b7a412af49082a87378203c9ecb04", size = 3977904, upload-time = "2026-11-01T09:41:11.672Z" }, ] [[package]] name = "fensu-cli" -version = "0.28.0" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/d9/f347f514fb224f6a1f4ca6e616ac49b573f9dceba4ff8d9efcd334f48bea/fensu_cli-0.28.0.tar.gz", hash = "sha256:1feb379ed92e193cbd4d28ab3d7729798b88120aaee2f59fd7f341519b6ee945", size = 765638, upload-time = "2026-10-27T00:39:17.752Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/17/3d0953c33387caa0a6becc9bb02210d4a020eaaf73ef07bdc0c8f8952562/fensu_cli-0.28.3.tar.gz", hash = "sha256:3ba38bedc88a9e6db8563f638aa6ba8649a3e5750de176e22a7b35561d14bd9e", size = 766571, upload-time = "2026-11-01T09:41:25.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/b3/57f1de89e1cb7534b0ef6415822c8982a247b9d2efe8d2fddd85e35389c8/fensu_cli-0.28.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13ecc7a224a4b32eef19456f21659e682e636ef53f305919aa57cae27c8e3fec", size = 7978247, upload-time = "2026-10-27T00:39:04.644Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ff/2e887d727f82c61ea7cb6773429b61d4711b93f7f94814eb9a7e891384dc/fensu_cli-0.28.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:149e5e22c8895c0a3ed3956b30e693e78df7dfb67ce8e0c5682b84c77e811e7e", size = 7490546, upload-time = "2026-10-27T00:39:07.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/4e/daa4c2d70e5b0398382f84c846aacc4a937c4d00b3938d00a67351b0f212/fensu_cli-0.28.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:99932d17956bb10da1ac677867567b504a961d0565fcda730c1e682434244113", size = 7570220, upload-time = "2026-10-27T00:39:09.852Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f9/c9e1095c507da3a53b3e7bf3f7c95f75824429ea45a72cb2c7af67a605f7/fensu_cli-0.28.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee51c5d6d0d5fa0b36c0f70d3d6fde429793b643f0075cdfdf3fc8b8d73e3ccb", size = 8086364, upload-time = "2026-10-27T00:39:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/9c/89/ece32e452d2b4ade243dd66a2612619be7400f6acf59f854ce674bb78e30/fensu_cli-0.28.0-py3-none-win_amd64.whl", hash = "sha256:acca6ac39cbe0614d3a8e4ce7c2c12cc5fd3c800fc4c66b5484c54dc19ef4a66", size = 7700477, upload-time = "2026-10-27T00:39:14.15Z" }, + { url = "https://files.pythonhosted.org/packages/ff/bb/09753cc0584c21af1d76eca7c9a6ea5e2d876d71c079c5505dda93742f3d/fensu_cli-0.28.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5179b032074f4ff5097950c0c0b063fd872cffcd23a4bf8a89c71ab0d3d8074b", size = 7963143, upload-time = "2026-11-01T09:41:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/47/59/663fb61d56172bfb053375e8623a23db389b746052fb00dd087cf6c1a76b/fensu_cli-0.28.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:41077b9ec6e2eb46405af809d1010b26519c8928d4264137ca4c6d47ce81d2e1", size = 7479791, upload-time = "2026-11-01T09:41:16.253Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7c/1a03bd25bc2a231ca03478ba3dfd8f21c909eae564c3df751a8982222beb/fensu_cli-0.28.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ac929d598046846c18eae62c8b75a9a49e9b6bbcadb46bd413a5da96ef6453e2", size = 7854377, upload-time = "2026-11-01T09:41:18.109Z" }, + { url = "https://files.pythonhosted.org/packages/17/5e/8395e4bd4e9e4d282a0949e42e721101127fb30f45cb71d6e5af05329051/fensu_cli-0.28.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ce453889008381184a416acaf274ffa4a46c7cb96ad4397de264816bb10626a", size = 8316670, upload-time = "2026-11-01T09:41:20.045Z" }, + { url = "https://files.pythonhosted.org/packages/89/40/6e3831cd39b16f163916b36c03010754c1cfb58583fd737517bd544751d5/fensu_cli-0.28.3-py3-none-win_amd64.whl", hash = "sha256:66ec08187a6dcc43392244ba4b7349ee29e8da63fbe7110114468f07a000d577", size = 7710605, upload-time = "2026-11-01T09:41:22.073Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.28.0" }, + { name = "fensu", specifier = ">=0.28.3" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" },