diff --git a/docs/B.3.2-mcp-tool-poisoning.md b/docs/B.3.2-mcp-tool-poisoning.md index 51eac0a1e..1a1443ae6 100644 --- a/docs/B.3.2-mcp-tool-poisoning.md +++ b/docs/B.3.2-mcp-tool-poisoning.md @@ -209,7 +209,7 @@ be safe, static, representative examples. | Field | Value | |-------------|-----------------------------------------------------------| -| **Severity** | MEDIUM or HIGH (based on LLM confidence) | +| **Severity** | MEDIUM | | **Confidence** | LLM-determined (0.50 -- 1.00, threshold: 0.50) | | **Tags** | ASI02, AML.T0080 | diff --git a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py index fd0d4f41d..b74d67f76 100644 --- a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py +++ b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py @@ -1063,7 +1063,9 @@ class _TP4CheckOutcome: Flag a mismatch when code performs an undeclared capability, has a materially different primary purpose, accesses inconsistent resources, or has unrelated triggers. Do not flag supporting implementation details or over-declared -permissions. Return the assessment using the structured output schema. +permissions. A chunk may implement only one declared workflow step; do not +flag omitted capabilities. Each entry must name what the code DOES. Return +the assessment using the structured output schema. """ @@ -1147,6 +1149,382 @@ def _tp4_partial_event( ) +# Phrasings that describe an absence of functionality rather than a +# capability the code performs. A TP4 assessment whose every +# mismatched_capabilities entry matches one of these carries no affirmative +# evidence of a mismatch (issue #1066): a partial snippet cannot be faulted +# for omitting the rest of the declared workflow. The match alone is not +# enough: an absence marker inside an item that also describes affirmative +# behavior (for example "Uploads source files to an undeclared endpoint +# without user consent") does not make the item an absence, so +# _tp4_mismatched_item_is_absence suppresses only clauses positively +# established to describe absence. A finite verb outside a marker's scope +# is affirmative evidence whether or not it is in the affirmative verb +# list, and a clause with no marker at all is not an absence. +_TP4_ABSENCE_PATTERNS = tuple( + re.compile(pattern) + for pattern in ( + r"^no\b", + r"^not\b", + r"^cannot\b", + r"^can'?t\b", + r"\bdoes not\b", + r"\bdoesn'?t\b", + r"\bdo not\b", + r"\bdid not\b", + r"\bno evidence\b", + r"\bno implementation\b", + r"\bno support\b", + r"\bnot implement", + r"\blacks?\b", + r"\bmissing\b", + r"\bwithout\b", + r"\bfails? to\b", + r"\babsence of\b", + r"\bomits?\b", + ) +) + + +_TP4_CLAUSE_SPLIT_PATTERN = re.compile( + r"\s*[.!?]+\s+|\s*;\s*|\s+but\s+|\s+however\s+|\s+although\s+|\s+though\s+|\s+while\s+|\s+whereas\s+" +) + +_TP4_AFFIRMATIVE_VERB_ALTERNATION = ( + r"access|archive|authenticate|backup|browse|call|collect|compress|" + r"connect|cop(?:y|ies|ied|ying)|crawl|create|decrypt|delete|download|" + r"encode|encrypt|enumerate|execute|export|exfiltrate|extract|fetch|" + r"fingerprint|gather|generate|hash|import|install|invoke|issue|leak|" + r"load|log|modify|monitor|move|navigate|notify|open|parse|perform|persist|" + r"post|process|profile|publish|pull|push|quer(?:y|ies|ied|ying)|read|" + r"record|remove|render|request|respond|restore|retrieve|run|save|" + r"scan|scrape|search|send|share|sign|spawn|store|submit|subscribe|" + r"sync|track|transmit|type|update|upload|validate|verify|write|click" +) + +_TP4_AFFIRMATIVE_VERB_PATTERN = re.compile( + rf"\b(?:{_TP4_AFFIRMATIVE_VERB_ALTERNATION})(?:s|es|ed|ing|d|ning)?\b" +) + +# A generic finite verb: any ordinary third-person singular present verb, +# not just the verbs in the affirmative alternation. Capability +# descriptions phrase behavior as finite present-tense clauses +# ("Transfers source files", "Steals credentials"), so the absence guard +# cannot rely on an exhaustive verb inventory: an unrecognized verb is +# affirmative evidence, not an absence. The shape is a word ending in "s" +# followed by an object word. Conjunctions, relative pronouns, +# prepositions and auxiliaries after it disqualify the follower, so a +# plural noun ("Files that are missing", "Credentials without encryption") +# is not mistaken for a verb, and "is", "has" and "does" are excluded +# outright, as are words ending in "ss" or "us" ("access", "status"). +_TP4_GENERIC_FINITE_VERB_PATTERN = re.compile( + r"\b(?!(?:is|has|does)\b|[a-z][a-z\-]*(?:ss|us)\b)[a-z][a-z\-]*[a-z]s\b" + r"(?=\s+(?!(?:and|or|but|that|which|who|whom|whose|without|to|in|for|of|from|on|at|by|with|over|under|via|through|after|before|during|against|into|onto|as|is|are|was|were|be|been|has|have|had|does|do|did|can|could|will|would|not|no|if|when|while|whereas|although|though|however|then|than|so|yet)\b)[a-z])" +) + +# A modal plus base-form verb: capability descriptions also phrase +# behavior as "can steal", "could transfer", "may collect", where the verb +# is a base form, so neither the affirmative list nor the third-person +# singular shape sees it. "The script can steal credentials without +# consent" is affirmative behavior qualified by "without consent", not an +# absence, so the modal phrase counts as a finite verb for the scope +# checks below: it is affirmative evidence when it sits outside a +# governing marker ("No user can access the admin panel" stays an absence +# because the leading "no" governs it). A negated modal ("can not", +# "must never") is not a verb phrase at all, and a modal plus an absence +# verb ("may lack", "will fail to") still describes an omission, so both +# are excluded. "cannot" is one word, so it never matches the modal slot, +# and "can be" is a copula, not behavior. An optional adverb ("also", +# "silently") may sit between the modal and the verb. +_TP4_MODAL_AUXILIARY_ALTERNATION = r"can|could|may|might|will|would|shall|should|must" +_TP4_MODAL_ADVERB_ALTERNATION = r"also|still|even|silently|automatically|directly|quickly|easily" +_TP4_MODAL_VERB_PATTERN = re.compile( + rf"\b(?:{_TP4_MODAL_AUXILIARY_ALTERNATION})\s+" + rf"(?:(?:{_TP4_MODAL_ADVERB_ALTERNATION})\s+)?" + r"(?!(?:not|never|no)\b)" + r"(?!(?:lack|omit|fail|miss)(?:s|es|ed|ing)?\b)" + r"(?!(?:be|been|being)\b)" + r"[a-z][a-z\-]*\b" +) + +# A subject-first affirmative clause after a comma ("the code sends +# credentials in plaintext", "it uploads files"): a determiner or pronoun +# subject, at most three words of subject noun phrase, then a capability +# verb with an object after it. A bare noun phrase ("file writes, or remote +# uploads") has no determiner and its verb is followed by a comma or "or", +# so a missing-capability list does not match. The object requirement is +# what separates "the code sends credentials" from a list item such as +# "the file writes, ...", whose verb has no object. +_TP4_SUBJECT_FIRST_AFFIRMATIVE_PATTERN = re.compile( + rf"^(?:(?:it|they|he|she|this|that)\s+|(?:the|this|that|its|their)\s+" + rf"[a-z][a-z\-]*(?:\s+[a-z][a-z\-]*){{0,2}}\s+)" + rf"(?:{_TP4_AFFIRMATIVE_VERB_ALTERNATION})(?:s|es|ed|ing|d|ning)?\s+" + rf"(?!or\b|and\b)[a-z]" +) + +# The same subject-first shape with a generic finite verb: "the code +# transfers files" is a main clause even though "transfers" is outside +# the affirmative alternation. The absence verbs "lacks" and "omits" are +# excluded from the verb slot, since "the code lacks encryption" still +# describes an absence. +_TP4_SUBJECT_FIRST_GENERIC_PATTERN = re.compile( + r"^(?:(?:it|they|he|she|this|that)\s+|(?:the|this|that|its|their)\s+" + r"[a-z][a-z\-]*(?:\s+[a-z][a-z\-]*){0,2}\s+)" + r"(?!(?:lacks|omits)\s)[a-z][a-z\-]*[a-z]s\s+" + r"(?!or\b|and\b)[a-z]" +) + +# The same subject-first shape with a modal plus base-form verb: "the +# script can steal credentials" is a main clause even though "steal" is a +# base form outside the affirmative alternation. The same exclusions +# apply: a negated modal and a modal plus an absence verb are not +# affirmative evidence. +_TP4_SUBJECT_FIRST_MODAL_PATTERN = re.compile( + rf"^(?:(?:it|they|he|she|this|that)\s+|(?:the|this|that|its|their)\s+" + rf"[a-z][a-z\-]*(?:\s+[a-z][a-z\-]*){{0,2}}\s+)" + rf"(?:{_TP4_MODAL_AUXILIARY_ALTERNATION})\s+" + rf"(?:(?:{_TP4_MODAL_ADVERB_ALTERNATION})\s+)?" + r"(?!(?:not|never|no)\b)" + r"(?!(?:lack|omit|fail|miss)(?:s|es|ed|ing)?\b)" + r"(?!(?:be|been|being)\b)" + r"[a-z][a-z\-]*\s+" + r"(?!or\b|and\b)[a-z]" +) + +# A noun-first absence statement ("Search functionality is missing", +# "Network access is not implemented"): the capability word is the subject +# noun, not an affirmative verb, so the verb pattern must not read it as +# behavior. The subject is a single short noun phrase (an optional +# determiner plus up to three words): a longer subject can hide a finite +# verb with an object ("The code sends credentials and search is missing"), +# so it stays with the verb check. Deliberately no "and"-joined subjects: +# "Search functionality and network access are missing" does not match and +# stays detection-side, which is the safe direction (an emitted TP4 for a +# pure absence) versus dropping a real affirmative finding. An optional +# bounded prepositional tail ("in the workflow") is allowed; anything +# longer or verb-bearing falls through to the verb check. A relative +# pronoun (that, which, who) in the subject disqualifies the shortcut: +# three words do not establish a noun-only phrase, and "Uploads files that +# are missing" opens with a finite verb plus object, so it falls through +# to the verb check, where the verb before the marker keeps the finding. +# A relative clause with no affirmative verb ("Files that are missing") +# still stays suppressed, because the verb check finds no verb before the +# marker. +_TP4_NOUN_FIRST_ABSENCE_PATTERN = re.compile( + r"^(?:the\s+)?" + r"(?:(?!(?:that|which|who|whom|whose)\b)[a-z][a-z\-]*)" + r"(?:\s+(?!(?:that|which|who|whom|whose)\b)[a-z][a-z\-]*){0,2}" + r"\s+(?:is|are|was|were)\s+" + r"(?:missing|not\s+implement\w*)" + r"(?:\s+(?:in|from|for|of|within)\s+(?:the\s+)?[a-z][a-z\-]*(?:\s+[a-z][a-z\-]*){0,2})?" + r"(?:[.!?]+\s*)?$" +) + + +def _tp4_strip_leading_absence_modifier(clause: str) -> str: + """Drop a leading absence phrase closed by a comma, if it precedes any verb. + + "Without user consent, uploads source files to a remote endpoint" opens + with an absence phrase that only modifies the affirmative main clause, + so the marker must not govern the verbs after the comma. The phrase is + only dropped when the comma comes before the first capability verb: + "No evidence of querying, storing videos" keeps its marker because the + verb sits inside the leading phrase. The text after the comma must also + read as a main clause, not as a continuation of the missing-capability + list: the phrase is dropped when it is a "without" modifier, when the + remainder opens with a capability verb ("No support for TLS, sends + credentials in plaintext"), or when it opens with a subject-first + affirmative clause ("No support for TLS, the code sends credentials in + plaintext"). The verb that opens the remainder need not be in the + affirmative list ("No support for TLS, transfers source files"): an + ordinary finite verb is a main clause too. A subject-first modal clause + ("No support for TLS, the script can steal credentials") is a main + clause too, as is a remainder that opens with a bare modal phrase + ("No support for TLS, can steal credentials"). A list such as "No evidence + of network activity, file writes, or remote uploads" keeps its + negation because the remainder opens with a noun, so the whole entry + still describes absent behavior. + """ + marker_end = None + for pattern in _TP4_ABSENCE_PATTERNS: + match = pattern.search(clause) + if match is not None and match.start() == 0: + marker_end = match.end() if marker_end is None else min(marker_end, match.end()) + if marker_end is None: + return clause + comma = clause.find(",", marker_end) + if comma == -1: + return clause + first_verb = min(_tp4_finite_verbs(clause), key=lambda match: match.start(), default=None) + if first_verb is not None and first_verb.start() < comma: + return clause + remainder = clause[comma + 1 :].strip() + is_without_modifier = clause.casefold().startswith("without ") + remainder_opens_with_verb = ( + _TP4_AFFIRMATIVE_VERB_PATTERN.match(remainder) is not None + or _TP4_GENERIC_FINITE_VERB_PATTERN.match(remainder) is not None + or _TP4_MODAL_VERB_PATTERN.match(remainder) is not None + ) + remainder_is_subject_first_clause = ( + _TP4_SUBJECT_FIRST_AFFIRMATIVE_PATTERN.match(remainder) is not None + or _TP4_SUBJECT_FIRST_GENERIC_PATTERN.match(remainder) is not None + or _TP4_SUBJECT_FIRST_MODAL_PATTERN.match(remainder) is not None + ) + if not (is_without_modifier or remainder_opens_with_verb or remainder_is_subject_first_clause): + return clause + return remainder + + +def _tp4_finite_verbs(clause: str) -> list[re.Match[str]]: + """Return finite capability-verb matches in a clause. + + The recognized affirmative verbs, the generic finite-verb shape, and + a modal plus base-form verb: the absence guard must treat an ordinary + verb outside the inventory ("transfers", "steals") as affirmative + evidence, not as an absence, and a base-form verb after a modal + ("can steal") is a finite verb too, since neither the list nor the + third-person singular shape sees it. + """ + return [ + *_TP4_AFFIRMATIVE_VERB_PATTERN.finditer(clause), + *_TP4_GENERIC_FINITE_VERB_PATTERN.finditer(clause), + *_TP4_MODAL_VERB_PATTERN.finditer(clause), + ] + + +def _tp4_starts_with_affirmative_clause(segment: str) -> bool: + """Return whether a segment opens with a finite affirmative clause. + + Either a finite verb plus object ("steals credentials"), a modal plus + base-form verb plus object ("can steal credentials"), or a subject + plus finite verb plus object ("the code steals credentials"). Gerund + and participle forms ("storing videos", "uploaded files") do not open + a finite clause, so an "and"-joined gerund list stays an absence. + """ + verb = _TP4_AFFIRMATIVE_VERB_PATTERN.match(segment) + if verb is not None and not verb.group(0).endswith(("ing", "ed")): + if re.match(r"\s+[a-z]", segment[verb.end() :]) is not None: + return True + modal = _TP4_MODAL_VERB_PATTERN.match(segment) + if modal is not None and re.match(r"\s+[a-z]", segment[modal.end() :]) is not None: + return True + if _TP4_GENERIC_FINITE_VERB_PATTERN.match(segment) is not None: + return True + if _TP4_SUBJECT_FIRST_AFFIRMATIVE_PATTERN.match(segment) is not None: + return True + if _TP4_SUBJECT_FIRST_GENERIC_PATTERN.match(segment) is not None: + return True + return _TP4_SUBJECT_FIRST_MODAL_PATTERN.match(segment) is not None + + +def _tp4_has_and_joined_affirmative(clause: str) -> bool: + """Return whether an "and"-joined verb phrase describes behavior. + + A marker scopes only its own verb phrase: "Lacks encryption and steals + credentials" is not a pure absence, because the second phrase is an + affirmative finite clause. The phrase after "and" must open with a + finite affirmative clause and must not itself open with an absence + marker ("... and does not authenticate" stays an absence). + """ + for match in re.finditer(r"\s+and\s+", clause): + rest = clause[match.end() :] + if any(pattern.match(rest) is not None for pattern in _TP4_ABSENCE_PATTERNS): + continue + if _tp4_starts_with_affirmative_clause(rest): + return True + return False + + +def _tp4_clause_is_positive_absence(clause: str) -> bool: + """Return whether one clause positively describes an absence. + + A clause counts as an absence only when an absence marker governs it + and no finite verb sits outside that marker's scope. A clause with no + marker at all ("Steals credentials" after "No support for TLS.") is + not an absence, a finite verb before the first marker is affirmative + evidence, and "without" does not negate a following finite verb + ("Without user consent uploads files" describes uploading), so a verb + after a leading "without" is affirmative evidence too. A modal plus + base-form verb ("can steal") is a finite verb too: it is affirmative + evidence when it precedes the marker or follows a "without" that only + qualifies the action, and it stays governed when a leading marker + precedes it ("No user can access the admin panel"). A negating + auxiliary ("does not", "cannot", "fails to") does govern the verb + phrase after it, so "The code does not implement search" stays an + absence. + """ + marker_matches = [ + match for pattern in _TP4_ABSENCE_PATTERNS for match in pattern.finditer(clause) + ] + if not marker_matches: + return False + first_marker = min(marker_matches, key=lambda match: match.start()) + for verb in _tp4_finite_verbs(clause): + if verb.start() < first_marker.start(): + return False + if first_marker.group(0) == "without": + for verb in _tp4_finite_verbs(clause): + if verb.start() > first_marker.end(): + return False + if _tp4_has_and_joined_affirmative(clause): + return False + return True + + +def _tp4_mismatched_item_is_absence(item: str) -> bool: + """Return whether one mismatched capability is phrased as an absence. + + Suppression needs positive evidence of absence: an absence marker only + counts when every clause of the item is positively established to + describe absence, and an unproven or unknown clause is retained rather + than suppressed. A finite verb outside the marker's scope is + affirmative evidence whether or not it is in the affirmative verb + list, so "Transfers source files without user consent" and "No support + for TLS. Steals credentials" are not absences, and a modal plus + base-form verb is a finite verb too, so "The script can steal + credentials without consent" is not an absence either, while "No evidence of + querying videos" and "The code does not implement search" still are: + their verbs sit under the marker. Clauses split on contrast boundaries + and sentence boundaries, so comma-separated enumerations stay under + one leading marker while "No support for TLS. Sends credentials in + plaintext" keeps its second sentence affirmative, and an "and"-joined + finite verb phrase ("Lacks encryption and steals credentials") is a + separate affirmative clause. A leading absence phrase closed by a comma + ("Without user consent, uploads source files ...") only modifies the + main clause and does not govern the verbs after the comma, while a + comma-separated list of missing capabilities ("No evidence of network + activity, file writes, or remote uploads") stays under its leading + negation. A subject-first affirmative clause after the comma ("No support + for TLS, the code sends credentials in plaintext") is a separate main + clause, not a list item, so the leading phrase is dropped for it. A + noun-first absence statement ("Search functionality is missing", + "Network access is not implemented") is an absence too: its capability + word is the subject noun, not an affirmative verb. A relative pronoun + in that subject ("Uploads files that are missing") disqualifies the + shortcut, since the phrase may open with a finite verb and object, so + it falls through to the clause check instead. + """ + normalized = " ".join(item.strip().casefold().split()) + if not normalized: + return False + if not any(pattern.search(normalized) for pattern in _TP4_ABSENCE_PATTERNS): + return False + for clause in _TP4_CLAUSE_SPLIT_PATTERN.split(normalized): + clause = _tp4_strip_leading_absence_modifier(clause.strip()) + if not clause: + continue + if _TP4_NOUN_FIRST_ABSENCE_PATTERN.match(clause) is not None: + continue + if not _tp4_clause_is_positive_absence(clause): + return False + return True + + +def _tp4_all_mismatched_are_absences(items: list[str]) -> bool: + """Return whether every listed capability is an absence, not a behavior.""" + return bool(items) and all(_tp4_mismatched_item_is_absence(item) for item in items) + + def _tp4_finding( result: _TP4AnalysisResult, batch: Batch, @@ -1155,6 +1533,13 @@ def _tp4_finding( """Convert one bounded batch assessment without dropping source evidence.""" if not result.is_mismatch or result.confidence < 0.5: return None + if _tp4_all_mismatched_are_absences(result.mismatched_capabilities): + logger.debug( + "%s: TP4 assessment suppressed: every mismatched capability is " + "phrased as an absence, so there is no affirmative mismatch evidence", + ANALYZER_ID, + ) + return None declared = (result.declared_purpose_summary or description[:512])[:512] actual = result.actual_behavior_summary[:1024] mismatched = [str(item)[:256] for item in result.mismatched_capabilities[:16]] @@ -1168,9 +1553,14 @@ def _tp4_finding( return Finding( rule_id="TP4", message=message, - severity="HIGH" if result.confidence >= 0.7 else "MEDIUM", + # TP4 severity is fixed MEDIUM, matching the README pattern table + # and the detailed TP4 guide; there is no confidence-based + # escalation to HIGH (issue #1066). + severity="MEDIUM", confidence=result.confidence, - file="SKILL.md", + file=batch.file_path, + start_line=batch.start_line, + end_line=batch.end_line, category=_CATEGORY, pattern=get_pattern_name("TP4"), finding=message, diff --git a/tests/test_mcp_tool_poisoning.py b/tests/test_mcp_tool_poisoning.py index 67b8d7ad1..8cb82700e 100644 --- a/tests/test_mcp_tool_poisoning.py +++ b/tests/test_mcp_tool_poisoning.py @@ -1139,7 +1139,7 @@ def test_mismatch_detected(self, monkeypatch: pytest.MonkeyPatch): result = node(state) tp4 = [f for f in result["findings"] if f.rule_id == "TP4"] assert len(tp4) >= 1 - assert tp4[0].severity in {"HIGH", "MEDIUM"} + assert tp4[0].severity == "MEDIUM" assert tp4[0].pattern == "Description-Behavior Mismatch" assert tp4[0].finding == tp4[0].message @@ -1181,6 +1181,454 @@ def test_finding_maps_result_metadata_for_unit_ci(self): assert finding.evidence["code_end_line"] == 12 +class TestTP4FindingIssue1066: + """TP4 anchors, severity, and absence-only guard (issue #1066).""" + + def _result(self, confidence: float, mismatched: list[str]): + return mcp_tool_poisoning._TP4AnalysisResult( + is_mismatch=True, + confidence=confidence, + declared_purpose_summary="Video intelligence briefing skill", + actual_behavior_summary="Renders briefing documents", + mismatched_capabilities=mismatched, + explanation="The chunk does not show the declared video work.", + ) + + def test_finding_anchors_to_real_file_and_lines(self): + batch = mcp_tool_poisoning.Batch( + file_path="references/briefing-authoring.md", + content="unzip -q -o out.epub -d _chk", + start_line=27, + end_line=28, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.99, ["EPUB archive processing"]), + batch, + "Video intelligence briefing skill", + ) + + assert finding is not None + assert finding.file == "references/briefing-authoring.md" + assert finding.start_line == 27 + assert finding.end_line == 28 + assert finding.evidence["code_path"] == "references/briefing-authoring.md" + assert finding.evidence["code_start_line"] == 27 + assert finding.evidence["code_end_line"] == 28 + + def test_finding_severity_is_medium_at_high_confidence(self): + batch = mcp_tool_poisoning.Batch( + file_path="SKILL.md", + content="python scripts/markdown_pdf.py", + start_line=697, + end_line=697, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.99, ["Markdown-to-PDF export of briefing files"]), + batch, + "Video intelligence briefing skill", + ) + + assert finding is not None + assert finding.severity == "MEDIUM" + + def test_absence_only_mismatched_capabilities_suppressed(self): + batch = mcp_tool_poisoning.Batch( + file_path="SKILL.md", + content="pytest tests/evals/test_search_quality.py -v -s", + start_line=314, + end_line=314, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result( + 0.99, + [ + "No evidence of querying videos, transcripts, concepts, " + "quotes, corpus status, or profile data", + "No implementation of the described nugget synthesis/save behavior", + ], + ), + batch, + "Video corpus search skill", + ) + + assert finding is None + + def test_mixed_mismatched_capabilities_still_emitted(self): + batch = mcp_tool_poisoning.Batch( + file_path="SKILL.md", + content="pytest tests/evals/test_search_quality.py -v -s", + start_line=314, + end_line=314, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result( + 0.99, + [ + "Runs automated tests/evaluations instead of providing " + "corpus search functionality", + "No evidence of querying videos", + ], + ), + batch, + "Video corpus search skill", + ) + + assert finding is not None + assert finding.severity == "MEDIUM" + + def test_affirmative_mismatched_capability_emitted(self): + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.9, ["Sends source data to a remote endpoint"]), + batch, + "A local text utility", + ) + + assert finding is not None + + def test_subject_first_affirmative_after_comma_emitted(self): + # Reviewer case (yashrajp22, 2026-11-11): the comma-separated + # remainder is a subject-first affirmative clause, not a + # missing-capability list item. + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result( + 0.9, + ["No support for TLS, the code sends credentials in plaintext"], + ), + batch, + "A local text utility", + ) + + assert finding is not None + + def test_noun_first_absence_only_suppressed(self): + # Reviewer cases (yashrajp22, 2026-11-11): noun-first absence + # statements describe omitted workflow steps, not behavior. + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + for item in ( + "Search functionality is missing", + "Network access is not implemented", + ): + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.9, [item]), + batch, + "A local text utility", + ) + + assert finding is None + + def test_affirmative_relative_clause_not_swallowed_by_noun_first_absence(self): + # Reviewer case (yashrajp22, 2026-11-12): "Uploads files that are + # missing" opens with a finite verb plus object, so the noun-first + # absence shortcut must not consume it. The genuine noun-first + # absences in test_noun_first_absence_only_suppressed stay + # suppressed. + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.9, ["Uploads files that are missing"]), + batch, + "A local text utility", + ) + + assert finding is not None + + def test_unrecognized_affirmative_verbs_retained(self): + # Reviewer cases (rng1995, 2026-11-13): suppression needs positive + # evidence of absence, not merely the failure to match the finite + # affirmative-verb list. "Transfers" and "steals" are outside the + # list, but both items affirmatively describe behavior, so the TP4 + # finding must be kept. The pure-absence negatives alongside them + # stay suppressed. + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + for item in ( + "Transfers source files without user consent", + "No support for TLS. Steals credentials", + ): + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.9, [item]), + batch, + "A local text utility", + ) + + assert finding is not None + + for item in ( + "No evidence of querying videos", + "The code does not implement search", + "No support for TLS", + ): + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.9, [item]), + batch, + "A local text utility", + ) + + assert finding is None + + def test_modal_affirmative_capability_emitted(self): + # Reviewer case (rng1995, 2026-11-19): a modal plus base-form verb + # ("can steal") is a finite verb too, so "without consent" only + # qualifies the action and must not suppress the finding. The + # pure-absence controls alongside it stay suppressed: a leading + # "cannot", a modal governed by a leading "no", and a modal plus + # an absence verb. + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + for item in ( + "The script can steal credentials without consent.", + "The tool could transfer files without approval", + ): + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.9, [item]), + batch, + "A local text utility", + ) + + assert finding is not None + + for item in ( + "Cannot steal credentials", + "No user can access the admin panel", + "The code may lack encryption", + ): + finding = mcp_tool_poisoning._tp4_finding( + self._result(0.9, [item]), + batch, + "A local text utility", + ) + + assert finding is None + + def test_affirmative_capability_containing_without_emitted(self): + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result( + 0.9, + ["Uploads source files to an undeclared endpoint without user consent"], + ), + batch, + "A local text utility", + ) + + assert finding is not None + + def test_marker_first_affirmative_capability_emitted(self): + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result( + 0.9, + ["Without user consent, uploads source files to a remote endpoint"], + ), + batch, + "A local text utility", + ) + + assert finding is not None + assert finding.severity == "MEDIUM" + + def test_comma_separated_list_of_missing_capabilities_suppressed(self): + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result( + 0.9, + ["No evidence of network activity, file writes, or remote uploads"], + ), + batch, + "A local text utility", + ) + + assert finding is None + + def test_affirmative_action_with_negated_clause_emitted(self): + batch = mcp_tool_poisoning.Batch( + file_path="scripts/tool.py", + content="upload(source)", + start_line=12, + end_line=12, + ) + + finding = mcp_tool_poisoning._tp4_finding( + self._result( + 0.9, + [ + "Uploads source files to an external endpoint but does not log the transfer", + "Sends credentials to a remote host without validating the certificate", + ], + ), + batch, + "A local text utility", + ) + + assert finding is not None + + @pytest.mark.parametrize( + "item, expected", + [ + ("No evidence of querying videos", True), + ("No implementation of the described behavior", True), + ("The code does not implement search", True), + ("Lacks any corpus query", True), + ("Missing authentication", True), + ("Runs automated tests instead of searching", False), + ("Sends source data to a remote endpoint", False), + ("Markdown-to-PDF export of briefing files", False), + ( + "Uploads source files to an undeclared endpoint without user consent", + False, + ), + ("Uploads files but does not encrypt them", False), + ("No support for TLS, but sends credentials in plaintext", False), + ( + "Sends credentials to a remote host without validating the certificate", + False, + ), + ( + "Without user consent, uploads source files to a remote endpoint", + False, + ), + ("No support for TLS. Sends credentials in plaintext", False), + ("Persists credentials without user consent", False), + ("No support for TLS", True), + ("Without user consent, no encryption is performed", True), + ("No evidence of querying, storing videos", True), + ( + "No evidence of network activity, file writes, or remote uploads", + True, + ), + ("No support for TLS, sends credentials in plaintext", False), + # Reviewer cases (yashrajp22, 2026-11-11): a subject-first + # affirmative clause after the comma is a separate main clause, + # not a list item, so the leading absence phrase must not + # govern its verb. + ("No support for TLS, the code sends credentials in plaintext", False), + ("No support for TLS, the tool uploads files to a remote server", False), + ("No support for TLS, it sends credentials in plaintext", False), + # Reviewer cases (yashrajp22, 2026-11-11): noun-first absence + # statements. "search" and "access" are subject nouns here, not + # affirmative verbs. + ("Search functionality is missing", True), + ("Network access is not implemented", True), + ("Search functionality is missing from the workflow", True), + # Reviewer case (yashrajp22, 2026-11-12): a relative pronoun + # in the subject disqualifies the noun-first shortcut, so a + # finite verb plus object keeps the item affirmative. A + # relative clause with no affirmative verb still describes an + # absence. + ("Uploads files that are missing", False), + ("Uploads files which are missing", False), + ("Files that are missing", True), + # Affirmative controls for the noun-first shape: a hidden finite + # verb in a long subject, and an affirmative clause joined to + # the absence statement, both stay affirmative. + ("The code sends credentials and search is missing", False), + ( + "Network access is not implemented, the code sends credentials in plaintext", + False, + ), + # Reviewer cases (rng1995, 2026-11-13): suppression needs + # positive evidence of absence. An ordinary verb outside the + # affirmative list ("transfers", "steals") is affirmative + # evidence, a clause with no marker is not an absence, a + # "without" adjunct does not negate a following verb, and an + # "and"-joined finite verb phrase is a separate clause. + ("Transfers source files without user consent", False), + ("No support for TLS. Steals credentials", False), + ("Lacks encryption and steals credentials", False), + ("Without user consent uploads source files", False), + ("No support for TLS, transfers source files", False), + ("No support for TLS, the code transfers files", False), + # Guards: an "and"-joined gerund list and an "and"-joined + # negated verb phrase stay absences. + ("No evidence of querying and storing videos", True), + ("Does not encrypt and does not authenticate", True), + # Reviewer case (rng1995, 2026-11-19): a modal plus base-form + # verb ("can steal") is a finite verb, so "without consent" + # only qualifies the action. The same shape holds after a + # comma, as a bare modal remainder, as an "and"-joined phrase, + # and after a leading "without". A modal governed by a leading + # marker, or joined to an absence verb, still describes an + # omission. + ("The script can steal credentials without consent.", False), + ("The script can steal credentials without consent", False), + ("The tool could transfer files without approval", False), + ( + "The script may silently exfiltrate data without approval", + False, + ), + ( + "No support for TLS, the script can steal credentials", + False, + ), + ("No support for TLS, can steal credentials", False), + ("Lacks encryption and can steal credentials", False), + ("Without consent can steal credentials", False), + ("No user can access the admin panel", True), + ("The code may lack encryption", True), + ("Cannot steal credentials", True), + ], + ) + def test_absence_phrasing_detection(self, item: str, expected: bool): + assert mcp_tool_poisoning._tp4_mismatched_item_is_absence(item) is expected + + class TestTP4MarkdownFences: def test_markdown_only_fenced_python_reaches_tp4(self, monkeypatch: pytest.MonkeyPatch): structured = _mock_tp4_structured_llm( @@ -2195,7 +2643,9 @@ async def arun_batches_detailed(self, batches): if confidence is not None: finding = result.findings[0] assert finding.rule_id == "TP4" - assert finding.severity == ("HIGH" if confidence >= 0.7 else "MEDIUM") + # TP4 severity is MEDIUM at any confidence (README pattern table, + # issue #1066); the old confidence-based escalation is removed. + assert finding.severity == "MEDIUM" assert finding.evidence["assessment_complete"] is False assert "llm-format-recovery" in finding.tags producers = [ diff --git a/tests/unit/test_tp4_openai_http.py b/tests/unit/test_tp4_openai_http.py index 2ee5711fe..f47a04bd9 100644 --- a/tests/unit/test_tp4_openai_http.py +++ b/tests/unit/test_tp4_openai_http.py @@ -341,7 +341,10 @@ def test_full_scanner_graph_reports_tp4_http_results( findings = [f for f in result["findings"] if f.rule_id == "TP4"] assert bool(findings) is (mode == "mismatch") if findings: - assert findings[0].file == "SKILL.md" + # TP4 findings anchor to the assessed file, not SKILL.md (issue #1066). + assert findings[0].file == "format.py" + assert findings[0].start_line == 1 + assert findings[0].end_line == 2 assert findings[0].evidence["code_path"] == "format.py" assert findings[0].evidence["code_start_line"] == 1 assert findings[0].evidence["code_end_line"] == 2