From 5c1c40f088a7b4a20f908f5c62d8b2689ce17aa1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 22:23:13 +0000 Subject: [PATCH 1/2] fix: intake honours a difficulty declared in the raw text (#274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Intake Agent sized every idea with the shared sizing heuristic and wrote that estimate into the prompt's `Difficulty:` header, silently discarding a difficulty the author had already stated — the ideas.md house style ends a bullet with "Difficulty large, supervised." Worse, the declaration was prose, so it was swallowed by the derived title and leaked into the slug/filename. A human who has thought about the work is better evidence than a keyword heuristic that only sees words, so a declaration now wins: - `_declared_fields` reads Difficulty/Autonomy/Priority from the raw text, including the trailing-autonomy idiom ("Difficulty large, supervised.") and spelling variants ("Too Large", "human required"). - Code fences and inline spans are masked first, so a prompt quoting a `Difficulty:` line (a repro, a transcript) does not re-size itself. - The IntakeDecision reports provenance (difficulty_source / autonomy_source / priority_source, plus the estimate it overrode) and flags the override in the risks block, so the dry-run review sees both numbers. - The declaration clause is stripped from the derived title/slug only; the prompt body stays verbatim. - `formalise` applies the same precedence. The sizing faculty is unchanged: it stays the pure estimator, and the override lives in the agent that owns the header. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012RXdgVSW8bUhVdEXEz6AAJ --- agents/conductors/intake/AGENTS.md | 10 +- agents/conductors/intake/_intake.py | 127 ++++++++++++++++++++--- agents/faculties/sizing/AGENTS.md | 4 +- agents/faculties/sizing/_sizing.py | 4 +- skills/intake/intake.md | 4 +- tests/test_intake_declared_difficulty.py | 93 +++++++++++++++++ 6 files changed, 224 insertions(+), 18 deletions(-) create mode 100644 tests/test_intake_declared_difficulty.py diff --git a/agents/conductors/intake/AGENTS.md b/agents/conductors/intake/AGENTS.md index 17f5d330..8f048b0e 100644 --- a/agents/conductors/intake/AGENTS.md +++ b/agents/conductors/intake/AGENTS.md @@ -55,9 +55,15 @@ Proposed path · Header (the block written verbatim) · Risks · Next action - **Difficulty** — from the shared `agents/faculties/sizing/` faculty, **persisted into the header**, because scope is decided during the intake back-and-forth. The Feature Agent later *trusts* this number rather than recomputing a divergent - one. + one. A difficulty **declared in the raw text** (`Difficulty: large`, or the + `ideas.md` house style `Difficulty large, supervised.`) **wins** over the + estimate: the author knows the scope, the heuristic only sees words. The + IntakeDecision reports which one it used (`difficulty_source`) and what the + heuristic would have said, and the declaration is kept out of the derived + title/slug. Declarations are read from prose only — a quoted `Difficulty:` + inside a code fence or backticks is not one. - **Autonomy** (`safe|supervised|human-required`) and **Priority** - (`low|normal|high`) — the "can an agent safely handle this?" and "how urgent?" + (`low|normal|high`) — the "can an agent safely handle this?" and "how urgent?" questions; a declared value wins here too. inputs, written into the header. **`too-large` means decompose, never dispatch.** A prompt sized `too-large` is diff --git a/agents/conductors/intake/_intake.py b/agents/conductors/intake/_intake.py index 25b0469c..47c041e8 100755 --- a/agents/conductors/intake/_intake.py +++ b/agents/conductors/intake/_intake.py @@ -16,7 +16,9 @@ classification lands in `triage/` — the existing unclassified bucket, reused not reinvented. Difficulty is OWNED here (scope is decided during the intake back-and-forth) and persisted into the header via the shared sizing faculty, so -the Feature Agent later trusts the same number. +the Feature Agent later trusts the same number — unless the author already +declared one in the raw text, which outranks the estimate (see "human-declared +header fields"). Stdlib only. Writes ONLY under --apply; every other path is read-only. """ @@ -108,6 +110,80 @@ "when there is time", "backlog"] +# --- human-declared header fields --------------------------------------------- +# Raw input often states its own scope: the ideas.md house style ends a bullet +# with "Difficulty large, supervised.", and a pasted bug report may carry a +# "Priority: high" line. A human who has thought about the work is better +# evidence than a keyword heuristic that only sees words, so a declaration +# WINS over the estimate here. The sizing faculty stays the pure estimator — +# the override belongs to the agent that owns the header (this one). +# +# Declarations are read only from prose: fenced blocks and inline code spans are +# masked out first, so a prompt *quoting* `Difficulty: large` (a repro, a +# transcript) does not silently re-size itself. +_DIFFICULTY_ALT = r"too[-\s]large|small|medium|large" +_AUTONOMY_ALT = r"human[-\s]required|supervised|safe" +_PRIORITY_ALT = r"high|normal|low" +# Between key and value: a colon/equals, "is", or nothing at all ("Difficulty large"). +_DECL_SEP = r"\s*(?::|=|\bis\b)?\s*" +_DECLARATION = re.compile( + rf"\bdifficulty{_DECL_SEP}({_DIFFICULTY_ALT})\b" + rf"(?:\s*[,/&]?\s*(?:and\s+)?({_AUTONOMY_ALT})\b)?" + rf"|\bautonomy{_DECL_SEP}({_AUTONOMY_ALT})\b" + rf"|\bpriority{_DECL_SEP}({_PRIORITY_ALT})\b", + re.IGNORECASE) +_CODE_SPAN = re.compile(r"```.*?```|`[^`\n]*`", re.DOTALL) + + +def _mask_code(text: str) -> str: + """`text` with code spans blanked to spaces (offsets and lines preserved).""" + return _CODE_SPAN.sub(lambda m: re.sub(r"\S", " ", m.group(0)), text) + + +def _declared_fields(text: str): + """(fields, spans) — the header fields the human stated in the raw text. + + First declaration of each field wins; `spans` are its offsets in `text` so + the clause can be kept out of the derived title/slug. + """ + fields, spans = {}, [] + for m in _DECLARATION.finditer(_mask_code(text)): + difficulty, trailing_autonomy, autonomy, priority = m.groups() + if difficulty: + fields.setdefault("difficulty", _norm_level(difficulty)) + if trailing_autonomy or autonomy: + fields.setdefault("autonomy", _norm_level(trailing_autonomy or autonomy)) + if priority: + fields.setdefault("priority", _norm_level(priority)) + spans.append(m.span()) + return fields, spans + + +def _norm_level(value: str) -> str: + """"Too Large" / "human required" -> the canonical hyphenated header value.""" + return re.sub(r"[\s-]+", "-", value.strip().lower()) + + +def _strip_declarations(text: str, spans: list) -> str: + """`text` with the declaration clauses removed — for title derivation only. + + The prompt body itself stays verbatim (word-vomit is intent); this exists so + "Fix the docstring. Difficulty: large." does not title the task — and name + the file — after its own difficulty declaration. + """ + if not spans: + return text + chars = list(text) + for start, end in spans: + chars[start:end] = " " * (end - start) + out = "".join(chars) + # Tidy the punctuation the removed clause left stranded (title use only). + out = re.sub(r"[ \t]+", " ", out) + out = re.sub(r"\s+([.,;:])", r"\1", out) + out = re.sub(r"([.,;:])(\s*[.,;:])+", r"\1", out) + return out if re.search(r"\w", out) else text + + def _slug(text: str, maxwords: int = 7) -> str: words = re.findall(r"[A-Za-z0-9]+", text.lower()) slug = "_".join(words[:maxwords]) @@ -268,13 +344,17 @@ def analyse(text: str, source: str): # Build a prompt-shaped dict the shared sizing faculty understands. p = {"text": text, "repos": repos, "words": len(text.split()), "target": target, "work_type": work_type} - level, score, factors = estimate_difficulty(p) - - autonomy = infer_autonomy(level, factors) - priority = infer_priority(text) + estimated, score, factors = estimate_difficulty(p) + + # A human declaration outranks the estimate (see "human-declared header + # fields"); the estimate is kept alongside it so the review can see both. + declared, decl_spans = _declared_fields(text) + level = declared.get("difficulty", estimated) + autonomy = declared.get("autonomy") or infer_autonomy(level, factors) + priority = declared.get("priority") or infer_priority(text) workflow = infer_workflow(target, repos) - title = _title(text) + title = _title(_strip_declarations(text, decl_spans)) slug = _slug(title) folder = work_type if confidence != "low" else "triage" if folder == "triage": @@ -302,12 +382,17 @@ def analyse(text: str, source: str): "difficulty": level, "difficulty_score": score, "difficulty_factors": factors, + "difficulty_estimated": estimated, + "difficulty_source": "declared" if "difficulty" in declared else "estimated", "autonomy": autonomy, + "autonomy_source": "declared" if "autonomy" in declared else "inferred", "priority": priority, + "priority_source": "declared" if "priority" in declared else "inferred", + "declared_fields": declared, "workflow": workflow, "proposed_path": proposed, "header": header, - "risks": _risks(level, factors, confidence, target), + "risks": _risks(level, factors, confidence, target, declared, estimated), "next_action": _next_action(proposed, confidence), } @@ -322,8 +407,16 @@ def _render_header(title, work_type, target_display, repos, level, autonomy, pri return "\n".join(lines) -def _risks(level, factors, confidence, target): +def _risks(level, factors, confidence, target, declared=None, estimated=None): out = [] + declared = declared or {} + if "difficulty" in declared and declared["difficulty"] != estimated: + out.append(f"Difficulty {declared['difficulty']} declared in the raw text " + f"— it overrides the heuristic estimate ({estimated}).") + for field in ("autonomy", "priority"): + if field in declared: + out.append(f"{field.capitalize()} {declared[field]} declared in the raw " + f"text — taken as written, not inferred.") if confidence == "low": out.append("Low classification confidence — filed to triage/ for a human " "to re-home once the work type is clear.") @@ -1548,12 +1641,15 @@ def _derive_fields(text: str, work_type: str, target: str) -> dict: p = {"text": text, "repos": repos, "words": len(text.split()), "target": target, "work_type": work_type} level, _score, factors = estimate_difficulty(p) + # Same precedence as conception: what the author already stated wins. + declared, _spans = _declared_fields(text) + level = declared.get("difficulty", level) return { "type": work_type, "target": REPO_DISPLAY.get(tgt, target if target != "-" else "?"), "difficulty": level, - "autonomy": infer_autonomy(level, factors), - "priority": infer_priority(text), + "autonomy": declared.get("autonomy") or infer_autonomy(level, factors), + "priority": declared.get("priority") or infer_priority(text), "status": "formalised", } @@ -2218,9 +2314,14 @@ def emit_human(d: dict): print(f"Work-type: {d['work_type']} (confidence: {d['classification_confidence']})") print(f"Target: {d['target_display']}") print(f"Repos resolved: {', '.join(d['repos_affected']) or '(none)'}") - print(f"Difficulty: {d['difficulty']} (score {d['difficulty_score']})") - print(f"Autonomy: {d['autonomy']}") - print(f"Priority: {d['priority']}") + if d.get("difficulty_source") == "declared": + print(f"Difficulty: {d['difficulty']} (declared; heuristic said " + f"{d['difficulty_estimated']}, score {d['difficulty_score']})") + else: + print(f"Difficulty: {d['difficulty']} (score {d['difficulty_score']})") + for field in ("autonomy", "priority"): + mark = " (declared)" if d.get(f"{field}_source") == "declared" else "" + print(f"{field.capitalize() + ':':<22}{d[field]}{mark}") print(f"Workflow: {d['workflow']}") print(f"Proposed path: {d['proposed_path']}") print("Header to be written:") diff --git a/agents/faculties/sizing/AGENTS.md b/agents/faculties/sizing/AGENTS.md index 00a61c78..120fbec3 100644 --- a/agents/faculties/sizing/AGENTS.md +++ b/agents/faculties/sizing/AGENTS.md @@ -16,7 +16,9 @@ recomputed by a divergent copy. - The **Intake Agent** (`agents/conductors/intake/`) sizes a task at *conception* time and persists the estimate into the prompt's `Difficulty:` - header — the number you see up front. + header — the number you see up front, unless the author declared their own in + the raw text (that override lives in intake, which owns the header; this + faculty stays a pure estimator). - The **Feature Agent** (`agents/conductors/feature/`) sizes a task at *selection / planning* time and acts on that same estimate. diff --git a/agents/faculties/sizing/_sizing.py b/agents/faculties/sizing/_sizing.py index fea4b0c8..11f27893 100755 --- a/agents/faculties/sizing/_sizing.py +++ b/agents/faculties/sizing/_sizing.py @@ -10,7 +10,9 @@ - the **Intake Agent** (`agents/conductors/intake/`) sizes a task at *conception* time and persists the estimate into the prompt's `Difficulty:` header, so the number you see up front is the same one the Feature Agent - acts on later. + acts on later. (Intake owns the header, so a difficulty the author declared + in the raw text overrides this estimate *there* — never in here: this + module stays the pure estimator.) Keeping the heuristic here — one definition, imported by both — is the whole point: a value Intake persists that the Feature Agent silently recomputed with a diff --git a/skills/intake/intake.md b/skills/intake/intake.md index 5de367c6..924b87f5 100644 --- a/skills/intake/intake.md +++ b/skills/intake/intake.md @@ -15,7 +15,9 @@ Shared routing context: `PyAutoBrain/skills/COMMANDS.md`. write. Nothing is created. 2. Review the `IntakeDecision` with the user — correct the work-type, target, difficulty, autonomy or priority in conversation (this is the back-and-forth - where scope, and therefore difficulty, is decided). + where scope, and therefore difficulty, is decided). A difficulty/autonomy/ + priority the raw text already declares is taken as written — the decision + marks it `(declared)` and shows what the heuristic would have estimated. 3. When it looks right, re-run with **`--apply`** to write the prompt file into PyAutoMind under `draft///.md` (or `draft/triage/` if the classification is genuinely unclear). diff --git a/tests/test_intake_declared_difficulty.py b/tests/test_intake_declared_difficulty.py new file mode 100644 index 00000000..74fbc85e --- /dev/null +++ b/tests/test_intake_declared_difficulty.py @@ -0,0 +1,93 @@ +"""tests/test_intake_declared_difficulty.py — a human declaration outranks the estimate. + +The Intake Agent sizes every idea with the shared sizing heuristic, but raw +input often states its own scope already (the ideas.md house style ends a +bullet with "Difficulty large, supervised."). Before this, that declaration was +silently overwritten by the keyword estimate — and, being prose, was also +swallowed by the derived title and so leaked into the filename. + +Locked here: the declaration wins, its provenance is reported, code spans are +not read as declarations, and ordinary prose still sizes by heuristic. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "agents" / "conductors" / "intake")) +from _intake import ( # noqa: E402 + _declared_fields, _derive_fields, _strip_declarations, analyse, +) + + +def test_declared_difficulty_beats_the_heuristic(): + d = analyse("Fix the typo in the @PyAutoHands docstring for Mask2D. " + "Difficulty: large.", "test") + assert d["difficulty"] == "large" + assert d["difficulty_source"] == "declared" + assert d["difficulty_estimated"] == "small" + assert "Difficulty: large" in d["header"] + + +def test_declaration_is_kept_out_of_the_title_and_slug(): + d = analyse("Fix the typo in the @PyAutoHands docstring for Mask2D. " + "Difficulty: large. Autonomy: supervised.", "test") + assert d["title"] == "Fix the typo in the @PyAutoHands docstring for Mask2D" + assert "large" not in d["proposed_path"] + assert "supervised" not in d["proposed_path"] + + +def test_ideas_house_style_trailing_autonomy(): + # `ideas.md` bullets end "Difficulty large, supervised." — one clause, two fields. + fields, spans = _declared_fields("Rework the grid. Difficulty large, supervised.") + assert fields == {"difficulty": "large", "autonomy": "supervised"} + assert spans + + +def test_declared_autonomy_and_priority_are_taken_as_written(): + d = analyse("Rework the @PyAutoHands grid API. Autonomy: human-required. " + "Priority: high.", "test") + assert d["autonomy"] == "human-required" + assert d["autonomy_source"] == "declared" + assert d["priority"] == "high" + assert d["priority_source"] == "declared" + + +def test_spelling_variants_normalise_to_header_values(): + fields, _ = _declared_fields("Difficulty is Too Large, human required.") + assert fields == {"difficulty": "too-large", "autonomy": "human-required"} + + +def test_code_spans_are_not_declarations(): + text = ("Intake drops the declaration. Repro:\n\n" + "```\nintake \"a task. Difficulty: large.\"\n```\n\n" + "Observed `Difficulty: too-large` in the header.\n") + fields, _ = _declared_fields(text) + assert fields == {} + + +def test_prose_without_a_declaration_still_sizes_by_heuristic(): + d = analyse("The @PyAutoHands mask is large and the difficulty of lensing " + "is well known.", "test") + assert d["difficulty_source"] == "estimated" + assert d["autonomy_source"] == "inferred" + assert d["priority_source"] == "inferred" + + +def test_strip_declarations_leaves_prose_alone_when_nothing_declared(): + text = "Just a sentence." + assert _strip_declarations(text, []) == text + + +def test_declaration_only_text_keeps_a_title_to_derive_from(): + d = analyse("Difficulty: large.", "test") + assert d["title"] != "Untitled" + + +def test_formalise_derives_the_declared_difficulty(): + body = ("# A backlog prompt\n\nRework the grid in @PyAutoHands.\n" + "Difficulty large, supervised.\n") + fields = _derive_fields(body, "refactor", "autoarray") + assert fields["difficulty"] == "large" + assert fields["autonomy"] == "supervised" From 787ffccfd53f35e16762cec3e034de5948f42945 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 22:36:22 +0000 Subject: [PATCH 2/2] fix: reconcile declared vs derived in the sizing faculty (#274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third instance of one family: a conductor derives a value from prose and ignores the header key the author declared. (1) the Feature Agent's ranker, fixed #217; (2) the Bug Agent; (3) the Intake Agent's Difficulty:. Intake never read a declared Difficulty at all. It hand-built the dict it passes the sizing faculty, omitting the declared_header() keys the file-loading path folds in, so estimate_difficulty ran with no declared key present, nothing reconciled derived against declared, and the derived level was persisted into the header — silently replacing what the input asked for. Reported case: raw text declaring `Difficulty: medium` filed as `too-large` (score 11), unchanged when the full header block was prepended, hand-corrected after --apply. The heuristic's biggest input is length, and length is a bad size proxy — a prompt is long when it carries a design, which is exactly what declaring a level exists to correct. Per the prompt, the precedence rule moves INTO the faculty both conductors already call rather than being re-implemented per conductor — three reconciliations were three chances to forget one, and two forgot: - `_sizing.effective_difficulty` is now the single rule: declared wins, derived level + score returned alongside so a disagreement is reported, not silently resolved. `_feature` drops its local copy and imports it; `_bug` and `_intake` now go through it. - `declared_header` reads `Type:` and `Autonomy:` too (it already read Difficulty/Priority/Status/Blocked-by), and `declared_inline` is its counterpart for unheadered conception prose — the ideas.md idiom "Difficulty large, supervised." included. Neither reads a fenced block: a prompt quoting a header documents it, rather than declaring it. - Intake reads both, keeps the declaration out of the derived title/slug (it was leaking into the filename), and honours a declared `Type:` over its prose classifier — the filing-note leg, where the prompt naming itself a bug throughout was classified `feature`. - The Bug Agent no longer re-homes a prompt that declares `Type: bug` on prose keywords: a defect report legitimately says "refactor" or "documentation" while describing a crash. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012RXdgVSW8bUhVdEXEz6AAJ --- agents/conductors/bug/BUG_TAXONOMY.md | 10 +- agents/conductors/bug/_bug.py | 26 ++- agents/conductors/feature/_feature.py | 16 +- agents/conductors/intake/AGENTS.md | 18 ++- agents/conductors/intake/INTAKE_TAXONOMY.md | 6 +- agents/conductors/intake/_intake.py | 134 +++++----------- agents/faculties/sizing/AGENTS.md | 19 ++- agents/faculties/sizing/_sizing.py | 127 ++++++++++++++- skills/intake/intake.md | 7 +- tests/test_declared_header_precedence.py | 169 ++++++++++++++++++++ tests/test_intake_declared_difficulty.py | 93 ----------- 11 files changed, 398 insertions(+), 227 deletions(-) create mode 100644 tests/test_declared_header_precedence.py delete mode 100644 tests/test_intake_declared_difficulty.py diff --git a/agents/conductors/bug/BUG_TAXONOMY.md b/agents/conductors/bug/BUG_TAXONOMY.md index 04e54de3..c5c4b970 100644 --- a/agents/conductors/bug/BUG_TAXONOMY.md +++ b/agents/conductors/bug/BUG_TAXONOMY.md @@ -86,9 +86,13 @@ or acknowledged YELLOW), never a check re-run here. ## Difficulty & selection (reused, severity-weighted) -Difficulty is the Feature Agent's heuristic verbatim (`F.estimate_difficulty` — repos -affected, prompt size, scientific complexity, architectural risk, test burden, memory -context; thresholds `≤2 small · ≤5 medium · ≤9 large · >9 too-large`). Selection scans +Difficulty is the sizing faculty's heuristic verbatim, read through its precedence +rule (`F.effective_difficulty` — repos affected, prompt size, scientific complexity, +architectural risk, test burden, memory context; thresholds `≤2 small · ≤5 medium · +≤9 large · >9 too-large`), so a `Difficulty:` the prompt DECLARES wins and the derived +level is reported alongside it. A declared `Type: bug` likewise blocks a prose-keyword +re-home suggestion: the author already answered that question, and a defect report +legitimately says "refactor" or "documentation" while describing a crash. Selection scans `bug/**` (excluding `README.md`), down-ranks paths referenced in `active.md` / `planned.md`, and ranks **severity-first** by default and under `--impact` (a bug list is a triage queue); `--difficulty easy` / `--budget` / `--model weak` flip to smallest-first diff --git a/agents/conductors/bug/_bug.py b/agents/conductors/bug/_bug.py index 0aafd067..7c4f5c5e 100755 --- a/agents/conductors/bug/_bug.py +++ b/agents/conductors/bug/_bug.py @@ -242,10 +242,19 @@ def health_validation(workflow: str, factors: dict) -> str: def re_home_check(p: dict) -> str | None: - """If a bug/ prompt is really a feature/refactor/docs/research task, say so.""" + """If a bug/ prompt is really a feature/refactor/docs/research task, say so. + + A prompt that DECLARES `Type: bug` is never re-homed on prose keywords: the + author already answered this question, and prose is the weaker signal (a bug + report legitimately says "refactor", "documentation" or "design decision" + while describing a defect). Same family as the difficulty leg — derive from + prose, but never over a declared header key (PyAutoBrain#217, #274). + """ wt = p["work_type"] if wt != "bug": return None + if (p.get("declared_type") or "") == "bug": + return None text = p["text"].lower() # A genuine defect signal keeps it a bug even if other words also fire. defect = F._hits(text, ["bug", "regression", "crash", "traceback", "wrong", @@ -264,7 +273,7 @@ def re_home_check(p: dict) -> str | None: def analyse_bug(p: dict, heart_check: str | None = None) -> dict: - level, score, factors = F.estimate_difficulty(p) + level, score, factors, derived_level = F.effective_difficulty(p) cls = classify(p, factors) workflow = recommended_workflow(p, factors) mishome = re_home_check(p) @@ -283,6 +292,11 @@ def analyse_bug(p: dict, heart_check: str | None = None) -> dict: "recommended_workflow": workflow, "rehome_suggestion": mishome, "difficulty": level, + "difficulty_declared": p.get("declared_difficulty"), + "difficulty_derived": derived_level, + "difficulty_disagreement": ( + p.get("declared_difficulty") is not None and derived_level != level + ), "difficulty_score": score, "difficulty_factors": factors, "health_validation": health_validation(workflow, factors), @@ -329,7 +343,7 @@ def select_bug(mind: Path, constraint: dict, limit: int): rows = [] for path in prompts: p = F.parse_prompt(path, mind) - level, score, factors = F.estimate_difficulty(p) + level, score, factors, _derived = F.effective_difficulty(p) cls = classify(p, factors) rows.append({ "path": p["path"], "difficulty": level, "score": score, @@ -385,7 +399,11 @@ def emit_human(mode: str, d: dict): print(f"Fix strategy: {d['fix_strategy']}") print(f"Recommended workflow: {d['recommended_workflow']}", end="") print(f" [re-home as {d['rehome_suggestion']}/]" if d["rehome_suggestion"] else "") - print(f"Difficulty: {d['difficulty']} (score {d['difficulty_score']})") + if d.get("difficulty_disagreement"): + print(f"Difficulty: {d['difficulty']} (declared; heuristic derived " + f"{d['difficulty_derived']}, score {d['difficulty_score']})") + else: + print(f"Difficulty: {d['difficulty']} (score {d['difficulty_score']})") print(f"Health validation: {d['health_validation']}") print("Risks:") for r in d["risks"]: diff --git a/agents/conductors/feature/_feature.py b/agents/conductors/feature/_feature.py index c15e5fe3..619bbb5b 100755 --- a/agents/conductors/feature/_feature.py +++ b/agents/conductors/feature/_feature.py @@ -35,7 +35,7 @@ KNOWN_REPOS, MEMORY_WIKIS, SCIENCE_KEYWORDS, RISK_KEYWORDS, AMBIGUITY_KEYWORDS, policy as _sizing_policy, TEST_KEYWORDS, normalise_repo, parse_prompt, discover_prompts, - empty_discovery_reason, estimate_difficulty, _hits, _within, + effective_difficulty, empty_discovery_reason, _hits, _within, declared_blocked, priority_rank, ) @@ -159,20 +159,6 @@ def risks(level: str, factors: dict, workflow: str): return out -def effective_difficulty(p: dict): - """(level, score, factors, derived_level) — the DECLARED level wins. - - REFERENCE.md promises that the `Difficulty:` Intake persists is "the value - the Feature Agent later acts on", so a declared level overrides the - re-derived one. The derived score is kept: it still orders prompts within a - level, and the derived LEVEL is returned alongside so a disagreement can be - reported rather than silently resolved — the disagreement is evidence about - the heuristic and is worth seeing. - """ - derived_level, score, factors = estimate_difficulty(p) - return p.get("declared_difficulty") or derived_level, score, factors, derived_level - - def analyse(p: dict): level, score, factors, derived_level = effective_difficulty(p) workflow, rehome = recommend_workflow(p, factors) diff --git a/agents/conductors/intake/AGENTS.md b/agents/conductors/intake/AGENTS.md index 8f048b0e..2401a8e2 100644 --- a/agents/conductors/intake/AGENTS.md +++ b/agents/conductors/intake/AGENTS.md @@ -55,15 +55,17 @@ Proposed path · Header (the block written verbatim) · Risks · Next action - **Difficulty** — from the shared `agents/faculties/sizing/` faculty, **persisted into the header**, because scope is decided during the intake back-and-forth. The Feature Agent later *trusts* this number rather than recomputing a divergent - one. A difficulty **declared in the raw text** (`Difficulty: large`, or the - `ideas.md` house style `Difficulty large, supervised.`) **wins** over the - estimate: the author knows the scope, the heuristic only sees words. The - IntakeDecision reports which one it used (`difficulty_source`) and what the - heuristic would have said, and the declaration is kept out of the derived - title/slug. Declarations are read from prose only — a quoted `Difficulty:` - inside a code fence or backticks is not one. + one. A difficulty **declared in the raw input** — a pasted header block, or the + `ideas.md` house style `Difficulty large, supervised.` — **wins** over the + estimate, via the faculty's shared `effective_difficulty`: the author knows the + scope, and the heuristic's biggest input is length, which is a bad size proxy + (a prompt is long when it carries a design, not when the work is large). The + IntakeDecision reports which one it used (`difficulty_source`) and the level + the heuristic derived, and the declaration is kept out of the derived + title/slug. A `Difficulty:` quoted inside a code fence or backticks is + documentation, not a declaration. - **Autonomy** (`safe|supervised|human-required`) and **Priority** - (`low|normal|high`) — the "can an agent safely handle this?" and "how urgent?" questions; a declared value wins here too. + (`low|normal|high`) — the "can an agent safely handle this?" and "how urgent?" questions; a declared value wins here too, as does a declared `Type:` over the prose classifier. inputs, written into the header. **`too-large` means decompose, never dispatch.** A prompt sized `too-large` is diff --git a/agents/conductors/intake/INTAKE_TAXONOMY.md b/agents/conductors/intake/INTAKE_TAXONOMY.md index f117bd43..48321a58 100644 --- a/agents/conductors/intake/INTAKE_TAXONOMY.md +++ b/agents/conductors/intake/INTAKE_TAXONOMY.md @@ -63,8 +63,10 @@ Primary-folder preference when several resolve: library repo → organism repo ## 4. Difficulty, autonomy, priority (the header inputs) - **Difficulty** (`small|medium|large|too-large`) — the shared sizing faculty's - `estimate_difficulty`, the *same* function the Feature Agent uses. Persisted so - the number is decided once, at conception, when scope is freshest. + `effective_difficulty`: `estimate_difficulty` (the *same* heuristic the Feature + Agent uses) unless the input DECLARES a level, which wins. Persisted so the + number is decided once, at conception, when scope is freshest — and a + declaration is exactly the author correcting a heuristic that weights length. - **Autonomy** (`safe|supervised|human-required`) — `human-required` when the work needs a design decision and no repo resolved; `supervised` for architectural risk / large / multi-repo; `safe` otherwise. diff --git a/agents/conductors/intake/_intake.py b/agents/conductors/intake/_intake.py index 47c041e8..a3d16224 100755 --- a/agents/conductors/intake/_intake.py +++ b/agents/conductors/intake/_intake.py @@ -39,7 +39,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "faculties" / "sizing")) from _sizing import ( # noqa: E402 WORK_TYPES, LIBRARY_REPOS, WORKSPACE_REPOS, ORGANISM_REPOS, KNOWN_REPOS, - RISK_KEYWORDS, AMBIGUITY_KEYWORDS, normalise_repo, estimate_difficulty, _hits, + RISK_KEYWORDS, AMBIGUITY_KEYWORDS, normalise_repo, declared_header, + declared_inline, effective_difficulty, strip_declarations, _hits, policy as _sizing_policy, BODY_MAP_PATH, ) @@ -110,80 +111,6 @@ "when there is time", "backlog"] -# --- human-declared header fields --------------------------------------------- -# Raw input often states its own scope: the ideas.md house style ends a bullet -# with "Difficulty large, supervised.", and a pasted bug report may carry a -# "Priority: high" line. A human who has thought about the work is better -# evidence than a keyword heuristic that only sees words, so a declaration -# WINS over the estimate here. The sizing faculty stays the pure estimator — -# the override belongs to the agent that owns the header (this one). -# -# Declarations are read only from prose: fenced blocks and inline code spans are -# masked out first, so a prompt *quoting* `Difficulty: large` (a repro, a -# transcript) does not silently re-size itself. -_DIFFICULTY_ALT = r"too[-\s]large|small|medium|large" -_AUTONOMY_ALT = r"human[-\s]required|supervised|safe" -_PRIORITY_ALT = r"high|normal|low" -# Between key and value: a colon/equals, "is", or nothing at all ("Difficulty large"). -_DECL_SEP = r"\s*(?::|=|\bis\b)?\s*" -_DECLARATION = re.compile( - rf"\bdifficulty{_DECL_SEP}({_DIFFICULTY_ALT})\b" - rf"(?:\s*[,/&]?\s*(?:and\s+)?({_AUTONOMY_ALT})\b)?" - rf"|\bautonomy{_DECL_SEP}({_AUTONOMY_ALT})\b" - rf"|\bpriority{_DECL_SEP}({_PRIORITY_ALT})\b", - re.IGNORECASE) -_CODE_SPAN = re.compile(r"```.*?```|`[^`\n]*`", re.DOTALL) - - -def _mask_code(text: str) -> str: - """`text` with code spans blanked to spaces (offsets and lines preserved).""" - return _CODE_SPAN.sub(lambda m: re.sub(r"\S", " ", m.group(0)), text) - - -def _declared_fields(text: str): - """(fields, spans) — the header fields the human stated in the raw text. - - First declaration of each field wins; `spans` are its offsets in `text` so - the clause can be kept out of the derived title/slug. - """ - fields, spans = {}, [] - for m in _DECLARATION.finditer(_mask_code(text)): - difficulty, trailing_autonomy, autonomy, priority = m.groups() - if difficulty: - fields.setdefault("difficulty", _norm_level(difficulty)) - if trailing_autonomy or autonomy: - fields.setdefault("autonomy", _norm_level(trailing_autonomy or autonomy)) - if priority: - fields.setdefault("priority", _norm_level(priority)) - spans.append(m.span()) - return fields, spans - - -def _norm_level(value: str) -> str: - """"Too Large" / "human required" -> the canonical hyphenated header value.""" - return re.sub(r"[\s-]+", "-", value.strip().lower()) - - -def _strip_declarations(text: str, spans: list) -> str: - """`text` with the declaration clauses removed — for title derivation only. - - The prompt body itself stays verbatim (word-vomit is intent); this exists so - "Fix the docstring. Difficulty: large." does not title the task — and name - the file — after its own difficulty declaration. - """ - if not spans: - return text - chars = list(text) - for start, end in spans: - chars[start:end] = " " * (end - start) - out = "".join(chars) - # Tidy the punctuation the removed clause left stranded (title use only). - out = re.sub(r"[ \t]+", " ", out) - out = re.sub(r"\s+([.,;:])", r"\1", out) - out = re.sub(r"([.,;:])(\s*[.,;:])+", r"\1", out) - return out if re.search(r"\w", out) else text - - def _slug(text: str, maxwords: int = 7) -> str: words = re.findall(r"[A-Za-z0-9]+", text.lower()) slug = "_".join(words[:maxwords]) @@ -338,23 +265,35 @@ def infer_autonomy(level: str, factors: dict) -> str: def analyse(text: str, source: str): """Classify raw text into a full IntakeDecision (never writes).""" repos = _repos_in(text) + # What the input DECLARES outranks what its prose merely suggests — the same + # rule the feature and bug conductors apply (the faculty owns it). Raw + # conception input may carry a full header block (a pasted prompt) or state + # a key mid-sentence, so both readers run; header lines win a tie. + inline, decl_spans = declared_inline(text) + header = declared_header(text) + declared = {k: v for k, v in { + "difficulty": header["declared_difficulty"] or inline.get("difficulty"), + "autonomy": header["declared_autonomy"] or inline.get("autonomy"), + "priority": header["priority"] or inline.get("priority"), + "type": header["declared_type"] or inline.get("type"), + }.items() if v} + work_type, confidence, type_hits = classify_work_type(text) + if declared.get("type"): + work_type, confidence = declared["type"], "high" target, target_display, repos = infer_target(text, repos) # Build a prompt-shaped dict the shared sizing faculty understands. p = {"text": text, "repos": repos, "words": len(text.split()), - "target": target, "work_type": work_type} - estimated, score, factors = estimate_difficulty(p) + "target": target, "work_type": work_type, + "declared_difficulty": declared.get("difficulty")} + level, score, factors, estimated = effective_difficulty(p) - # A human declaration outranks the estimate (see "human-declared header - # fields"); the estimate is kept alongside it so the review can see both. - declared, decl_spans = _declared_fields(text) - level = declared.get("difficulty", estimated) autonomy = declared.get("autonomy") or infer_autonomy(level, factors) priority = declared.get("priority") or infer_priority(text) workflow = infer_workflow(target, repos) - title = _title(_strip_declarations(text, decl_spans)) + title = _title(strip_declarations(text, decl_spans)) slug = _slug(title) folder = work_type if confidence != "low" else "triage" if folder == "triage": @@ -375,6 +314,7 @@ def analyse(text: str, source: str): "title": title, "work_type": work_type, "classification_confidence": confidence, + "work_type_source": "declared" if declared.get("type") else "inferred", "type_signals": type_hits, "target": target, "target_display": target_display, @@ -382,7 +322,11 @@ def analyse(text: str, source: str): "difficulty": level, "difficulty_score": score, "difficulty_factors": factors, - "difficulty_estimated": estimated, + "difficulty_declared": declared.get("difficulty"), + "difficulty_derived": estimated, + "difficulty_disagreement": ( + "difficulty" in declared and estimated != level + ), "difficulty_source": "declared" if "difficulty" in declared else "estimated", "autonomy": autonomy, "autonomy_source": "declared" if "autonomy" in declared else "inferred", @@ -1638,12 +1582,18 @@ def _derive_fields(text: str, work_type: str, target: str) -> dict: tgt = normalise_repo(target) if target != "-" else "?" if tgt in KNOWN_REPOS and tgt not in repos: repos = sorted(set(repos) | {tgt}) - p = {"text": text, "repos": repos, "words": len(text.split()), - "target": target, "work_type": work_type} - level, _score, factors = estimate_difficulty(p) # Same precedence as conception: what the author already stated wins. - declared, _spans = _declared_fields(text) - level = declared.get("difficulty", level) + inline, _spans = declared_inline(text) + header = declared_header(text) + declared = {k: v for k, v in { + "difficulty": header["declared_difficulty"] or inline.get("difficulty"), + "autonomy": header["declared_autonomy"] or inline.get("autonomy"), + "priority": header["priority"] or inline.get("priority"), + }.items() if v} + p = {"text": text, "repos": repos, "words": len(text.split()), + "target": target, "work_type": work_type, + "declared_difficulty": declared.get("difficulty")} + level, _score, factors, _derived = effective_difficulty(p) return { "type": work_type, "target": REPO_DISPLAY.get(tgt, target if target != "-" else "?"), @@ -2311,12 +2261,14 @@ def emit_human(d: dict): print("== IntakeDecision ==") print(f"Source: {d['source']}") print(f"Title: {d['title']}") - print(f"Work-type: {d['work_type']} (confidence: {d['classification_confidence']})") + wt_mark = ("declared" if d.get("work_type_source") == "declared" + else f"confidence: {d['classification_confidence']}") + print(f"Work-type: {d['work_type']} ({wt_mark})") print(f"Target: {d['target_display']}") print(f"Repos resolved: {', '.join(d['repos_affected']) or '(none)'}") if d.get("difficulty_source") == "declared": - print(f"Difficulty: {d['difficulty']} (declared; heuristic said " - f"{d['difficulty_estimated']}, score {d['difficulty_score']})") + print(f"Difficulty: {d['difficulty']} (declared; heuristic derived " + f"{d['difficulty_derived']}, score {d['difficulty_score']})") else: print(f"Difficulty: {d['difficulty']} (score {d['difficulty_score']})") for field in ("autonomy", "priority"): diff --git a/agents/faculties/sizing/AGENTS.md b/agents/faculties/sizing/AGENTS.md index 120fbec3..3ff55e18 100644 --- a/agents/faculties/sizing/AGENTS.md +++ b/agents/faculties/sizing/AGENTS.md @@ -12,13 +12,26 @@ A PyAutoBrain read-only reasoning faculty. It owns the organism's single difficulty heuristic so the number is defined **once** and shared, never recomputed by a divergent copy. +It also owns the **precedence rule** over that heuristic. `estimate_difficulty` +derives; `effective_difficulty` reconciles — a **declared** `Difficulty:` wins, +and the derived level comes back alongside it so a disagreement is reported +rather than silently resolved (the disagreement is evidence about the +heuristic). `declared_header` reads the keys a filed prompt declares +(`Type:`/`Difficulty:`/`Autonomy:`/`Priority:`/`Status:`/`Blocked-by:`); +`declared_inline` reads the same keys out of unheadered conception prose — the +`ideas.md` idiom `Difficulty large, supervised.` included. Neither reads a +fenced block: a prompt quoting a header is documenting it, not declaring it. + +The rule lives here rather than in each conductor because it was re-implemented +per conductor three times and forgotten twice — the ranker (#217), the Bug Agent +and Intake (#274) all derived over a declared key. One heuristic, one +reconciliation, every conductor. + ## Who consults it - The **Intake Agent** (`agents/conductors/intake/`) sizes a task at *conception* time and persists the estimate into the prompt's `Difficulty:` - header — the number you see up front, unless the author declared their own in - the raw text (that override lives in intake, which owns the header; this - faculty stays a pure estimator). + header — the number you see up front. - The **Feature Agent** (`agents/conductors/feature/`) sizes a task at *selection / planning* time and acts on that same estimate. diff --git a/agents/faculties/sizing/_sizing.py b/agents/faculties/sizing/_sizing.py index 11f27893..f41a056e 100755 --- a/agents/faculties/sizing/_sizing.py +++ b/agents/faculties/sizing/_sizing.py @@ -10,9 +10,16 @@ - the **Intake Agent** (`agents/conductors/intake/`) sizes a task at *conception* time and persists the estimate into the prompt's `Difficulty:` header, so the number you see up front is the same one the Feature Agent - acts on later. (Intake owns the header, so a difficulty the author declared - in the raw text overrides this estimate *there* — never in here: this - module stays the pure estimator.) + acts on later. + +`estimate_difficulty` is the heuristic; `effective_difficulty` is the PRECEDENCE +RULE over it — a difficulty the author DECLARED wins, with the derived level +returned alongside so a disagreement stays visible. The rule lives here, not in +the conductors: each conductor that reconciled it for itself was one more chance +to forget, and three of them did (PyAutoBrain#217, then #274). `declared_header` +reads the keys a filed prompt declares; `declared_inline` reads the same keys out +of unheadered conception prose. Neither reads a code fence — a prompt QUOTING a +header is documenting it, not declaring it. Keeping the heuristic here — one definition, imported by both — is the whole point: a value Intake persists that the Feature Agent silently recomputed with a @@ -234,13 +241,15 @@ def empty_discovery_reason(mind: Path, work_type: str) -> str: # Parsing them here — beside the derivation — keeps declared and derived in one # place, and gives the bug/refactor conductors the same reading for free. DIFFICULTY_LEVELS = ("small", "medium", "large", "too-large") +AUTONOMY_LEVELS = ("safe", "supervised", "human-required") # `medium` is not a documented Priority: value but occurs in the live backlog; # read it as normal rather than dropping the prompt's stated intent. PRIORITY_RANK = {"high": 0, "normal": 1, "medium": 1, "low": 2} DEFAULT_PRIORITY_RANK = 1 _HEADER_KEY_RE = re.compile( - r"^\s*(difficulty|status|priority|blocked-by|closes-when)\s*:\s*(.+?)\s*$", re.I + r"^\s*(difficulty|type|autonomy|status|priority|blocked-by|closes-when)" + r"\s*:\s*(.+?)\s*$", re.I ) @@ -259,7 +268,8 @@ def declared_header(text: str) -> dict: does exactly that) must not be read as declaring it. Same rule, and the same reason, as PyAutoMind `lifecycle.py:draft_gate_refs`. """ - out = {"declared_difficulty": None, "status": None, + out = {"declared_difficulty": None, "declared_type": None, + "declared_autonomy": None, "status": None, "priority": None, "blocked_by": [], "closes_when": []} in_fence = False for line in text.splitlines(): @@ -278,6 +288,14 @@ def declared_header(text: str) -> dict: v = value.lower() if v in DIFFICULTY_LEVELS and out["declared_difficulty"] is None: out["declared_difficulty"] = v + elif key == "type": + v = value.lower() + if v in WORK_TYPES and out["declared_type"] is None: + out["declared_type"] = v + elif key == "autonomy": + v = _norm_level(value) + if v in AUTONOMY_LEVELS and out["declared_autonomy"] is None: + out["declared_autonomy"] = v elif key == "status" and out["status"] is None: out["status"] = value.lower() elif key == "priority" and out["priority"] is None: @@ -289,6 +307,83 @@ def declared_header(text: str) -> dict: return out +# --- declarations in unstructured prose --------------------------------------- +# `declared_header` reads header LINES, which is what a filed prompt carries. +# Conception input has no header yet: the ideas.md house style ends a bullet +# with "Difficulty large, supervised.", and a pasted report writes +# "… Difficulty: medium." mid-sentence. Same precedence, a looser reader — kept +# here beside the header reader so "what counts as a declaration" is defined +# once for every conductor. +_DIFFICULTY_ALT = r"too[-\s]large|small|medium|large" +_AUTONOMY_ALT = r"human[-\s]required|supervised|safe" +_PRIORITY_ALT = r"high|normal|low" +_TYPE_ALT = "|".join(sorted(WORK_TYPES, key=len, reverse=True)) +# Between key and value: a colon/equals, "is", or nothing ("Difficulty large"). +_DECL_SEP = r"\s*(?::|=|\bis\b)?\s*" +_DECLARATION = re.compile( + rf"\bdifficulty{_DECL_SEP}({_DIFFICULTY_ALT})\b" + rf"(?:\s*[,/&]?\s*(?:and\s+)?({_AUTONOMY_ALT})\b)?" + rf"|\bautonomy{_DECL_SEP}({_AUTONOMY_ALT})\b" + rf"|\bpriority{_DECL_SEP}({_PRIORITY_ALT})\b" + rf"|\btype{_DECL_SEP}({_TYPE_ALT})\b", + re.IGNORECASE) +_CODE_SPAN = re.compile(r"```.*?```|`[^`\n]*`", re.DOTALL) + + +def _mask_code(text: str) -> str: + """`text` with code spans blanked to spaces (offsets and lines preserved).""" + return _CODE_SPAN.sub(lambda m: re.sub(r"\S", " ", m.group(0)), text) + + +def _norm_level(value: str) -> str: + """"Too Large" / "human required" -> the canonical hyphenated header value.""" + return re.sub(r"[\s-]+", "-", value.strip().lower()) + + +def declared_inline(text: str): + """(fields, spans) — declarations made in prose, for unheadered raw input. + + Fenced blocks and inline code spans are masked first: a prompt that *quotes* + a `Difficulty:` line (a repro, a transcript — the bug prompt for this very + fix does exactly that) is documenting, not declaring. Same rule as + `declared_header`. First declaration of each key wins; `spans` are its + offsets in `text`, so a caller can keep the clause out of a derived title. + """ + fields, spans = {}, [] + for m in _DECLARATION.finditer(_mask_code(text)): + difficulty, trailing_autonomy, autonomy, priority, work_type = m.groups() + if difficulty: + fields.setdefault("difficulty", _norm_level(difficulty)) + if trailing_autonomy or autonomy: + fields.setdefault("autonomy", _norm_level(trailing_autonomy or autonomy)) + if priority: + fields.setdefault("priority", _norm_level(priority)) + if work_type: + fields.setdefault("type", work_type.lower()) + spans.append(m.span()) + return fields, spans + + +def strip_declarations(text: str, spans: list) -> str: + """`text` with the declaration clauses removed — for title derivation only. + + The prompt body itself stays verbatim (word-vomit is intent); this exists so + "Fix the docstring. Difficulty: large." does not title the task — and name + the file — after its own difficulty declaration. + """ + if not spans: + return text + chars = list(text) + for start, end in spans: + chars[start:end] = " " * (end - start) + out = "".join(chars) + # Tidy the punctuation the removed clause left stranded (title use only). + out = re.sub(r"[ \t]+", " ", out) + out = re.sub(r"\s+([.,;:])", r"\1", out) + out = re.sub(r"([.,;:])(\s*[.,;:])+", r"\1", out) + return out if re.search(r"\w", out) else text + + def priority_rank(p: dict) -> int: return PRIORITY_RANK.get(p.get("priority") or "", DEFAULT_PRIORITY_RANK) @@ -422,6 +517,28 @@ def estimate_difficulty(p: dict): return level, score, factors +def effective_difficulty(p: dict): + """(level, score, factors, derived_level) — the DECLARED level wins. + + The single precedence rule, defined here rather than per conductor. Three + conductors size a prompt (feature, bug, intake) and each one that re-derived + difficulty while ignoring `declared_difficulty` shipped the same bug + (PyAutoBrain#217, then #274) — one heuristic with three reconciliations is + three chances to forget one. + + REFERENCE.md promises that the `Difficulty:` Intake persists is "the value + the Feature Agent later acts on", so a declared level overrides the + re-derived one. Length is the heuristic's biggest input and a bad size proxy + — a prompt is long when it carries a design, not when the work is large — + which is exactly what declaring a level exists to correct. The derived score + is kept (it still orders prompts within a level) and the derived LEVEL is + returned alongside, so a disagreement is reported rather than silently + resolved: it is evidence about the heuristic and worth seeing. + """ + derived_level, score, factors = estimate_difficulty(p) + return p.get("declared_difficulty") or derived_level, score, factors, derived_level + + # --- runnable read-only entrypoint (parity with the other faculties) --------- # The heuristic above is a shared *substrate* imported by intake + feature; this # thin CLI lets a human (or `pyauto-brain sizing`) read the SizingSurface for one diff --git a/skills/intake/intake.md b/skills/intake/intake.md index 924b87f5..48fa217b 100644 --- a/skills/intake/intake.md +++ b/skills/intake/intake.md @@ -15,9 +15,10 @@ Shared routing context: `PyAutoBrain/skills/COMMANDS.md`. write. Nothing is created. 2. Review the `IntakeDecision` with the user — correct the work-type, target, difficulty, autonomy or priority in conversation (this is the back-and-forth - where scope, and therefore difficulty, is decided). A difficulty/autonomy/ - priority the raw text already declares is taken as written — the decision - marks it `(declared)` and shows what the heuristic would have estimated. + where scope, and therefore difficulty, is decided). A `Type:`/`Difficulty:`/ + `Autonomy:`/`Priority:` the raw input already declares is taken as written — + the decision marks it `(declared)` and shows the level the heuristic derived, + so a disagreement stays visible instead of being silently resolved. 3. When it looks right, re-run with **`--apply`** to write the prompt file into PyAutoMind under `draft///.md` (or `draft/triage/` if the classification is genuinely unclear). diff --git a/tests/test_declared_header_precedence.py b/tests/test_declared_header_precedence.py new file mode 100644 index 00000000..e1d93bec --- /dev/null +++ b/tests/test_declared_header_precedence.py @@ -0,0 +1,169 @@ +"""tests/test_declared_header_precedence.py — declared header keys outrank prose. + +One family of bug, three instances: a conductor derives a value from prose and +ignores the header key the author declared. (1) the Feature Agent's ranker +(fixed, PyAutoBrain#217); (2) the Bug Agent re-homing a prompt that declares +`Type: bug`; (3) the Intake Agent persisting its derived `Difficulty:` over the +declared one (PyAutoBrain#274). + +The precedence rule therefore lives in the sizing faculty — `effective_difficulty`, +the one place every conductor already calls — rather than being re-implemented +per conductor. Locked here: declared wins, the derived level is still reported so +a disagreement stays visible, declarations are read from prose *and* header lines +but never from code fences, and a declaration never leaks into the derived title. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "agents" / "faculties" / "sizing")) +sys.path.insert(0, str(ROOT / "agents" / "conductors" / "feature")) +sys.path.insert(0, str(ROOT / "agents" / "conductors" / "bug")) +sys.path.insert(0, str(ROOT / "agents" / "conductors" / "intake")) +from _sizing import ( # noqa: E402 + declared_header, declared_inline, effective_difficulty, estimate_difficulty, + strip_declarations, +) +import _bug # noqa: E402 +from _intake import _derive_fields, analyse # noqa: E402 + + +# --- the faculty: one precedence rule ---------------------------------------- + +def test_effective_difficulty_prefers_the_declared_level(): + p = {"text": "x " * 900, "repos": ["autofit", "autoarray"], "words": 900, + "target": "autofit", "work_type": "feature", + "declared_difficulty": "medium"} + level, _score, _factors, derived = effective_difficulty(p) + assert level == "medium" + assert derived == estimate_difficulty(p)[0] != "medium" + + +def test_effective_difficulty_falls_back_to_the_estimate(): + p = {"text": "a short one", "repos": [], "words": 3, "target": "?", + "work_type": "bug"} + level, _score, _factors, derived = effective_difficulty(p) + assert level == derived == "small" + + +def test_declared_inline_reads_the_ideas_house_style(): + # `ideas.md` bullets end "Difficulty large, supervised." — one clause, two keys. + fields, spans = declared_inline("Rework the grid. Difficulty large, supervised.") + assert fields == {"difficulty": "large", "autonomy": "supervised"} + assert spans + + +def test_declared_inline_normalises_spelling_variants(): + fields, _ = declared_inline("Difficulty is Too Large, human required.") + assert fields == {"difficulty": "too-large", "autonomy": "human-required"} + + +def test_code_spans_are_documentation_not_declarations(): + text = ("Intake drops the declaration. Repro:\n\n" + "```\nintake \"a task. Difficulty: large.\"\n```\n\n" + "Observed `Difficulty: too-large` in the header.\n") + assert declared_inline(text)[0] == {} + assert declared_header(text)["declared_difficulty"] is None + + +def test_declared_header_reads_type_and_autonomy(): + h = declared_header("Type: bug\nTarget: pyautobrain\nDifficulty: medium\n" + "Autonomy: safe\nPriority: high\n") + assert h["declared_type"] == "bug" + assert h["declared_difficulty"] == "medium" + assert h["declared_autonomy"] == "safe" + assert h["priority"] == "high" + + +def test_strip_declarations_is_a_no_op_without_declarations(): + assert strip_declarations("Just a sentence.", []) == "Just a sentence." + + +# --- intake: conception ------------------------------------------------------ + +def test_intake_persists_the_declared_difficulty_not_its_own(): + # The reported case: a long prompt (length drives the score) declaring medium. + text = ("Add an arXiv inbox tier to @PyAutoHands, @PyAutoHeart and " + "@PyAutoMind. It changes the public API and the architecture of the " + "release path, and needs new tests. " + "context " * 800 + + " Difficulty: medium.") + d = analyse(text, "test") + assert d["difficulty"] == "medium" + assert d["difficulty_derived"] == "too-large" + assert d["difficulty_disagreement"] is True + assert "Difficulty: medium" in d["header"] + + +def test_intake_honours_a_prepended_header_block(): + text = ("# A task\n\nType: bug\nTarget: pyautohands\nDifficulty: medium\n" + "Autonomy: supervised\nPriority: high\n\n" + "Something in @PyAutoHands is broken. " + "detail " * 800) + d = analyse(text, "test") + assert (d["difficulty"], d["autonomy"], d["priority"]) == ( + "medium", "supervised", "high") + + +def test_intake_declaration_stays_out_of_the_title_and_slug(): + d = analyse("Fix the typo in the @PyAutoHands docstring. " + "Difficulty: large. Autonomy: supervised.", "test") + assert d["title"] == "Fix the typo in the @PyAutoHands docstring" + assert "large" not in d["proposed_path"] + assert "supervised" not in d["proposed_path"] + + +def test_intake_declared_type_beats_prose_classification(): + # The filing note's case: prose full of feature verbs, `Type: bug` declared. + text = ("Type: bug\n\nAdd support for a new capability that implements a new " + "@PyAutoHands surface — except it is a crash report.") + assert analyse(text, "test")["work_type"] == "bug" + assert analyse(text, "test")["work_type_source"] == "declared" + + +def test_intake_without_declarations_still_derives(): + d = analyse("The @PyAutoHands mask is large and the difficulty of the " + "problem is well known.", "test") + assert d["difficulty_source"] == "estimated" + assert d["work_type_source"] == "inferred" + assert d["autonomy_source"] == "inferred" + + +def test_formalise_derives_the_declared_difficulty(): + body = ("# A backlog prompt\n\nRework the grid in @PyAutoHands.\n" + "Difficulty large, supervised.\n") + fields = _derive_fields(body, "refactor", "pyautohands") + assert fields["difficulty"] == "large" + assert fields["autonomy"] == "supervised" + + +# --- the bug conductor ------------------------------------------------------- + +def _bug_prompt(text: str, **extra): + p = {"path": "draft/bug/pyautohands/x.md", "work_type": "bug", + "target": "pyautohands", "repos": ["pyautohands"], "text": text, + "lines": text.count("\n") + 1, "words": len(text.split()), + "declared_difficulty": None, "declared_type": None, + "declared_autonomy": None, "status": None, "priority": None, + "blocked_by": [], "closes_when": []} + p.update(extra) + return p + + +def test_bug_agent_does_not_rehome_a_declared_bug_on_prose(): + # Prose a defect report legitimately contains; no "bug"/"crash" word in it, + # so the prose classifier would re-home it to docs/. + text = ("The documentation build emits a docstring typo — a design decision " + "is needed on the tutorial layout.") + assert _bug.re_home_check(_bug_prompt(text)) == "docs" + assert _bug.re_home_check(_bug_prompt(text, declared_type="bug")) is None + + +def test_bug_decision_reports_the_declared_difficulty(): + d = _bug.analyse_bug(_bug_prompt("It crashes. " + "detail " * 900, + declared_difficulty="small")) + assert d["difficulty"] == "small" + assert d["difficulty_declared"] == "small" + assert d["difficulty_derived"] != "small" + assert d["difficulty_disagreement"] is True diff --git a/tests/test_intake_declared_difficulty.py b/tests/test_intake_declared_difficulty.py deleted file mode 100644 index 74fbc85e..00000000 --- a/tests/test_intake_declared_difficulty.py +++ /dev/null @@ -1,93 +0,0 @@ -"""tests/test_intake_declared_difficulty.py — a human declaration outranks the estimate. - -The Intake Agent sizes every idea with the shared sizing heuristic, but raw -input often states its own scope already (the ideas.md house style ends a -bullet with "Difficulty large, supervised."). Before this, that declaration was -silently overwritten by the keyword estimate — and, being prose, was also -swallowed by the derived title and so leaked into the filename. - -Locked here: the declaration wins, its provenance is reported, code spans are -not read as declarations, and ordinary prose still sizes by heuristic. -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "agents" / "conductors" / "intake")) -from _intake import ( # noqa: E402 - _declared_fields, _derive_fields, _strip_declarations, analyse, -) - - -def test_declared_difficulty_beats_the_heuristic(): - d = analyse("Fix the typo in the @PyAutoHands docstring for Mask2D. " - "Difficulty: large.", "test") - assert d["difficulty"] == "large" - assert d["difficulty_source"] == "declared" - assert d["difficulty_estimated"] == "small" - assert "Difficulty: large" in d["header"] - - -def test_declaration_is_kept_out_of_the_title_and_slug(): - d = analyse("Fix the typo in the @PyAutoHands docstring for Mask2D. " - "Difficulty: large. Autonomy: supervised.", "test") - assert d["title"] == "Fix the typo in the @PyAutoHands docstring for Mask2D" - assert "large" not in d["proposed_path"] - assert "supervised" not in d["proposed_path"] - - -def test_ideas_house_style_trailing_autonomy(): - # `ideas.md` bullets end "Difficulty large, supervised." — one clause, two fields. - fields, spans = _declared_fields("Rework the grid. Difficulty large, supervised.") - assert fields == {"difficulty": "large", "autonomy": "supervised"} - assert spans - - -def test_declared_autonomy_and_priority_are_taken_as_written(): - d = analyse("Rework the @PyAutoHands grid API. Autonomy: human-required. " - "Priority: high.", "test") - assert d["autonomy"] == "human-required" - assert d["autonomy_source"] == "declared" - assert d["priority"] == "high" - assert d["priority_source"] == "declared" - - -def test_spelling_variants_normalise_to_header_values(): - fields, _ = _declared_fields("Difficulty is Too Large, human required.") - assert fields == {"difficulty": "too-large", "autonomy": "human-required"} - - -def test_code_spans_are_not_declarations(): - text = ("Intake drops the declaration. Repro:\n\n" - "```\nintake \"a task. Difficulty: large.\"\n```\n\n" - "Observed `Difficulty: too-large` in the header.\n") - fields, _ = _declared_fields(text) - assert fields == {} - - -def test_prose_without_a_declaration_still_sizes_by_heuristic(): - d = analyse("The @PyAutoHands mask is large and the difficulty of lensing " - "is well known.", "test") - assert d["difficulty_source"] == "estimated" - assert d["autonomy_source"] == "inferred" - assert d["priority_source"] == "inferred" - - -def test_strip_declarations_leaves_prose_alone_when_nothing_declared(): - text = "Just a sentence." - assert _strip_declarations(text, []) == text - - -def test_declaration_only_text_keeps_a_title_to_derive_from(): - d = analyse("Difficulty: large.", "test") - assert d["title"] != "Untitled" - - -def test_formalise_derives_the_declared_difficulty(): - body = ("# A backlog prompt\n\nRework the grid in @PyAutoHands.\n" - "Difficulty large, supervised.\n") - fields = _derive_fields(body, "refactor", "autoarray") - assert fields["difficulty"] == "large" - assert fields["autonomy"] == "supervised"