From b3f9ac3612015d972cc1cc73d0c98c3e4a048542 Mon Sep 17 00:00:00 2001 From: adongwanai Date: Wed, 2 Sep 2026 18:31:51 +0800 Subject: [PATCH] feat(sleep): add community rule manifest --- docs/reference/cli.md | 9 +- docs/sleep/README.md | 7 + docs/sleep/community-rules.md | 99 +++++++++ mkdocs.yml | 1 + skillopt_sleep/__main__.py | 286 ++++++++++++++++++++++++- skillopt_sleep/community.py | 390 ++++++++++++++++++++++++++++++++++ tests/test_community_rules.py | 318 +++++++++++++++++++++++++++ 7 files changed, 1104 insertions(+), 6 deletions(-) create mode 100644 docs/sleep/community-rules.md create mode 100644 skillopt_sleep/community.py create mode 100644 tests/test_community_rules.py diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 6aefbb20..9f75a775 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -124,7 +124,7 @@ python -m skillopt_sleep [options] ``` Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, -`unschedule`, and `evalkit`. `evalkit` is also available as +`unschedule`, `export-rules`, `import-rules`, and `evalkit`. `evalkit` is also available as `python -m skillopt_sleep.evalkit` and compares two conditions on one fixed task manifest (McNemar + bootstrap CI). Exactly one of its `--b` comparison input or `--aa` identity-check flag is required. See `docs/sleep/evalkit.md`. @@ -168,6 +168,13 @@ roots. Use `--skill-root` for another integration-specific location. Configure the canonical `multi_skill_fanout` key to enable proposal fan-out; `multi_skill_report` remains a compatibility alias. +`export-rules` converts accepted skill additions from one staging report into a +versioned, transcript-free manifest. `import-rules` first displays the complete +manifest for review; after `--reviewed` is supplied, it evaluates every rule on +the importer's task file `val` split and stages only strict, no-regression improvements. +Publisher-reported effects do not affect the gate. See +[community rule exchange](../sleep/community-rules.md). + The `mock` and `handoff` backends make no network calls. A real backend sends mining, replay, judging, and reflection prompts derived from harvested transcripts and tasks to its selected provider. Review that provider's diff --git a/docs/sleep/README.md b/docs/sleep/README.md index f47556ee..f4147a0e 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -103,9 +103,16 @@ skillopt-sleep status # show state + the latest staged proposal skillopt-sleep adopt --legacy # apply a reviewed managed proposal skillopt-sleep adopt --skill NAME # adopt one staged skill (repeatable) skillopt-sleep adopt --all-skills # adopt every still-pending fan-out skill +skillopt-sleep export-rules ... # export accepted rules without transcripts +skillopt-sleep import-rules ... # review + locally gate community rules skillopt-sleep schedule # install a nightly cron entry for this project ``` +Community manifests contain distilled rules and aggregate effect metadata, not +session transcripts. Imports require an explicit review acknowledgement and put +every rule through a strict local no-regression gate before staging. See +[community rule exchange](community-rules.md). + > **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base > commands above. Cursor source/backend/plugin support, VS Code Copilot > transcript harvesting, Pi source/backend support, Sleep handoff, non-Azure diff --git a/docs/sleep/community-rules.md b/docs/sleep/community-rules.md new file mode 100644 index 00000000..988061ec --- /dev/null +++ b/docs/sleep/community-rules.md @@ -0,0 +1,99 @@ +# Community rule exchange + +SkillOpt-Sleep can exchange distilled skill rules without exchanging session +transcripts. Imported rules are untrusted candidates: each rule must be reviewed +and must strictly improve the importer's own task set without regressing any task +before it is staged for adoption. + +This is a local workflow. It does not upload manifests, operate a registry, or +automatically adopt imported rules. + +An empirical gate is not a sandbox or a security review. Imported text changes +agent behavior during validation, so inspect the rule and license before passing +`--reviewed`, and use the same provider and execution-boundary precautions as an +ordinary Sleep replay. + +## Export accepted rules + +Export reads an accepted staging report and includes only accepted `skill/add` +edits. Memory edits, tasks, responses, session identifiers, and evidence logs are +not copied. + +```bash +skillopt-sleep export-rules \ + --staging .skillopt-sleep/staging/20260722-031700 \ + --output community-rules.json \ + --category coding \ + --license MIT +``` + +The observed effect belongs to the complete candidate set evaluated by that +staging run, not to an individual rule in isolation. The exporter labels this as +`"scope": "candidate_set"`. It also refuses secret-shaped rule or rationale text, +but that is not an anonymization guarantee. Inspect the output before publishing +it to a public Git repository. + +## Review and import + +The first invocation prints the complete manifest and exits without running it: + +```bash +skillopt-sleep import-rules --manifest community-rules.json +``` + +After reviewing every rule and the manifest license, run the local gate with a +reviewed task file and an explicit target skill: + +```bash +skillopt-sleep import-rules \ + --manifest community-rules.json \ + --reviewed \ + --project /path/to/project \ + --target-skill-path .agents/skills/my-skill/SKILL.md \ + --tasks-file reviewed-tasks.json \ + --backend codex +``` + +The importer evaluates rules sequentially using only the task file's `val` +split; `train` and `test` remain outside the import decision. For each rule it +replays the same validation tasks against the current skill and the candidate +skill. A rule enters the staged proposal only when its configured gate score +strictly increases and no task score decreases. Publisher-reported effects are +informational and never participate in the local decision. + +Review an accepted `proposed_SKILL.md`, then use the existing explicit adoption +step: + +```bash +skillopt-sleep adopt --legacy +``` + +## Manifest v1 + +```json +{ + "schema": "skillopt.community-rules", + "schema_version": 1, + "license": "MIT", + "rules": [ + { + "id": "rule-63e143cbd8ab167d", + "category": "coding", + "rule": "Run focused tests before reporting a change as complete.", + "rationale": "Prevents false completion reports.", + "observed_effect": { + "metric": "local_gate_score", + "baseline": 0.5, + "candidate": 0.75, + "delta": 0.25, + "sample_size": 20, + "scope": "candidate_set" + } + } + ] +} +``` + +The v1 parser rejects unknown fields. This keeps the public artifact bounded to +the rule, a provenance-free rationale, aggregate effect metadata, category, and +license; raw trajectories have no field in the format. diff --git a/mkdocs.yml b/mkdocs.yml index b5934ad6..8690b468 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -50,6 +50,7 @@ nav: - Deep Learning Analogy: guide/dl-analogy.md - SkillOpt-Sleep: - Overview: sleep/README.md + - Community Rule Exchange: sleep/community-rules.md - Paired A/B Evalkit: sleep/evalkit.md - Multi-skill Staging: sleep/multi-skill-staging.md - OpenAI-compatible Endpoints: sleep/openai-compatible-endpoints.md diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index e3b7794c..e1549435 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -6,6 +6,8 @@ python -m skillopt_sleep adopt # apply the latest staged proposal (with backup) python -m skillopt_sleep adopt --skill NAME # adopt one staged skill (repeatable) python -m skillopt_sleep harvest # just print what would be mined (debug) + python -m skillopt_sleep export-rules # export accepted, distilled rules + python -m skillopt_sleep import-rules # review + locally gate community rules Common flags: --project PATH project to evolve (default: cwd) @@ -30,12 +32,26 @@ import json import os import sys +from dataclasses import asdict +from datetime import datetime, timezone from typing import Any, Dict -from skillopt_sleep.backend import CursorBackendError +from skillopt_sleep.backend import CursorBackendError, build_backend +from skillopt_sleep.community import ( + CommunityRuleError, + export_rules_from_staging, + gate_community_rules, + load_rule_manifest, +) from skillopt_sleep.config import load_config -from skillopt_sleep.cycle import _one_line_display_text, run_sleep_cycle +from skillopt_sleep.cycle import ( + _one_line_display_text, + _read_live_baseline, + _render_report_md, + run_sleep_cycle, +) from skillopt_sleep.harvest_sources import harvest_for_config +from skillopt_sleep.memory import ensure_skill_scaffold from skillopt_sleep.mine import mine from skillopt_sleep.staging import ( StagingError, @@ -45,10 +61,25 @@ latest_staging, pending_staged_skills, staged_skills, + write_staging, ) from skillopt_sleep.staging import adopt as adopt_staging from skillopt_sleep.state import SleepState from skillopt_sleep.tasks_file import load_tasks_file, make_tasks_payload, write_tasks_file +from skillopt_sleep.types import SleepReport + +_BACKEND_CHOICES = [ + "", + "mock", + "claude", + "codex", + "copilot", + "cursor", + "pi", + "opencode", + "handoff", + "azure_openai", +] def _read_text(path: str) -> str: @@ -97,9 +128,7 @@ def _report_payload(rep, outcome) -> Dict[str, Any]: def _add_common(p: argparse.ArgumentParser) -> None: p.add_argument("--project", default="") p.add_argument("--scope", default="", choices=["", "all", "invoked"]) - p.add_argument("--backend", default="", - choices=["", "mock", "claude", "codex", "copilot", "cursor", "pi", - "opencode", "handoff", "azure_openai"]) + p.add_argument("--backend", default="", choices=_BACKEND_CHOICES) p.add_argument("--model", default="") p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary") p.add_argument("--cursor-path", default="", help="path to the Cursor Agent CLI") @@ -847,6 +876,218 @@ def cmd_unschedule(args) -> int: return 0 if ok else 1 +def cmd_export_rules(args) -> int: + try: + output, manifest = export_rules_from_staging( + args.staging, + args.output, + category=args.category, + license_id=args.license, + ) + except (CommunityRuleError, OSError, json.JSONDecodeError) as exc: + _print_run_failure(args, "rule_export_refused", exc) + return 2 + payload = { + "ok": True, + "output": output, + "schema": manifest.schema, + "schema_version": manifest.schema_version, + "rules": len(manifest.rules), + } + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print(f"[sleep] exported {len(manifest.rules)} community rule(s): {_display_value(output)}") + print("[sleep] review the manifest before publishing it; no transcript data is included by the exporter") + return 0 + + +def _print_rule_review(args, manifest) -> int: + payload = { + "ok": False, + "review_required": True, + "license": manifest.license, + "rules": [ + { + "id": rule.id, + "category": rule.category, + "rule": rule.rule, + "rationale": rule.rationale, + "observed_effect": asdict(rule.observed_effect), + } + for rule in manifest.rules + ], + } + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print( + f"[sleep] review required for {len(manifest.rules)} imported rule(s); " + f"license={_display_value(manifest.license)}" + ) + for rule in manifest.rules: + print(f" [{_display_value(rule.id)}] {_display_value(rule.category)}: {_display_value(rule.rule)}") + print(f" rationale: {_display_value(rule.rationale)}") + effect = rule.observed_effect + print( + f" publisher effect ({_display_value(effect.scope)}): " + f"{_display_value(effect.metric)} {effect.baseline:.3f} -> " + f"{effect.candidate:.3f} (delta={effect.delta:+.3f}, n={effect.sample_size})" + ) + print("[sleep] rerun with --reviewed after inspecting every rule and its license") + return 2 + + +def cmd_import_rules(args) -> int: + try: + manifest = load_rule_manifest(args.manifest) + except (CommunityRuleError, OSError, json.JSONDecodeError) as exc: + _print_run_failure(args, "rule_import_refused", exc) + return 2 + if not args.reviewed: + return _print_rule_review(args, manifest) + if not args.tasks_file: + _print_run_failure(args, "rule_import_refused", "--tasks-file is required for the local gate") + return 2 + + try: + tasks, task_meta = load_tasks_file(args.tasks_file) + cfg = _cfg_from_args(args, task_meta=task_meta) + tasks, task_meta = load_tasks_file( + args.tasks_file, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + _print_run_failure(args, "rule_import_refused", exc) + return 2 + if cfg.get("backend", "mock") != "mock" and task_meta.get("reviewed") is not True: + _print_run_failure( + args, + "rule_import_refused", + 'real-backend gating requires a tasks file with "reviewed": true', + ) + return 2 + if cfg.get("backend", "mock") == "handoff": + _print_run_failure(args, "rule_import_refused", "handoff backend is not supported for rule import") + return 2 + gate_tasks = [task for task in tasks if task.split == "val"] + if not gate_tasks: + _print_run_failure( + args, + "rule_import_refused", + "tasks file must contain at least one val task for the held-out gate", + ) + return 2 + + project = cfg.get("invoked_project") or os.getcwd() + live_skill_path = cfg.managed_skill_path() + live_memory_path = os.path.join(project, "CLAUDE.md") + try: + raw_skill, live_skill_sha256, live_skill_realpath = _read_live_baseline( + live_skill_path, "skill" + ) + memory, live_memory_sha256, live_memory_realpath = _read_live_baseline( + live_memory_path, "memory" + ) + skill = raw_skill or ensure_skill_scaffold( + "", + name=cfg.get("managed_skill_name", "skillopt-sleep-learned"), + description="Preferences and procedures learned from past local agent sessions.", + ) + backend = build_backend( + backend=cfg.get("backend", "mock"), + model=cfg.get("model", ""), + optimizer_backend=cfg.get("optimizer_backend", ""), + optimizer_model=cfg.get("optimizer_model", ""), + target_backend=cfg.get("target_backend", ""), + target_model=cfg.get("target_model", ""), + codex_path=cfg.get("codex_path", ""), + pi_path=cfg.get("pi_path", ""), + cursor_path=cfg.get("cursor_path", ""), + opencode_path=cfg.get("opencode_path", ""), + opencode_tool_replay=cfg.get("opencode_tool_replay", False), + azure_endpoint=cfg.get("azure_endpoint", ""), + preferences=cfg.get("preferences", ""), + project_dir=project, + ) + result = gate_community_rules( + backend, + gate_tasks, + skill, + memory, + manifest, + gate_metric=cfg.get("gate_metric", "mixed"), + gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), + ) + now = datetime.now(timezone.utc).isoformat() + report = SleepReport( + night=0, + project=project, + started_at=now, + ended_at=now, + n_tasks=len(gate_tasks), + n_replayed=len(gate_tasks), + baseline_score=result.baseline_score, + candidate_score=result.candidate_score, + accepted=result.accepted, + gate_action=("community_rules_accepted" if result.accepted else "community_rules_rejected"), + edits=result.accepted_edits, + rejected_edits=result.rejected_edits, + unmatched_edits=result.unmatched_edits, + tokens_used=backend.tokens_used(), + notes=[ + f"community import: {len(manifest.rules)} reviewed rules; " + f"{len(result.accepted_edits)} passed the local no-regression gate" + ], + gate_no_regression=True, + gate_trials=result.trials, + ) + staging_dir = write_staging( + project, + report=report, + proposed_skill=result.new_skill if result.accepted else None, + proposed_memory=None, + live_skill_path=live_skill_path, + live_memory_path=live_memory_path, + live_skill_sha256=live_skill_sha256, + live_memory_sha256=live_memory_sha256, + live_skill_realpath=live_skill_realpath, + live_memory_realpath=live_memory_realpath, + report_md=_render_report_md(report, cfg), + ) + except CursorBackendError as exc: + _print_run_failure(args, "backend_failed", exc) + return 1 + except (CommunityRuleError, StagingError, OSError) as exc: + _print_run_failure(args, "rule_import_refused", exc) + return 2 + + payload = { + "ok": True, + "accepted": result.accepted, + "baseline": result.baseline_score, + "candidate": result.candidate_score, + "accepted_rules": len(result.accepted_edits), + "rejected_rules": len(result.rejected_edits), + "duplicate_rules": len(result.unmatched_edits), + "staging_dir": staging_dir, + "trials": result.trials, + } + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print( + f"[sleep] local rule gate {result.baseline_score:.3f} -> " + f"{result.candidate_score:.3f}; accepted {len(result.accepted_edits)}/" + f"{len(manifest.rules)}" + ) + print(f"[sleep] staged: {_display_value(staging_dir)}") + if result.accepted: + print("[sleep] review it, then: python -m skillopt_sleep adopt --legacy") + return 0 + + def main(argv=None) -> int: parser = argparse.ArgumentParser(prog="skillopt_sleep", description="SkillOpt-Sleep nightly self-evolution") sub = parser.add_subparsers(dest="cmd", required=True) @@ -895,6 +1136,37 @@ def main(argv=None) -> int: p_eval.add_argument("--seed", type=int, default=42) p_eval.add_argument("--allow-graded", action="store_true") p_eval.add_argument("--json", action="store_true") + p_export_rules = sub.add_parser( + "export-rules", + help="export accepted skill additions as a transcript-free rule manifest", + ) + p_export_rules.add_argument("--staging", required=True) + p_export_rules.add_argument("--output", required=True) + p_export_rules.add_argument("--category", required=True) + p_export_rules.add_argument("--license", required=True) + p_export_rules.add_argument("--json", action="store_true") + p_import_rules = sub.add_parser( + "import-rules", + help="review and locally gate a community rule manifest", + ) + p_import_rules.add_argument("--project", default="") + p_import_rules.add_argument("--backend", default="", choices=_BACKEND_CHOICES) + p_import_rules.add_argument("--model", default="") + p_import_rules.add_argument("--codex-path", default="") + p_import_rules.add_argument("--cursor-path", default="") + p_import_rules.add_argument("--pi-path", default="") + p_import_rules.add_argument("--opencode-path", default="") + p_import_rules.add_argument("--opencode-tool-replay", action="store_true") + p_import_rules.add_argument("--target-skill-path", default="") + p_import_rules.add_argument("--tasks-file", default="") + p_import_rules.add_argument("--json", action="store_true") + p_import_rules.set_defaults(scope="") + p_import_rules.add_argument("--manifest", required=True) + p_import_rules.add_argument( + "--reviewed", + action="store_true", + help="confirm that every imported rule and the manifest license were reviewed", + ) args = parser.parse_args(argv) if args.cmd == "run": @@ -924,6 +1196,10 @@ def main(argv=None) -> int: if args.json: argv.append("--json") return evalkit_main(argv) + if args.cmd == "export-rules": + return cmd_export_rules(args) + if args.cmd == "import-rules": + return cmd_import_rules(args) parser.print_help() return 2 diff --git a/skillopt_sleep/community.py b/skillopt_sleep/community.py new file mode 100644 index 00000000..6d5d36f9 --- /dev/null +++ b/skillopt_sleep/community.py @@ -0,0 +1,390 @@ +"""Privacy-preserving community rule exchange for SkillOpt-Sleep.""" +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Tuple + +from skillopt_sleep.backend import Backend +from skillopt_sleep.gate import select_gate_score +from skillopt_sleep.memory import apply_edits_detailed +from skillopt_sleep.replay import aggregate_scores, replay_batch +from skillopt_sleep.staging import redact_secrets +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + +RULE_MANIFEST_SCHEMA = "skillopt.community-rules" +RULE_MANIFEST_VERSION = 1 +_MAX_RULES = 100 +_MAX_RULE_CHARS = 4000 +_MAX_RATIONALE_CHARS = 2000 +_RULE_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +class CommunityRuleError(ValueError): + """A community rule manifest or export source is invalid.""" + + +@dataclass(frozen=True) +class ObservedEffect: + metric: str + baseline: float + candidate: float + delta: float + sample_size: int + scope: str = "candidate_set" + + +@dataclass(frozen=True) +class CommunityRule: + id: str + category: str + rule: str + rationale: str + observed_effect: ObservedEffect + + +@dataclass(frozen=True) +class RuleManifest: + license: str + rules: Tuple[CommunityRule, ...] + schema: str = RULE_MANIFEST_SCHEMA + schema_version: int = RULE_MANIFEST_VERSION + + def to_dict(self) -> Dict[str, Any]: + return { + "schema": self.schema, + "schema_version": self.schema_version, + "license": self.license, + "rules": [asdict(rule) for rule in self.rules], + } + + +@dataclass +class CommunityGateResult: + accepted: bool + baseline_score: float + candidate_score: float + new_skill: str + accepted_edits: List[EditRecord] + rejected_edits: List[EditRecord] + unmatched_edits: List[EditRecord] + trials: List[Dict[str, Any]] + + +def _object(value: Any, label: str) -> Dict[str, Any]: + if not isinstance(value, dict): + raise CommunityRuleError(f"{label} must be a JSON object") + return value + + +def _exact_keys(value: Dict[str, Any], expected: set[str], label: str) -> None: + unknown = sorted(set(value) - expected) + missing = sorted(expected - set(value)) + if unknown: + raise CommunityRuleError(f"{label} has unknown fields: {', '.join(unknown)}") + if missing: + raise CommunityRuleError(f"{label} is missing fields: {', '.join(missing)}") + + +def _text(value: Any, label: str, max_chars: int) -> str: + if not isinstance(value, str) or not value.strip(): + raise CommunityRuleError(f"{label} must be non-empty text") + text = value.strip() + if len(text) > max_chars: + raise CommunityRuleError(f"{label} exceeds {max_chars} characters") + return text + + +def _number(value: Any, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise CommunityRuleError(f"{label} must be a finite number") + result = float(value) + if not math.isfinite(result): + raise CommunityRuleError(f"{label} must be a finite number") + return result + + +def _parse_effect(payload: Any, label: str) -> ObservedEffect: + value = _object(payload, label) + _exact_keys( + value, + {"metric", "baseline", "candidate", "delta", "sample_size", "scope"}, + label, + ) + metric = _text(value["metric"], f"{label}.metric", 128) + baseline = _number(value["baseline"], f"{label}.baseline") + candidate = _number(value["candidate"], f"{label}.candidate") + delta = _number(value["delta"], f"{label}.delta") + sample_size = value["sample_size"] + if isinstance(sample_size, bool) or not isinstance(sample_size, int) or sample_size < 1: + raise CommunityRuleError(f"{label}.sample_size must be a positive integer") + if value["scope"] != "candidate_set": + raise CommunityRuleError(f"{label}.scope must be 'candidate_set'") + if not math.isclose(candidate - baseline, delta, abs_tol=1e-9): + raise CommunityRuleError(f"{label}.delta must equal candidate - baseline") + return ObservedEffect(metric, baseline, candidate, delta, sample_size) + + +def parse_rule_manifest(payload: Any) -> RuleManifest: + value = _object(payload, "manifest") + _exact_keys(value, {"schema", "schema_version", "license", "rules"}, "manifest") + if value["schema"] != RULE_MANIFEST_SCHEMA: + raise CommunityRuleError("manifest has an unsupported schema") + if ( + type(value["schema_version"]) is not int + or value["schema_version"] != RULE_MANIFEST_VERSION + ): + raise CommunityRuleError("manifest has an unsupported schema version") + license_id = _text(value["license"], "manifest.license", 128) + raw_rules = value["rules"] + if not isinstance(raw_rules, list) or not raw_rules: + raise CommunityRuleError("manifest.rules must be a non-empty array") + if len(raw_rules) > _MAX_RULES: + raise CommunityRuleError(f"manifest.rules exceeds {_MAX_RULES} entries") + + rules: List[CommunityRule] = [] + seen_ids: set[str] = set() + for index, raw_rule in enumerate(raw_rules): + label = f"manifest.rules[{index}]" + row = _object(raw_rule, label) + _exact_keys( + row, + {"id", "category", "rule", "rationale", "observed_effect"}, + label, + ) + rule_id = _text(row["id"], f"{label}.id", 128) + if not _RULE_ID_RE.fullmatch(rule_id): + raise CommunityRuleError(f"{label}.id contains unsupported characters") + if rule_id in seen_ids: + raise CommunityRuleError(f"duplicate rule id: {rule_id}") + seen_ids.add(rule_id) + rules.append(CommunityRule( + id=rule_id, + category=_text(row["category"], f"{label}.category", 128), + rule=_text(row["rule"], f"{label}.rule", _MAX_RULE_CHARS), + rationale=_text( + row["rationale"], f"{label}.rationale", _MAX_RATIONALE_CHARS + ), + observed_effect=_parse_effect( + row["observed_effect"], f"{label}.observed_effect" + ), + )) + return RuleManifest(license=license_id, rules=tuple(rules)) + + +def load_rule_manifest(path: str) -> RuleManifest: + source = os.path.abspath(os.path.expanduser(path)) + with open(source, encoding="utf-8") as handle: + return parse_rule_manifest(json.load(handle)) + + +def write_rule_manifest(path: str, manifest: RuleManifest) -> str: + validated = parse_rule_manifest(manifest.to_dict()) + output = os.path.abspath(os.path.expanduser(path)) + parent = os.path.dirname(output) + if parent: + os.makedirs(parent, exist_ok=True) + with open(output, "w", encoding="utf-8") as handle: + json.dump(validated.to_dict(), handle, ensure_ascii=False, indent=2) + handle.write("\n") + return output + + +def _rule_id(category: str, rule: str) -> str: + digest = hashlib.sha256(f"{category}\0{rule}".encode("utf-8")).hexdigest() + return f"rule-{digest[:16]}" + + +def export_rules_from_staging( + staging_dir: str, + output_path: str, + *, + category: str, + license_id: str, +) -> tuple[str, RuleManifest]: + """Export accepted skill additions without copying transcripts or task data.""" + category = _text(category, "category", 128) + license_id = _text(license_id, "license", 128) + report_path = os.path.join(os.path.abspath(staging_dir), "report.json") + with open(report_path, encoding="utf-8") as handle: + report = _object(json.load(handle), "staging report") + if report.get("accepted") is not True: + raise CommunityRuleError("staging report has no accepted candidate to export") + baseline = _number(report.get("baseline_score"), "report.baseline_score") + candidate = _number(report.get("candidate_score"), "report.candidate_score") + sample_size = report.get("n_tasks") + if isinstance(sample_size, bool) or not isinstance(sample_size, int) or sample_size < 1: + raise CommunityRuleError("report.n_tasks must be a positive integer") + raw_edits = report.get("edits") + if not isinstance(raw_edits, list): + raise CommunityRuleError("report.edits must be an array") + + effect = ObservedEffect( + metric="local_gate_score", + baseline=baseline, + candidate=candidate, + delta=candidate - baseline, + sample_size=sample_size, + ) + rules: List[CommunityRule] = [] + for index, raw_edit in enumerate(raw_edits): + edit = _object(raw_edit, f"report.edits[{index}]") + if edit.get("target") != "skill" or edit.get("op") != "add": + continue + rule = _text(edit.get("content"), f"report.edits[{index}].content", _MAX_RULE_CHARS) + rationale = _text( + edit.get("rationale"), + f"report.edits[{index}].rationale", + _MAX_RATIONALE_CHARS, + ) + if redact_secrets(rule) != rule or redact_secrets(rationale) != rationale: + raise CommunityRuleError( + f"report.edits[{index}] contains secret-shaped text; review it manually" + ) + rules.append(CommunityRule( + id=_rule_id(category, rule), + category=category, + rule=rule, + rationale=rationale, + observed_effect=effect, + )) + if not rules: + raise CommunityRuleError("staging report has no accepted skill additions to export") + manifest = RuleManifest(license=license_id, rules=tuple(rules)) + return write_rule_manifest(output_path, manifest), manifest + + +def _task_deltas( + tasks: List[TaskRecord], + baseline_pairs: List[Tuple[TaskRecord, ReplayResult]], + candidate_pairs: List[Tuple[TaskRecord, ReplayResult]], + metric: str, + mixed_weight: float, +) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + for task, (_, baseline), (_, candidate) in zip( + tasks, baseline_pairs, candidate_pairs + ): + before = select_gate_score( + baseline.hard, baseline.soft, metric, mixed_weight + ) + after = select_gate_score( + candidate.hard, candidate.soft, metric, mixed_weight + ) + scores_are_finite = math.isfinite(before) and math.isfinite(after) + rows.append({ + "task_id": task.id, + "baseline_score": before, + "candidate_score": after, + "status": ( + "regressed" + if not scores_are_finite + else "improved" + if after > before + else "regressed" + if after < before + else "unchanged" + ), + "scores_are_finite": scores_are_finite, + }) + return rows + + +def gate_community_rules( + backend: Backend, + tasks: List[TaskRecord], + skill: str, + memory: str, + manifest: RuleManifest, + *, + gate_metric: str = "mixed", + gate_mixed_weight: float = 0.5, +) -> CommunityGateResult: + """Put each imported rule through a strict, no-regression local gate.""" + if not tasks: + raise CommunityRuleError("local gate requires at least one reviewed task") + current_skill = skill + current_pairs = replay_batch(backend, tasks, current_skill, memory) + base_hard, base_soft = aggregate_scores(current_pairs) + original_score = select_gate_score( + base_hard, base_soft, gate_metric, gate_mixed_weight + ) + if not math.isfinite(original_score): + raise CommunityRuleError("local gate produced a non-finite baseline score") + current_score = original_score + accepted_edits: List[EditRecord] = [] + rejected_edits: List[EditRecord] = [] + unmatched_edits: List[EditRecord] = [] + trials: List[Dict[str, Any]] = [] + + for rule in manifest.rules: + edit = EditRecord( + target="skill", + op="add", + content=rule.rule, + rationale=rule.rationale, + ) + candidate_skill, applied, unmatched = apply_edits_detailed( + current_skill, [edit] + ) + if unmatched: + unmatched_edits.extend(unmatched) + trials.append({ + "rule_id": rule.id, + "category": rule.category, + "baseline_score": current_score, + "candidate_score": current_score, + "accepted": False, + "reason": "duplicate_or_empty", + "task_deltas": [], + }) + continue + candidate_pairs = replay_batch(backend, tasks, candidate_skill, memory) + cand_hard, cand_soft = aggregate_scores(candidate_pairs) + candidate_score = select_gate_score( + cand_hard, cand_soft, gate_metric, gate_mixed_weight + ) + task_deltas = _task_deltas( + tasks, + current_pairs, + candidate_pairs, + gate_metric, + gate_mixed_weight, + ) + regressed = any(row["status"] == "regressed" for row in task_deltas) + accepted = ( + math.isfinite(candidate_score) + and candidate_score > current_score + and not regressed + ) + trials.append({ + "rule_id": rule.id, + "category": rule.category, + "baseline_score": current_score, + "candidate_score": candidate_score, + "accepted": accepted, + "reason": "improved" if accepted else "no_strict_lift_or_regression", + "task_deltas": task_deltas, + }) + if accepted: + current_skill = candidate_skill + current_pairs = candidate_pairs + current_score = candidate_score + accepted_edits.extend(applied) + else: + rejected_edits.extend(applied) + + return CommunityGateResult( + accepted=bool(accepted_edits), + baseline_score=original_score, + candidate_score=current_score, + new_skill=current_skill, + accepted_edits=accepted_edits, + rejected_edits=rejected_edits, + unmatched_edits=unmatched_edits, + trials=trials, + ) diff --git a/tests/test_community_rules.py b/tests/test_community_rules.py new file mode 100644 index 00000000..4ee2bea5 --- /dev/null +++ b/tests/test_community_rules.py @@ -0,0 +1,318 @@ +"""Tests for transcript-free community rules and local empirical gating.""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from contextlib import redirect_stdout +from io import StringIO + +from skillopt_sleep.__main__ import main +from skillopt_sleep.backend import Backend, MockBackend, exact_score, keyword_soft_score +from skillopt_sleep.community import ( + CommunityRule, + CommunityRuleError, + ObservedEffect, + RuleManifest, + export_rules_from_staging, + gate_community_rules, + parse_rule_manifest, + write_rule_manifest, +) +from skillopt_sleep.tasks_file import make_tasks_payload, write_tasks_file +from skillopt_sleep.types import TaskRecord + + +def _effect() -> ObservedEffect: + return ObservedEffect( + metric="local_gate_score", + baseline=0.2, + candidate=0.8, + delta=0.6, + sample_size=10, + ) + + +def _manifest(rule: str) -> RuleManifest: + return RuleManifest( + license="MIT", + rules=(CommunityRule( + id="rule-example", + category="coding", + rule=rule, + rationale="Improves formatting consistency.", + observed_effect=_effect(), + ),), + ) + + +class _RegressionBackend(Backend): + def attempt(self, task, skill, memory, sample_id=0): + enabled = "Use the community strategy." in skill + if task.id in {"improve-1", "improve-2"}: + return task.reference if enabled else "wrong" + return "wrong" if enabled else task.reference + + def judge(self, task, response): + return ( + exact_score(task.reference, response), + keyword_soft_score(task.reference, response), + "", + ) + + def reflect(self, *args, **kwargs): + raise AssertionError("community imports must not invoke reflection") + + +class TestCommunityManifest(unittest.TestCase): + def test_manifest_rejects_transcript_fields(self): + payload = _manifest("Use a stable formatter.").to_dict() + payload["transcripts"] = ["private session"] + with self.assertRaisesRegex(CommunityRuleError, "unknown fields: transcripts"): + parse_rule_manifest(payload) + + def test_manifest_rejects_inconsistent_effect_delta(self): + payload = _manifest("Use a stable formatter.").to_dict() + payload["rules"][0]["observed_effect"]["delta"] = 99 + with self.assertRaisesRegex(CommunityRuleError, "delta must equal"): + parse_rule_manifest(payload) + + def test_export_contains_only_accepted_skill_additions(self): + with tempfile.TemporaryDirectory() as tmp: + report = { + "accepted": True, + "baseline_score": 0.25, + "candidate_score": 0.75, + "n_tasks": 12, + "edits": [ + { + "target": "skill", + "op": "add", + "content": "Verify generated files before reporting success.", + "anchor": "", + "rationale": "Prevents false completion reports.", + }, + { + "target": "memory", + "op": "add", + "content": "Private preference", + "anchor": "", + "rationale": "Local only.", + }, + ], + } + with open(os.path.join(tmp, "report.json"), "w", encoding="utf-8") as handle: + json.dump(report, handle) + output = os.path.join(tmp, "community-rules.json") + + _, manifest = export_rules_from_staging( + tmp, + output, + category="coding", + license_id="MIT", + ) + with open(output, encoding="utf-8") as handle: + raw = json.load(handle) + + self.assertEqual(len(manifest.rules), 1) + self.assertEqual(raw["rules"][0]["observed_effect"]["scope"], "candidate_set") + self.assertNotIn("Private preference", json.dumps(raw)) + self.assertNotIn("transcript", json.dumps(raw).lower()) + + +class TestCommunityGate(unittest.TestCase): + def test_rule_must_improve_local_tasks(self): + task = TaskRecord( + id="commit", + project="/repo", + intent="write a commit subject", + reference_kind="exact", + reference="feat: add parser", + tags=["rule:commit-imperative"], + ) + rule = MockBackend.RULE_TEXT["commit-imperative"] + + result = gate_community_rules(MockBackend(), [task], "# Skill\n", "", _manifest(rule)) + + self.assertTrue(result.accepted) + self.assertGreater(result.candidate_score, result.baseline_score) + self.assertIn(rule, result.new_skill) + self.assertEqual(len(result.accepted_edits), 1) + + def test_publisher_effect_does_not_override_local_rejection(self): + task = TaskRecord( + id="commit", + project="/repo", + intent="write a commit subject", + reference_kind="exact", + reference="feat: add parser", + tags=["rule:commit-imperative"], + ) + + result = gate_community_rules( + MockBackend(), + [task], + "# Skill\n", + "", + _manifest("An unrelated rule with a claimed large effect."), + ) + + self.assertFalse(result.accepted) + self.assertEqual(len(result.rejected_edits), 1) + self.assertEqual(result.trials[0]["reason"], "no_strict_lift_or_regression") + + def test_any_local_regression_blocks_aggregate_lift(self): + tasks = [ + TaskRecord(id="improve-1", project="/repo", intent="one", reference="answer one"), + TaskRecord(id="improve-2", project="/repo", intent="two", reference="answer two"), + TaskRecord(id="regress", project="/repo", intent="three", reference="answer three"), + ] + + result = gate_community_rules( + _RegressionBackend(), + tasks, + "# Skill\n", + "", + _manifest("Use the community strategy."), + gate_metric="hard", + ) + + self.assertFalse(result.accepted) + self.assertGreater(result.trials[0]["candidate_score"], result.baseline_score) + self.assertTrue(any( + row["status"] == "regressed" for row in result.trials[0]["task_deltas"] + )) + + +class TestCommunityCli(unittest.TestCase): + def test_import_requires_explicit_rule_review(self): + with tempfile.TemporaryDirectory() as tmp: + manifest_path = write_rule_manifest( + os.path.join(tmp, "rules.json"), + _manifest("Use a stable formatter."), + ) + out = StringIO() + with redirect_stdout(out): + rc = main(["import-rules", "--manifest", manifest_path, "--json"]) + + payload = json.loads(out.getvalue()) + self.assertEqual(rc, 2) + self.assertTrue(payload["review_required"]) + self.assertEqual(payload["rules"][0]["id"], "rule-example") + + def test_reviewed_import_stages_only_a_locally_accepted_rule(self): + with tempfile.TemporaryDirectory() as project: + target = os.path.join(project, ".agents", "skills", "demo", "SKILL.md") + os.makedirs(os.path.dirname(target)) + with open(target, "w", encoding="utf-8") as handle: + handle.write("# Demo\n") + rule = MockBackend.RULE_TEXT["commit-imperative"] + manifest_path = write_rule_manifest( + os.path.join(project, "rules.json"), _manifest(rule) + ) + tasks_path = os.path.join(project, "tasks.json") + tasks_payload = make_tasks_payload( + [TaskRecord( + id="commit", + project=project, + intent="write a commit subject", + reference_kind="exact", + reference="feat: add parser", + tags=["rule:commit-imperative"], + split="val", + )], + project=project, + target_skill_path=target, + ) + tasks_payload["reviewed"] = True + write_tasks_file(tasks_path, tasks_payload) + out = StringIO() + with redirect_stdout(out): + rc = main([ + "import-rules", + "--manifest", manifest_path, + "--reviewed", + "--project", project, + "--backend", "mock", + "--tasks-file", tasks_path, + "--json", + ]) + payload = json.loads(out.getvalue()) + with open( + os.path.join(payload["staging_dir"], "proposed_SKILL.md"), + encoding="utf-8", + ) as handle: + proposed = handle.read() + + self.assertEqual(rc, 0) + self.assertTrue(payload["accepted"]) + self.assertEqual(payload["accepted_rules"], 1) + self.assertIn(rule, proposed) + + def test_import_gate_uses_val_split_only(self): + with tempfile.TemporaryDirectory() as project: + target = os.path.join(project, ".agents", "skills", "demo", "SKILL.md") + os.makedirs(os.path.dirname(target)) + with open(target, "w", encoding="utf-8") as handle: + handle.write("# Demo\n") + manifest_path = write_rule_manifest( + os.path.join(project, "rules.json"), + _manifest(MockBackend.RULE_TEXT["commit-imperative"]), + ) + tasks_path = os.path.join(project, "tasks.json") + tasks_payload = make_tasks_payload( + [ + TaskRecord( + id="train-commit", + project=project, + intent="write a commit subject", + reference_kind="exact", + reference="feat: add parser", + tags=["rule:commit-imperative"], + split="train", + ), + TaskRecord( + id="val-units", + project=project, + intent="report a measurement", + reference_kind="exact", + reference="10 kg", + tags=["rule:units-si"], + split="val", + ), + TaskRecord( + id="test-json", + project=project, + intent="return JSON", + reference_kind="exact", + reference='{"ok": true}', + tags=["rule:json-only"], + split="test", + ), + ], + project=project, + target_skill_path=target, + ) + tasks_payload["reviewed"] = True + write_tasks_file(tasks_path, tasks_payload) + out = StringIO() + with redirect_stdout(out): + rc = main([ + "import-rules", + "--manifest", manifest_path, + "--reviewed", + "--project", project, + "--backend", "mock", + "--tasks-file", tasks_path, + "--json", + ]) + payload = json.loads(out.getvalue()) + + self.assertEqual(rc, 0) + self.assertFalse(payload["accepted"]) + self.assertEqual(payload["trials"][0]["task_deltas"][0]["task_id"], "val-units") + + +if __name__ == "__main__": + unittest.main()