perf: focused agent skill and batched Snowflake retention inspection

.github/workflows/publish.yml

@@ -80,6 +80,7 @@ jobs:
             exit 1
           fi
           unzip -l dist/*.whl | grep 'scripts/dupscore/dupscore.toml'
+          unzip -l dist/*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md'
           python -m venv /tmp/sqlbuild-wheel-smoke
           /tmp/sqlbuild-wheel-smoke/bin/python -m pip install dist/*.whl
           /tmp/sqlbuild-wheel-smoke/bin/python -c "import sqlbuild._native as native; assert native.catalogue_json()"

scripts/docs_skill/_helpers/mdx.py

@@ -55,7 +55,7 @@ def _clean_mdx_body(body: str) -> str:
         stripped: str = line.strip()
         if _is_mdx_only_line(stripped):
             continue
-        cleaned_lines.append(_shift_markdown_heading(line))
+        cleaned_lines.append(line)
 
     return normalize_blank_lines("\n".join(cleaned_lines)).strip()
 
@@ -72,12 +72,6 @@ def _is_mdx_only_line(stripped: str) -> bool:
     return bool(re.fullmatch(r"<img\s+[^>]*/>", stripped))
 
 
-def _shift_markdown_heading(line: str) -> str:
-    if re.match(r"^#{1,6} ", line):
-        return f"#{line}"
-    return line
-
-
 def normalize_blank_lines(contents: str) -> str:
     """Collapse runs of blank lines to one blank line."""
 

scripts/docs_skill/_helpers/navigation.py

@@ -4,6 +4,9 @@
 from pathlib import Path
 from typing import Any
 
+from scripts.docs_skill.constants import UNGROUPED_LABEL
+from scripts.docs_skill.models import NavigationGroup
+
 
 def list_ordered_page_paths(*, docs_root: Path) -> list[Path]:
     """Return existing MDX pages in navigation order followed by unlisted pages."""
@@ -47,3 +50,38 @@ def _collect_navigation_pages(node: Any) -> list[str]:
             pages.extend(_collect_navigation_pages(node.get(key, [])))
 
     return pages
+
+
+def list_navigation_groups(*, docs_root: Path) -> list[NavigationGroup]:
+    """Return pages grouped by top-level navigation group, then unlisted pages."""
+
+    ordered_paths: list[Path] = list_ordered_page_paths(docs_root=docs_root)
+    docs_json_path: Path = docs_root / "docs.json"
+    top_level_nodes: list[Any] = []
+    if docs_json_path.exists():
+        navigation: Any = json.loads(docs_json_path.read_text(encoding="utf-8")).get(
+            "navigation", {}
+        )
+        top_level_nodes = navigation.get("groups", []) if isinstance(navigation, dict) else []
+
+    groups: list[NavigationGroup] = []
+    grouped_pages: set[str] = set()
+    for node in top_level_nodes:
+        label: str = str(node.get("group", UNGROUPED_LABEL)) if isinstance(node, dict) else ""
+        member_pages: set[str] = set(_collect_navigation_pages(node))
+        page_paths: tuple[Path, ...] = tuple(
+            path
+            for path in ordered_paths
+            if path.with_suffix("").as_posix() in member_pages
+            and path.with_suffix("").as_posix() not in grouped_pages
+        )
+        grouped_pages.update(path.with_suffix("").as_posix() for path in page_paths)
+        if page_paths:
+            groups.append(NavigationGroup(label=label, page_paths=page_paths))
+
+    remaining: tuple[Path, ...] = tuple(
+        path for path in ordered_paths if path.with_suffix("").as_posix() not in grouped_pages
+    )
+    if remaining:
+        groups.append(NavigationGroup(label=UNGROUPED_LABEL, page_paths=remaining))
+    return groups

scripts/docs_skill/_helpers/output.py

@@ -1,11 +1,27 @@
-"""Generated skill output handling."""
+"""Generated docs reference output handling."""
 
 from pathlib import Path
 
+from scripts.docs_skill.constants import GENERATED_MARKER
 
-def write_skill_markdown(*, output_path: Path, contents: str) -> None:
-    """Write generated skill Markdown and report its destination."""
 
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.write_text(contents, encoding="utf-8")
-    print(f"Wrote {output_path}")
+def write_reference_pages(*, output_dir: Path, pages: dict[str, str]) -> None:
+    """Write generated reference pages and remove stale generated pages."""
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    for relative_path, contents in pages.items():
+        destination: Path = output_dir / relative_path
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(contents, encoding="utf-8")
+    stale_paths: list[Path] = [
+        path
+        for path in output_dir.rglob("*.md")
+        if path.relative_to(output_dir).as_posix() not in pages
+        and GENERATED_MARKER in path.read_text(encoding="utf-8")
+    ]
+    for stale_path in stale_paths:
+        stale_path.unlink()
+    for directory in sorted(output_dir.rglob("*"), key=lambda path: len(path.parts), reverse=True):
+        if directory.is_dir() and not any(directory.iterdir()):
+            directory.rmdir()
+    print(f"Wrote {len(pages)} reference pages to {output_dir} ({len(stale_paths)} removed)")

scripts/docs_skill/_helpers/render.py

@@ -1,50 +1,101 @@
-"""Markdown rendering for SQLBuild docs skills."""
+"""Markdown rendering for bundled SQLBuild docs reference pages."""
 
-from pathlib import Path
+import os
+import re
+from pathlib import Path, PurePosixPath
 
 from scripts.docs_skill._helpers.mdx import normalize_blank_lines, parse_mdx_page
-from scripts.docs_skill._helpers.navigation import list_ordered_page_paths
+from scripts.docs_skill._helpers.navigation import list_navigation_groups
 from scripts.docs_skill.constants import (
+    DOCS_BASE_URL,
     GENERATED_MARKER,
-    INTRODUCTION_PAGE_NAME,
-    SKILL_FRONTMATTER,
+    INDEX_FILENAME,
+    TABLE_OF_CONTENTS_MIN_LINES,
 )
-from scripts.docs_skill.models import MdxPage
+from scripts.docs_skill.models import MdxPage, NavigationGroup
 
+_INTERNAL_LINK: re.Pattern[str] = re.compile(r"\]\((/[^)\s#]*)(#[^)\s]*)?\)")
+_SECTION_HEADING: re.Pattern[str] = re.compile(r"^## (.+)$", re.MULTILINE)
 
-def build_skill_markdown(*, docs_root: Path, include_introduction: bool = True) -> str:
-    """Build one Markdown skill from all ordered documentation pages."""
 
-    page_paths: list[Path] = list_ordered_page_paths(docs_root=docs_root)
-    if not include_introduction:
-        page_paths = [
-            path for path in page_paths if path.with_suffix("").as_posix() != INTRODUCTION_PAGE_NAME
-        ]
+def build_reference_pages(*, docs_root: Path) -> dict[str, str]:
+    """Render every documentation page plus a grouped index, keyed by relative output path."""
 
-    sections: list[str] = [_render_skill_header(page_paths=page_paths)]
-    for page_path in page_paths:
-        absolute_path: Path = docs_root / page_path
-        page: MdxPage = parse_mdx_page(absolute_path.read_text(encoding="utf-8"))
-        sections.append(_render_page_section(page=page, source_path=page_path))
+    groups: list[NavigationGroup] = list_navigation_groups(docs_root=docs_root)
+    page_names: set[str] = set()
+    for group in groups:
+        page_names.update(path.with_suffix("").as_posix() for path in group.page_paths)
+    known_pages: frozenset[str] = frozenset(page_names)
+    pages: dict[str, MdxPage] = {}
+    rendered: dict[str, str] = {}
+    for group in groups:
+        for page_path in group.page_paths:
+            page_name: str = page_path.with_suffix("").as_posix()
+            page: MdxPage = parse_mdx_page((docs_root / page_path).read_text(encoding="utf-8"))
+            pages[page_name] = page
+            rendered[f"{page_name}.md"] = _render_page(
+                page=page, page_name=page_name, known_pages=known_pages
+            )
+    rendered[INDEX_FILENAME] = _render_index(groups=groups, pages=pages)
+    return rendered
 
-    return normalize_blank_lines("\n\n".join(sections)).strip() + "\n"
 
-
-def _render_skill_header(*, page_paths: list[Path]) -> str:
-    page_list: str = "\n".join(f"- `{path.with_suffix('').as_posix()}`" for path in page_paths)
+def _render_page(*, page: MdxPage, page_name: str, known_pages: frozenset[str]) -> str:
+    body: str = _rewrite_internal_links(
+        body=page.body, page_name=page_name, known_pages=known_pages
+    )
+    description_block: str = f"> {page.description}\n\n" if page.description else ""
+    contents_block: str = _table_of_contents(body=body)
     return (
-        f"{SKILL_FRONTMATTER}\n\n"
-        f"{GENERATED_MARKER}\n\n"
-        "# SQLBuild Skill\n\n"
-        "This file is generated from the SQLBuild documentation. Use it as the source of truth "
-        "for SQLBuild syntax, project structure, configuration, testing, adapters, and CLI "
-        f"behavior.\n\n## Included Pages\n\n{page_list}"
+        normalize_blank_lines(
+            f"{GENERATED_MARKER}\n\n# {page.title}\n\n{description_block}"
+            f"Online: {DOCS_BASE_URL}/{page_name}\n\n{contents_block}{body}"
+        ).strip()
+        + "\n"
     )
 
 
-def _render_page_section(*, page: MdxPage, source_path: Path) -> str:
-    description_block: str = f"\n\n{page.description}" if page.description else ""
-    body_block: str = f"\n\n{page.body}" if page.body else ""
-    return f"""## {page.title}
+def _table_of_contents(*, body: str) -> str:
+    if body.count("\n") < TABLE_OF_CONTENTS_MIN_LINES:
+        return ""
+    headings: list[str] = _SECTION_HEADING.findall(_without_code_blocks(body))
+    if not headings:
+        return ""
+    return "## Contents\n\n" + "".join(f"- {heading}\n" for heading in headings) + "\n"
+
+
+def _without_code_blocks(body: str) -> str:
+    return re.sub(r"```.*?```", "", body, flags=re.DOTALL)
+
+
+def _rewrite_internal_links(*, body: str, page_name: str, known_pages: frozenset[str]) -> str:
+    source_dir: PurePosixPath = PurePosixPath(page_name).parent
 
-Source: `{source_path.as_posix()}`{description_block}{body_block}"""
+    def replace(match: re.Match[str]) -> str:
+        target: str = match.group(1).strip("/") or "index"
+        anchor: str = match.group(2) or ""
+        if target not in known_pages:
+            return f"]({DOCS_BASE_URL}/{target}{anchor})"
+        relative: str = os.path.relpath(f"{target}.md", start=source_dir.as_posix() or ".")
+        return f"]({PurePosixPath(relative).as_posix()}{anchor})"
+
+    return _INTERNAL_LINK.sub(replace, body)
+
+
+def _render_index(*, groups: list[NavigationGroup], pages: dict[str, MdxPage]) -> str:
+    sections: list[str] = []
+    for group in groups:
+        lines: list[str] = [f"## {group.label}", ""]
+        for page_path in group.page_paths:
+            page_name: str = page_path.with_suffix("").as_posix()
+            page: MdxPage = pages[page_name]
+            summary: str = f" - {page.description}" if page.description else ""
+            lines.append(f"- [{page.title}]({page_name}.md) (`{page_name}`){summary}")
+        sections.append("\n".join(lines))
+    return (
+        f"{GENERATED_MARKER}\n\n# SQLBuild documentation index\n\n"
+        "Bundled copies of every page on the SQLBuild documentation site, matching the "
+        "installed SQLBuild version. Open only the page you need.\n\n"
+        + "\n\n".join(sections)
+        + "\n"
+    )

scripts/docs_skill/constants.py

@@ -1,20 +1,14 @@
-"""Stable constants for SQLBuild docs skill generation."""
+"""Stable constants for SQLBuild docs reference generation."""
 
 import tempfile
 from pathlib import Path
 
 DEFAULT_REPO_URL: str = "https://github.com/chio-labs/sqlbuild-docs"
 DEFAULT_CLONE_DIR: Path = Path(tempfile.gettempdir()) / "sqlbuild-docs-skill-source"
-DEFAULT_OUTPUT_PATH: Path = Path("src/sqlbuild/.agents/skills/sqlbuild/SKILL.md")
+DEFAULT_OUTPUT_DIR: Path = Path("src/sqlbuild/.agents/skills/sqlbuild/references/docs")
+DOCS_BASE_URL: str = "https://docs.sqlbuild.com"
 GENERATED_MARKER: str = "<!-- generated-by: sqlbuild skills -->"
-SKILL_DESCRIPTION: str = (
-    "ALWAYS load this skill when doing ANY SQLBuild work. This includes models, tests, audits, "
-    "scenarios, configuration, CLI behavior, adapters, dependencies, documentation, and related "
-    "code."
-)
-SKILL_FRONTMATTER: str = f"""---
-name: sqlbuild
-description: {SKILL_DESCRIPTION}
----"""
-INTRODUCTION_PAGE_NAME: str = "index"
+INDEX_FILENAME: str = "CONTENTS.md"
 FRONTMATTER_DELIMITER: str = "---"
+TABLE_OF_CONTENTS_MIN_LINES: int = 100
+UNGROUPED_LABEL: str = "Other pages"

scripts/docs_skill/main/generate.py

@@ -1,39 +1,39 @@
-"""Focused entry for SQLBuild docs skill generation."""
+"""Focused entry for SQLBuild docs reference generation."""
 
 from __future__ import annotations
 
 import argparse
 from pathlib import Path
 
 from scripts.docs_skill._helpers.clone import clone_docs_repo
-from scripts.docs_skill._helpers.output import write_skill_markdown
-from scripts.docs_skill._helpers.render import build_skill_markdown
-from scripts.docs_skill.constants import DEFAULT_CLONE_DIR, DEFAULT_OUTPUT_PATH, DEFAULT_REPO_URL
+from scripts.docs_skill._helpers.output import write_reference_pages
+from scripts.docs_skill._helpers.render import build_reference_pages
+from scripts.docs_skill.constants import DEFAULT_CLONE_DIR, DEFAULT_OUTPUT_DIR, DEFAULT_REPO_URL
 
 
 def generate_docs_skill(argv: list[str] | None = None) -> int:
-    """Generate and write the SQLBuild documentation skill."""
+    """Generate the bundled documentation reference pages for the SQLBuild skill."""
 
     args: argparse.Namespace = _parse_args(argv)
     docs_root: Path = args.docs_root or clone_docs_repo(
         repo_url=args.repo_url,
         clone_dir=args.clone_dir,
     )
-    skill_markdown: str = build_skill_markdown(
-        docs_root=docs_root,
-        include_introduction=not args.exclude_introduction,
+    write_reference_pages(
+        output_dir=args.output_dir,
+        pages=build_reference_pages(docs_root=docs_root),
     )
-    write_skill_markdown(output_path=args.output, contents=skill_markdown)
     return 0
 
 
 def _parse_args(argv: list[str] | None) -> argparse.Namespace:
     parser: argparse.ArgumentParser = argparse.ArgumentParser(
-        description="Clone sqlbuild-docs and convert all MDX pages into a single SKILL.md."
+        description=(
+            "Clone sqlbuild-docs and convert each MDX page into a bundled skill reference page."
+        )
     )
     parser.add_argument("--repo-url", default=DEFAULT_REPO_URL)
     parser.add_argument("--clone-dir", type=Path, default=DEFAULT_CLONE_DIR)
     parser.add_argument("--docs-root", type=Path)
-    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT_PATH)
-    parser.add_argument("--exclude-introduction", action="store_true")
+    parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
     return parser.parse_args(argv)

scripts/docs_skill/models.py

@@ -1,6 +1,7 @@
-"""Structured models for SQLBuild docs skill generation."""
+"""Structured models for SQLBuild docs reference generation."""
 
 from dataclasses import dataclass
+from pathlib import Path
 
 
 @dataclass(frozen=True, slots=True)
@@ -10,3 +11,11 @@ class MdxPage:
     title: str
     description: str
     body: str
+
+
+@dataclass(frozen=True, slots=True)
+class NavigationGroup:
+    """One navigation group label and its pages in documentation order."""
+
+    label: str
+    page_paths: tuple[Path, ...]