docs: rewrite the README and build the agent skill from the website docs

scripts/docs_skill/_helpers/mdx.py

@@ -2,7 +2,7 @@
 
 import re
 
-from scripts.docs_skill.constants import FRONTMATTER_DELIMITER
+from scripts.docs_skill.constants import COMPONENT_TAG_END, FRONTMATTER_DELIMITER
 from scripts.docs_skill.models import MdxPage
 
 
@@ -41,6 +41,7 @@ def _split_frontmatter(contents: str) -> tuple[dict[str, str], str]:
 def _clean_mdx_body(body: str) -> str:
     cleaned_lines: list[str] = []
     in_code_block: bool = False
+    in_component_tag: bool = False
 
     for raw_line in body.splitlines():
         line: str = raw_line.rstrip()
@@ -53,6 +54,12 @@ def _clean_mdx_body(body: str) -> str:
             continue
 
         stripped: str = line.strip()
+        if in_component_tag:
+            in_component_tag = COMPONENT_TAG_END not in stripped
+            continue
+        if re.match(r"<[A-Z][\w.]*\b", stripped) and COMPONENT_TAG_END not in stripped:
+            in_component_tag = True
+            continue
         if _is_mdx_only_line(stripped):
             continue
         cleaned_lines.append(line)

scripts/docs_skill/_helpers/navigation.py

@@ -4,83 +4,61 @@
 from pathlib import Path
 from typing import Any
 
-from scripts.docs_skill.constants import UNGROUPED_LABEL
+from scripts.docs_skill.constants import DOCS_SLUG, INDEX_PAGE, UNGROUPED_LABEL
 from scripts.docs_skill.models import NavigationGroup
 
 
-def list_ordered_page_paths(*, docs_root: Path) -> list[Path]:
-    """Return existing MDX pages in navigation order followed by unlisted pages."""
+def list_ordered_page_paths(*, docs_root: Path, sidebar_path: Path | None = None) -> list[Path]:
+    """Return existing pages in sidebar order followed by unlisted pages."""
 
-    docs_json_path: Path = docs_root / "docs.json"
-    if docs_json_path.exists():
-        docs_json: dict[str, Any] = json.loads(docs_json_path.read_text(encoding="utf-8"))
-        ordered_pages: list[str] = _collect_navigation_pages(docs_json.get("navigation", {}))
-    else:
-        ordered_pages = sorted(path.with_suffix("").as_posix() for path in docs_root.rglob("*.mdx"))
-
-    page_paths: list[Path] = []
-    seen_pages: set[str] = set()
-    for page in ordered_pages:
-        page_path: Path = Path(f"{page}.mdx")
-        if page in seen_pages or not (docs_root / page_path).exists():
-            continue
-        page_paths.append(page_path)
-        seen_pages.add(page)
-
-    for mdx_path in sorted(docs_root.rglob("*.mdx")):
-        page: str = mdx_path.relative_to(docs_root).with_suffix("").as_posix()
-        if page not in seen_pages:
-            page_paths.append(mdx_path.relative_to(docs_root))
-            seen_pages.add(page)
-
-    return page_paths
+    paths: list[Path] = []
+    for group in list_navigation_groups(docs_root=docs_root, sidebar_path=sidebar_path):
+        paths.extend(group.page_paths)
+    return paths
 
 
 def _collect_navigation_pages(node: Any) -> list[str]:
-    pages: list[str] = []
-
     if isinstance(node, str):
-        return [node]
+        return [INDEX_PAGE if node == DOCS_SLUG else node.removeprefix(f"{DOCS_SLUG}/")]
     if isinstance(node, list):
+        pages: list[str] = []
         for item in node:
             pages.extend(_collect_navigation_pages(item))
         return pages
     if isinstance(node, dict):
-        for key in ("root", "groups", "pages"):
-            pages.extend(_collect_navigation_pages(node.get(key, [])))
+        return _collect_navigation_pages(node.get("slug", node.get("items", [])))
+    return []
 
-    return pages
 
+def list_navigation_groups(
+    *, docs_root: Path, sidebar_path: Path | None = None
+) -> list[NavigationGroup]:
+    """Flatten nested sidebar groups within each top-level group, then add unlisted pages."""
 
-def list_navigation_groups(*, docs_root: Path) -> list[NavigationGroup]:
-    """Return pages grouped by top-level navigation group, then unlisted pages."""
-
-    ordered_paths: list[Path] = list_ordered_page_paths(docs_root=docs_root)
-    docs_json_path: Path = docs_root / "docs.json"
-    top_level_nodes: list[Any] = []
-    if docs_json_path.exists():
-        navigation: Any = json.loads(docs_json_path.read_text(encoding="utf-8")).get(
-            "navigation", {}
-        )
-        top_level_nodes = navigation.get("groups", []) if isinstance(navigation, dict) else []
-
+    sidebar_path = sidebar_path or docs_root.resolve().parents[2] / "sidebar.json"
+    nodes: list[Any] = (
+        json.loads(sidebar_path.read_text(encoding="utf-8")) if sidebar_path.exists() else []
+    )
+    available: dict[str, Path] = {}
+    for suffix in ("*.md", "*.mdx"):
+        for path in sorted(docs_root.rglob(suffix)):
+            relative_path: Path = path.relative_to(docs_root)
+            available[relative_path.with_suffix("").as_posix()] = relative_path
     groups: list[NavigationGroup] = []
-    grouped_pages: set[str] = set()
-    for node in top_level_nodes:
-        label: str = str(node.get("group", UNGROUPED_LABEL)) if isinstance(node, dict) else ""
-        member_pages: set[str] = set(_collect_navigation_pages(node))
-        page_paths: tuple[Path, ...] = tuple(
-            path
-            for path in ordered_paths
-            if path.with_suffix("").as_posix() in member_pages
-            and path.with_suffix("").as_posix() not in grouped_pages
-        )
-        grouped_pages.update(path.with_suffix("").as_posix() for path in page_paths)
-        if page_paths:
-            groups.append(NavigationGroup(label=label, page_paths=page_paths))
-
+    seen: set[str] = set()
+    for node in nodes:
+        members: list[Path] = []
+        for page in _collect_navigation_pages(node):
+            if page in available and page not in seen:
+                members.append(available[page])
+                seen.add(page)
+        if members:
+            label: str = (
+                node.get("label", UNGROUPED_LABEL) if isinstance(node, dict) else UNGROUPED_LABEL
+            )
+            groups.append(NavigationGroup(label=label, page_paths=tuple(members)))
     remaining: tuple[Path, ...] = tuple(
-        path for path in ordered_paths if path.with_suffix("").as_posix() not in grouped_pages
+        available[page] for page in sorted(available) if page not in seen
     )
     if remaining:
         groups.append(NavigationGroup(label=UNGROUPED_LABEL, page_paths=remaining))

scripts/docs_skill/_helpers/render.py

@@ -8,8 +8,11 @@
 from scripts.docs_skill._helpers.navigation import list_navigation_groups
 from scripts.docs_skill.constants import (
     DOCS_BASE_URL,
+    DOCS_URL_PATH,
     GENERATED_MARKER,
     INDEX_FILENAME,
+    INDEX_PAGE,
+    SITE_BASE_URL,
     TABLE_OF_CONTENTS_MIN_LINES,
 )
 from scripts.docs_skill.models import MdxPage, NavigationGroup
@@ -18,10 +21,12 @@
 _SECTION_HEADING: re.Pattern[str] = re.compile(r"^## (.+)$", re.MULTILINE)
 
 
-def build_reference_pages(*, docs_root: Path) -> dict[str, str]:
+def build_reference_pages(*, docs_root: Path, sidebar_path: Path | None = None) -> dict[str, str]:
     """Render every documentation page plus a grouped index, keyed by relative output path."""
 
-    groups: list[NavigationGroup] = list_navigation_groups(docs_root=docs_root)
+    groups: list[NavigationGroup] = list_navigation_groups(
+        docs_root=docs_root, sidebar_path=sidebar_path
+    )
     page_names: set[str] = set()
     for group in groups:
         page_names.update(path.with_suffix("").as_posix() for path in group.page_paths)
@@ -49,7 +54,8 @@ def _render_page(*, page: MdxPage, page_name: str, known_pages: frozenset[str])
     return (
         normalize_blank_lines(
             f"{GENERATED_MARKER}\n\n# {page.title}\n\n{description_block}"
-            f"Online: {DOCS_BASE_URL}/{page_name}\n\n{contents_block}{body}"
+            f"Online: {DOCS_BASE_URL}/{'' if page_name == INDEX_PAGE else page_name + '/'}"
+            f"\n\n{contents_block}{body}"
         ).strip()
         + "\n"
     )
@@ -72,10 +78,13 @@ def _rewrite_internal_links(*, body: str, page_name: str, known_pages: frozenset
     source_dir: PurePosixPath = PurePosixPath(page_name).parent
 
     def replace(match: re.Match[str]) -> str:
-        target: str = match.group(1).strip("/") or "index"
+        path: str = match.group(1)
         anchor: str = match.group(2) or ""
+        if path != DOCS_URL_PATH and not path.startswith(f"{DOCS_URL_PATH}/"):
+            return f"]({SITE_BASE_URL}{path}{anchor})"
+        target: str = path.removeprefix(DOCS_URL_PATH).strip("/") or INDEX_PAGE
         if target not in known_pages:
-            return f"]({DOCS_BASE_URL}/{target}{anchor})"
+            return f"]({DOCS_BASE_URL}/{target}/{anchor})"
         relative: str = os.path.relpath(f"{target}.md", start=source_dir.as_posix() or ".")
         return f"]({PurePosixPath(relative).as_posix()}{anchor})"
 

scripts/docs_skill/constants.py

@@ -1,12 +1,15 @@
 """Stable constants for SQLBuild docs reference generation."""
 
-import tempfile
 from pathlib import Path
 
-DEFAULT_REPO_URL: str = "https://github.com/chio-labs/sqlbuild-docs"
-DEFAULT_CLONE_DIR: Path = Path(tempfile.gettempdir()) / "sqlbuild-docs-skill-source"
+DEFAULT_DOCS_ROOT: Path = Path(__file__).resolve().parents[2] / "website/src/content/docs/docs"
 DEFAULT_OUTPUT_DIR: Path = Path("src/sqlbuild/.agents/skills/sqlbuild/references/docs")
-DOCS_BASE_URL: str = "https://docs.sqlbuild.com"
+SITE_BASE_URL: str = "https://sqlbuild.com"
+DOCS_BASE_URL: str = f"{SITE_BASE_URL}/docs"
+DOCS_SLUG: str = "docs"
+DOCS_URL_PATH: str = f"/{DOCS_SLUG}"
+INDEX_PAGE: str = "index"
+COMPONENT_TAG_END: str = ">"
 GENERATED_MARKER: str = "<!-- generated-by: sqlbuild skills -->"
 INDEX_FILENAME: str = "CONTENTS.md"
 FRONTMATTER_DELIMITER: str = "---"

scripts/docs_skill/main/generate.py

@@ -5,35 +5,28 @@
 import argparse
 from pathlib import Path
 
-from scripts.docs_skill._helpers.clone import clone_docs_repo
 from scripts.docs_skill._helpers.output import write_reference_pages
 from scripts.docs_skill._helpers.render import build_reference_pages
-from scripts.docs_skill.constants import DEFAULT_CLONE_DIR, DEFAULT_OUTPUT_DIR, DEFAULT_REPO_URL
+from scripts.docs_skill.constants import DEFAULT_DOCS_ROOT, DEFAULT_OUTPUT_DIR
 
 
 def generate_docs_skill(argv: list[str] | None = None) -> int:
     """Generate the bundled documentation reference pages for the SQLBuild skill."""
 
     args: argparse.Namespace = _parse_args(argv)
-    docs_root: Path = args.docs_root or clone_docs_repo(
-        repo_url=args.repo_url,
-        clone_dir=args.clone_dir,
-    )
     write_reference_pages(
         output_dir=args.output_dir,
-        pages=build_reference_pages(docs_root=docs_root),
+        pages=build_reference_pages(docs_root=args.docs_root),
     )
     return 0
 
 
 def _parse_args(argv: list[str] | None) -> argparse.Namespace:
     parser: argparse.ArgumentParser = argparse.ArgumentParser(
         description=(
-            "Clone sqlbuild-docs and convert each MDX page into a bundled skill reference page."
+            "Convert local SQLBuild website documentation into bundled skill reference pages."
         )
     )
-    parser.add_argument("--repo-url", default=DEFAULT_REPO_URL)
-    parser.add_argument("--clone-dir", type=Path, default=DEFAULT_CLONE_DIR)
-    parser.add_argument("--docs-root", type=Path)
+    parser.add_argument("--docs-root", type=Path, default=DEFAULT_DOCS_ROOT)
     parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
     return parser.parse_args(argv)

src/sqlbuild/.agents/skills/sqlbuild/SKILL.md

@@ -163,4 +163,4 @@ diff.
 - Full documentation for this SQLBuild version: [references/docs/CONTENTS.md](references/docs/CONTENTS.md)
   lists every page; open only the page you need, for example `references/docs/cli/build.md` or
   `references/docs/concepts/snapshots.md`. The same pages are online at
-  https://docs.sqlbuild.com if you need a newer version.
+  https://sqlbuild.com/docs/ if you need a newer version.