From c3b8f7e797f30d3ce80cf847c4225596749f4699 Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Sun, 18 Oct 2026 13:46:27 +0200 Subject: [PATCH 1/6] Tutorial tests: run beginner-path pages against a real Gateway in CI --- .github/workflows/tutorials.yml | 70 +++++ .gitignore | 4 + .../first-7-days/day-1-make-it-yours.md | 4 +- .../day-1-make-it-yours.test.sh | 48 +++ .../tutorial-tests/day-3-automation.test.sh | 56 ++++ scripts/tutorial-tests/day-4-skills.test.sh | 42 +++ .../tutorial-tests/day-7-production.test.sh | 55 ++++ scripts/tutorial-tests/extract.mjs | 102 +++++++ scripts/tutorial-tests/heartbeat.test.sh | 65 ++++ scripts/tutorial-tests/lib.sh | 278 ++++++++++++++++++ scripts/tutorial-tests/quick-start.test.sh | 64 ++++ scripts/tutorial-tests/run.mjs | 85 ++++++ scripts/tutorial-tests/tt.mjs | 90 ++++++ 13 files changed, 961 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/tutorials.yml create mode 100755 scripts/tutorial-tests/day-1-make-it-yours.test.sh create mode 100755 scripts/tutorial-tests/day-3-automation.test.sh create mode 100755 scripts/tutorial-tests/day-4-skills.test.sh create mode 100755 scripts/tutorial-tests/day-7-production.test.sh create mode 100644 scripts/tutorial-tests/extract.mjs create mode 100755 scripts/tutorial-tests/heartbeat.test.sh create mode 100755 scripts/tutorial-tests/lib.sh create mode 100755 scripts/tutorial-tests/quick-start.test.sh create mode 100644 scripts/tutorial-tests/run.mjs create mode 100644 scripts/tutorial-tests/tt.mjs diff --git a/.github/workflows/tutorials.yml b/.github/workflows/tutorials.yml new file mode 100644 index 0000000..8153a70 --- /dev/null +++ b/.github/workflows/tutorials.yml @@ -0,0 +1,70 @@ +name: Tutorial tests + +# Runs the beginner path's commands for real against the pinned OpenClaw +# (scripts/openclaw-version.txt): onboarding, a Gateway on loopback with token +# auth, automations, skills, backups, audits. One script per page in +# scripts/tutorial-tests/. Steps that need a model key or a channel account are +# listed as skipped, with the reason. See scripts/README.md. +# +# Report only for now: the job passes even when a page fails, and the summary +# and tutorial-report.json say which steps failed. To make it blocking, set the +# repository variable TUTORIALS_STRICT=1 (Settings > Secrets and variables > +# Actions > Variables), or run it manually with "strict" ticked. + +on: + push: + branches: [gh-pages, q/ci-tutorials] # TEMP: q/ci-tutorials while iterating + paths: + - 'docs/**' + - 'scripts/tutorial-tests/**' + - 'scripts/openclaw-version.txt' + - '.github/workflows/tutorials.yml' + pull_request: + paths: + - 'docs/**' + - 'scripts/tutorial-tests/**' + - 'scripts/openclaw-version.txt' + - '.github/workflows/tutorials.yml' + schedule: + - cron: '17 5 * * 1' # Mondays 05:17 UTC + workflow_dispatch: + inputs: + strict: + description: Fail the job when a page fails + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: tutorials-${{ github.ref }} + cancel-in-progress: true + +jobs: + tutorials: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + TUTORIALS_STRICT: ${{ (github.event_name == 'workflow_dispatch' && inputs.strict) && '1' || vars.TUTORIALS_STRICT || '0' }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: 24 # openclaw needs >=24.16 + - name: Install pinned OpenClaw + run: | + npm i --prefix tools/openclaw --no-save --ignore-scripts --no-audit --no-fund \ + "openclaw@$(tr -d '[:space:]' < scripts/openclaw-version.txt)" + tools/openclaw/node_modules/.bin/openclaw --version + node --version; npm --version + - name: Run tutorial tests + run: node scripts/tutorial-tests/run.mjs --json tutorial-report.json + - uses: actions/upload-artifact@v7 + if: always() + with: + name: tutorial-report + path: | + tutorial-report.json + tutorial-report/ + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index c90cb90..bf2e203 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ yarn-error.log* # Snippet gates /tools/openclaw snippet-report.json + +# Tutorial tests +tutorial-report.json +/tutorial-report diff --git a/docs/guides/first-7-days/day-1-make-it-yours.md b/docs/guides/first-7-days/day-1-make-it-yours.md index 3ea8d72..82aee59 100644 --- a/docs/guides/first-7-days/day-1-make-it-yours.md +++ b/docs/guides/first-7-days/day-1-make-it-yours.md @@ -53,7 +53,7 @@ The workspace holds a handful of Markdown files that load at the start of every | `MEMORY.md` | Curated long-term facts and decisions (optional). | | `memory/YYYY-MM-DD.md` | Daily notes the agent writes as it goes. | -You can edit them with any text editor, or in the Control UI under **Settings → Agents → Files**. If you delete one by accident, `openclaw setup` recreates missing defaults without overwriting the files you kept. +You can edit them with any text editor, or in the Control UI under **Settings → Agents → Files**. If you delete one by accident, `openclaw setup --baseline` recreates missing defaults without overwriting the files you kept. (Bare `openclaw setup` on a configured install opens a chat instead.) --- @@ -127,7 +127,7 @@ Edit `~/.openclaw/workspace/IDENTITY.md`: - **Avatar:** avatars/nova.png ``` -Replace `(pick one)` with a single emoji. Placeholder text in parentheses is ignored, not saved. The avatar is a workspace-relative path, an `http(s)` URL, or a data URI; drop the line if you don't have one. +Replace `(pick one)` with a single emoji, or delete the line. Only the template's own placeholder text is ignored; anything else you leave there, including `(pick one)`, is saved as the value. The avatar is a workspace-relative path, an `http(s)` URL, or a data URI; drop the line if you don't have one. To push the file into the agent's config (used by the Control UI and other surfaces), run: diff --git a/scripts/tutorial-tests/day-1-make-it-yours.test.sh b/scripts/tutorial-tests/day-1-make-it-yours.test.sh new file mode 100755 index 0000000..30439f0 --- /dev/null +++ b/scripts/tutorial-tests/day-1-make-it-yours.test.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tutorial test: docs/guides/first-7-days/day-1-make-it-yours.md +source "$(dirname "$0")/lib.sh" +page docs/guides/first-7-days/day-1-make-it-yours.md + +onboard +setup -e '^ok$' "onboarding seeded the workspace (BOOTSTRAP.md, AGENTS.md, SOUL.md, IDENTITY.md, USER.md)" \ + "cd ~/.openclaw/workspace && ls BOOTSTRAP.md AGENTS.md SOUL.md IDENTITY.md USER.md > /dev/null && echo ok" +gateway_start + +# Meet your bot +skip "openclaw dashboard" "opens a browser" +skip "openclaw tui" "interactive terminal UI" +skip "openclaw agent --agent main --message \"Who are you?\"" "needs a model provider key" + +# The workspace files: deleting one and recreating it with setup +setup "delete AGENTS.md and keep a custom SOUL.md" \ + "rm ~/.openclaw/workspace/AGENTS.md && echo '# kept' > ~/.openclaw/workspace/SOUL.md" +run "openclaw setup --baseline" +setup -e '^ok$' "AGENTS.md is back and SOUL.md was not overwritten" \ + "test -f ~/.openclaw/workspace/AGENTS.md && grep -qx '# kept' ~/.openclaw/workspace/SOUL.md && echo ok" + +# Write your SOUL.md +file_block "~/.openclaw/workspace/SOUL.md" +skip "openclaw agent --agent main --message \"Who are you?\"" "needs a model provider key" + +# Set your identity +file_block "~/.openclaw/workspace/IDENTITY.md" +run -e 'Name: Nova' "openclaw agents set-identity --agent main --from-identity" +setup -e '"name": "Nova"' "identity stored in config" "openclaw config get agents.entries.main.identity" + +# Tell it about you +file_block "~/.openclaw/workspace/USER.md" +run "openclaw config set agents.defaults.userTimezone Europe/Berlin" +setup -e 'Europe/Berlin' "timezone stored" "openclaw config get agents.defaults.userTimezone" +setup "config still valid with the gateway running" "openclaw config validate && openclaw health" + +# Have a real conversation +skip_step "The five example prompts" "need a model provider key" + +# Verify it remembers +skip "openclaw agent --agent main --message \"What do you know about me?\"" "needs a model provider key" +skip "cat ~/.openclaw/workspace/MEMORY.md" "MEMORY.md is written by the agent during a conversation; a fresh install has none" +skip "ls ~/.openclaw/workspace/memory/" "daily notes are written by the agent during a conversation" +run "openclaw memory search \"timezone\"" +run "openclaw memory index --force" + +finish diff --git a/scripts/tutorial-tests/day-3-automation.test.sh b/scripts/tutorial-tests/day-3-automation.test.sh new file mode 100755 index 0000000..479e726 --- /dev/null +++ b/scripts/tutorial-tests/day-3-automation.test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Tutorial test: docs/guides/first-7-days/day-3-automation.md +source "$(dirname "$0")/lib.sh" +page docs/guides/first-7-days/day-3-automation.md + +onboard +gateway_start + +# Before you start: where messages go +run "openclaw config set commands.ownerAllowFrom '[\"telegram:123456789\"]'" + +# Your first automation (CLI form). Delivery targets a Telegram chat that does +# not exist here; creating and listing the job must still work. +run -e 'Morning brief' "openclaw cron add \"0 8 * * 1-5\"..." +run -e 'Morning brief' "openclaw cron list" +capture jobId "openclaw cron list --json" '(j.jobs||j).find(x=>x.name==="Morning brief").id' +skip "openclaw cron run --wait" "runs the job's prompt: needs a model provider key" + +# Practical automation ideas: each one is created for real +run "openclaw cron add \"0 8 * * *\"..." +run "openclaw cron add \"every 2h\" \"Check github.com/myorg/myrepo..." +run "openclaw cron add \"0 * * * *\"..." +run "openclaw cron add \"every 2h\" \"Check the current BTC..." +run "openclaw cron add \"0 21 * * *\"..." +run "openclaw cron add \"0 18 * * 0\"..." +run -e 'Weekly summary' "openclaw cron list --all" + +# The heartbeat monitor +config_block 1 +run "openclaw config validate" +capture heartbeatJobId "openclaw cron list --all --json" '(j.jobs||j).find(x=>/Heartbeat/.test(x.name||x.displayName||"")).id' +run "openclaw cron scratch --set \"If a background job..." +setup -e 'background job' "scratch reads back" "openclaw cron scratch $PH_heartbeatJobId" + +# Model choice and cost +run "openclaw gateway usage-cost --days 7" + +# Managing automations +run -e 'Morning brief' "openclaw cron list" +run -e 'Morning brief' "openclaw cron show " +run "openclaw cron disable " +run "openclaw cron enable " +skip "openclaw cron edit " "no fields given on the page; see the cron edit reference" + +# Debugging +run "openclaw cron status" +run "openclaw cron runs " +skip "openclaw cron run --wait" "runs the job's prompt: needs a model provider key" +run "openclaw system heartbeat last" +skip "openclaw system event --text \"Check whether my backup finished\" --mode now" "wakes a heartbeat turn: needs a model provider key" +skip "openclaw logs --follow" "streams until interrupted" +run "openclaw gateway status" +run "openclaw cron rm " +setup -e '^gone$' "the job is gone" "openclaw cron list --all --json | grep -q '\"$PH_jobId\"' && echo still-there || echo gone" + +finish diff --git a/scripts/tutorial-tests/day-4-skills.test.sh b/scripts/tutorial-tests/day-4-skills.test.sh new file mode 100755 index 0000000..e295a8d --- /dev/null +++ b/scripts/tutorial-tests/day-4-skills.test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tutorial test: docs/guides/first-7-days/day-4-skills.md +source "$(dirname "$0")/lib.sh" +page docs/guides/first-7-days/day-4-skills.md + +onboard +gateway_start + +# Start with the skills you already have +run -e 'weather' "openclaw skills list" +run "openclaw skills check" +run -e 'github' "openclaw skills info github" +skip "openclaw agent --agent main --message \"What's the weather in Berlin this afternoon?\"" "needs a model provider key" + +# Find more on ClawHub (network: clawhub.ai) +run "openclaw skills search \"calendar\"" +run "openclaw skills search \"rss digest\" --limit 10" + +# Check before you install +run -x 0,1 "openclaw skills verify @asleep123/caldav-calendar --card" + +# Install your first skill +run "openclaw skills install @asleep123/caldav-calendar" +setup -e '^ok$' "skill landed in the workspace" "test -f ~/.openclaw/workspace/skills/caldav-calendar/SKILL.md && echo ok" +run -e 'caldav' "openclaw skills list" + +# Managing skills +run "openclaw skills list" +run "openclaw skills list --eligible" +run "openclaw skills update --all" +run "openclaw skills update @asleep123/caldav-calendar" +config_block 1 +setup -e 'false' "weather is disabled in config" "openclaw config get skills.entries.weather.enabled" + +skip "npm i -g clawhub" "installs the third-party ClawHub CLI globally; out of scope for this test" +skip "clawhub uninstall @asleep123/caldav-calendar" "needs the ClawHub CLI (above) and asks for confirmation" + +# Building vs installing: the minimal skill must load +file_block "~/.openclaw/workspace/skills/my-custom-skill/SKILL.md" +setup -e 'my-custom-skill' "custom skill is listed" "openclaw skills list" + +finish diff --git a/scripts/tutorial-tests/day-7-production.test.sh b/scripts/tutorial-tests/day-7-production.test.sh new file mode 100755 index 0000000..cf77e10 --- /dev/null +++ b/scripts/tutorial-tests/day-7-production.test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Tutorial test: docs/guides/first-7-days/day-7-production.md +source "$(dirname "$0")/lib.sh" +page docs/guides/first-7-days/day-7-production.md + +onboard +skip "openclaw update" "would replace the pinned version under test" +skip "openclaw plugins update --all" "nothing installed from a registry on a fresh install; covered by the upgrade section skip" + +# A baseline config: the token comes from ~/.openclaw/.env, as the comment says +setup "put the Gateway token in ~/.openclaw/.env" \ + "printf 'OPENCLAW_GATEWAY_TOKEN=%s\n' \"\$(openclaw config get gateway.auth.token)\" > ~/.openclaw/.env && chmod 600 ~/.openclaw/.env" +config_block 1 +setup -e 'OPENCLAW_GATEWAY_TOKEN' "openclaw.json no longer holds the token itself" "openclaw config get gateway.auth.token --json 2>/dev/null || grep -o 'OPENCLAW_GATEWAY_TOKEN' ~/.openclaw/openclaw.json" +run "openclaw config validate" +gateway_start +skip "openclaw gateway restart" "needs an installed service; CI started the Gateway with the new config instead" +run -e 'ask' "openclaw approvals get" +run -x 0,1 "openclaw security audit --deep" +run "openclaw security audit --fix" +run "openclaw secrets audit --check" + +# Cost check +run "openclaw gateway usage-cost" +run "openclaw gateway usage-cost --days 7 --all-agents" + +# Monitoring +run "openclaw status" +run "openclaw status --deep" +run "openclaw gateway status" +run "openclaw doctor" +skip "openclaw logs --follow" "streams until interrupted" +run -e 'Daily self-check' "openclaw cron add \"30 7 * * *\"..." + +# Backups +run "mkdir -p ~/Backups/openclaw" +run -e 'erif' "openclaw backup create --output ~/Backups/openclaw --verify" +capture archive "ls -t ~/Backups/openclaw/*.tar.gz | head -1 | xargs -n1 basename | sed 's/\\.tar\\.gz\$//' | node -e 'process.stdout.write(JSON.stringify([require(\"fs\").readFileSync(0,\"utf8\").trim()]))'" 'j[0]' +setup "openclaw backup verify (inline on the page)" "openclaw backup verify ~/Backups/openclaw/$PH_archive.tar.gz" + +# Upgrades +run "openclaw backup create --output ~/Backups/openclaw --verify" +run "openclaw update status" +skip "gh release view --repo openclaw/openclaw" "GitHub CLI call, not OpenClaw; needs a GH token" +run "openclaw update --dry-run" +skip "openclaw update" "would replace the pinned version under test" +skip "openclaw plugins update --all" "would update bundled plugins away from the pin" +run "openclaw skills update --all" +run "openclaw doctor --fix" +run "openclaw status --deep" +skip "openclaw gateway stop" "stops the installed service; CI stops its foreground Gateway at the end" +run -e 'restore|Restore' "openclaw backup restore ~/Backups/openclaw/.tar.gz --target ~/openclaw-restore" +setup -e '^ok$' "restore wrote a staging copy" "test -d ~/openclaw-restore && echo ok" + +finish diff --git a/scripts/tutorial-tests/extract.mjs b/scripts/tutorial-tests/extract.mjs new file mode 100644 index 0000000..a93a605 --- /dev/null +++ b/scripts/tutorial-tests/extract.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node +// extract.mjs: print the runnable parts of a docs page as JSON. +// +// node scripts/tutorial-tests/extract.mjs docs/getting-started/quick-start.md +// +// Output: { page, title, commands: [{ line, text }], blocks: [{ lang, title, line, body }] } +// commands: every command line in bash/sh/shell/zsh/console blocks, with +// `\` continuations joined, multi-line quoted strings kept +// together (joined with "\n"), comment-only lines dropped and +// trailing `# comments` stripped. `line` is the 1-based page line. +// blocks: every fenced block, with its `title="..."` and raw body. +// +// Used by lib.sh so tests run the page's own text instead of a copy. +// No dependencies. + +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +export function extractPage(file) { + const src = readFileSync(file, 'utf8'); + const lines = src.split('\n'); + + const fm = /^---\n([\s\S]*?)\n---/.exec(src); + const title = fm ? ((/^title:\s*"?(.*?)"?\s*$/m.exec(fm[1]) || [])[1] || '') : ''; + + const SHELL = new Set(['bash', 'sh', 'shell', 'zsh', 'console']); + const blocks = []; + for (let i = 0; i < lines.length; i++) { + const m = /^(\s*)(`{3,}|~{3,})\s*([^\s`{]*)\s*(.*)$/.exec(lines[i]); + if (!m) continue; + const [, indent, fence, lang, meta] = m; + const close = new RegExp(`^\\s*${fence[0] === '`' ? '`' : '~'}{${fence.length},}\\s*$`); + const start = i; + const body = []; + for (i++; i < lines.length && !close.test(lines[i]); i++) { + body.push(lines[i].startsWith(indent) ? lines[i].slice(indent.length) : lines[i].trimStart()); + } + const t = (/title=(?:"([^"]*)"|'([^']*)'|(\S+))/.exec(meta) || []).slice(1).find((x) => x !== undefined) || ''; + blocks.push({ lang: lang.toLowerCase(), title: t, meta, line: start + 1, body: body.join('\n') }); + } + + // Quote state after scanning `s`, starting in state `q` ('' | "'" | '"'). + function scanQuotes(s, q) { + for (let k = 0; k < s.length; k++) { + const c = s[k]; + if (q === '"' && c === '\\') { k++; continue; } + if (!q && c === '\\') { k++; continue; } + if (q) { if (c === q) q = ''; continue; } + if (c === '"' || c === "'") q = c; + } + return q; + } + + // Strip a trailing ` # comment` that is outside quotes. + function stripComment(s) { + let q = ''; + for (let k = 0; k < s.length; k++) { + const c = s[k]; + if (c === '\\' && q !== "'") { k++; continue; } + if (q) { if (c === q) q = ''; continue; } + if (c === '"' || c === "'") { q = c; continue; } + if (c === '#' && (k === 0 || /\s/.test(s[k - 1]))) return s.slice(0, k).trimEnd(); + } + return s; + } + + const commands = []; + for (const b of blocks) { + if (!SHELL.has(b.lang)) continue; + const body = b.body.split('\n'); + for (let k = 0; k < body.length; k++) { + let text = body[k]; + if (b.lang === 'console') { if (!/^\s*\$ /.test(text)) continue; text = text.replace(/^\s*\$ /, ''); } + if (!text.trim() || /^\s*#/.test(text)) continue; + const line = b.line + 1 + k; + let q = scanQuotes(text, ''); + while (k + 1 < body.length && (q || /\\\s*$/.test(text))) { + if (q) { text += '\n' + body[++k]; } + else { text = text.replace(/\\\s*$/, '') + ' ' + body[++k].trim(); } + q = scanQuotes(text, ''); + } + commands.push({ line, text: stripComment(text).trim() }); + } + } + + // Inline `openclaw ...` code spans outside fences (tables, prose). Tests may + // run these too, but they are not counted as "uncovered" when left out. + const inline = []; + const fenced = new Set(); + for (const b of blocks) for (let n = b.line; n <= b.line + b.body.split('\n').length + 1; n++) fenced.add(n); + lines.forEach((l, idx) => { + if (fenced.has(idx + 1)) return; + for (const m of l.matchAll(/`(openclaw [^`]+)`/g)) inline.push({ line: idx + 1, text: m[1].trim() }); + }); + + return { page: file, title, commands, inline, blocks }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + if (!process.argv[2]) { console.error('usage: extract.mjs '); process.exit(2); } + process.stdout.write(JSON.stringify(extractPage(process.argv[2]), null, 2) + '\n'); +} diff --git a/scripts/tutorial-tests/heartbeat.test.sh b/scripts/tutorial-tests/heartbeat.test.sh new file mode 100755 index 0000000..7782bca --- /dev/null +++ b/scripts/tutorial-tests/heartbeat.test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Tutorial test: docs/guides/heartbeat.md +source "$(dirname "$0")/lib.sh" +page docs/guides/heartbeat.md + +onboard +gateway_start + +# How heartbeat works: one system-owned job per agent +run -e 'Heartbeat \(main\)' "openclaw cron list --all" + +# Quick setup +config_block 1 +run "openclaw config set commands.ownerAllowFrom '[\"telegram:123456789\"]'" +skip "openclaw gateway restart" "needs an installed service; the running Gateway reloads config itself" +run "openclaw cron list --all" +capture jobId "openclaw cron list --all --json" '(j.jobs||j).find(x=>/Heartbeat \(main\)/.test(x.name||x.displayName||"")).id' +run "openclaw cron scratch --set \"- Anything urgent..." +skip "openclaw system event --text \"Check for urgent follow-ups\" --mode now" "wakes a heartbeat turn: needs a model provider key" +run "openclaw system heartbeat last" + +# Monitor scratch +run -e 'Anything urgent' "openclaw cron scratch " +run "openclaw cron scratch --set \"...\"" +setup "write checklist.md" "printf -- '- If a task is blocked, say what is missing.\n' > checklist.md" +run "openclaw cron scratch --file checklist.md" +setup -e 'blocked' "scratch now holds the file" "openclaw cron scratch $PH_jobId" +run "openclaw cron scratch --unset" + +# Migrating from HEARTBEAT.md: a legacy file is imported into scratch +setup "write a legacy HEARTBEAT.md" "printf '# Heartbeat\n\n- Check whether the nightly backup finished.\n' > ~/.openclaw/workspace/HEARTBEAT.md" +run "openclaw doctor --fix" +setup -e '^ok$' "HEARTBEAT.md was removed from the workspace" "test ! -e ~/.openclaw/workspace/HEARTBEAT.md && echo ok" +setup -e 'nightly backup' "its text is now the monitor scratch" "openclaw cron scratch $PH_jobId" +setup -e '^ok$' "the original was archived under backups/heartbeat-migration/" "ls ~/.openclaw/backups/heartbeat-migration/ | grep -q . && echo ok" + +# Quiet replies and visibility +config_block 2 + +# Recipes: each table row is created for real (deliveries go nowhere here) +run "openclaw automations add \"every 30m\"..." +run "openclaw automations add \"every 1h\" \"List new issues..." +run "openclaw automations add \"0 8 * * *\"..." +run "openclaw automations add \"every 1h\" --name system-health..." +run "openclaw automations add \"0 6 * * 1-5\"..." +run -e 'code-health' "openclaw cron list --all" + +# Cost optimization +config_block 3 + +# Multi-agent heartbeat: validated, not applied (it adds an ops agent and workspace) +config_block 4 --dry-run + +# Debugging +skip "openclaw logs --follow" "streams until interrupted" +run "openclaw system heartbeat last --json" +run "openclaw cron list --all" +run "openclaw cron runs --limit 10" +run "openclaw cron runs --status skipped" +run "openclaw config validate" + +# Complete configuration reference: validated, not applied +config_block 5 --dry-run + +finish diff --git a/scripts/tutorial-tests/lib.sh b/scripts/tutorial-tests/lib.sh new file mode 100755 index 0000000..41669ca --- /dev/null +++ b/scripts/tutorial-tests/lib.sh @@ -0,0 +1,278 @@ +# lib.sh: shared helpers for tutorial tests. Source it from a *.test.sh file. +# +# A test names its page, then walks it in order: +# +# page docs/getting-started/quick-start.md +# onboard # CI stand-in for the page's onboarding +# gateway_start # loopback + token, in the background +# run "openclaw gateway status" # a command copied from the page +# run "openclaw cron add \"0 8 * * *\"..." # "..." = the one page command with this prefix +# config_block 1 # 1st openclaw.json block: patch + validate +# file_block "~/.openclaw/workspace/SOUL.md" +# skip "openclaw tui" "interactive terminal UI" +# finish +# +# `run` and `skip` look the command up on the page (whitespace-insensitive) and +# fail if it is not there, so a test cannot drift from its page silently. The +# command that runs is the page's text, with filled from +# `ph name value`. Every process gets a throwaway HOME, so `~/.openclaw/...` +# on the page means the sandbox. +# +# Env: OPENCLAW_BIN pinned binary (default tools/openclaw/node_modules/.bin/openclaw) +# TT_OUT where .json results go (default ./tutorial-report) +# TT_DRY=1 only check that every run/skip/block exists on the page; run nothing +# TT_ALLOW_GATEWAY=1 allow gateway_start outside GitHub Actions (off by default: +# never start a Gateway on a workstation by accident) + +set -uo pipefail + +TT_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +TT_ROOT=$(cd "$TT_LIB_DIR/../.." && pwd) +TT_OUT=${TT_OUT:-$TT_ROOT/tutorial-report} +TT_DRY=${TT_DRY:-0} +TT_PIN=$(tr -d '[:space:]' < "$TT_ROOT/scripts/openclaw-version.txt") +TT_STEP_TIMEOUT=${TT_STEP_TIMEOUT:-120} +TT_GW_PID= +TT_N=0 + +if [[ -n "${OPENCLAW_BIN:-}" ]]; then TT_BIN=$OPENCLAW_BIN +elif [[ -x "$TT_ROOT/tools/openclaw/node_modules/.bin/openclaw" ]]; then TT_BIN=$TT_ROOT/tools/openclaw/node_modules/.bin/openclaw +else TT_BIN=$(command -v openclaw || true) +fi + +_tt() { node "$TT_LIB_DIR/tt.mjs" "$@"; } +_gh() { [[ "${GITHUB_ACTIONS:-}" == "true" ]]; } +_group() { if _gh; then echo "::group::$*"; else echo "--- $*"; fi; } +_endgroup() { if _gh; then echo "::endgroup::"; fi; } + +# _record kind status line cmd reason seconds exit +_record() { + local json + json=$(node -e 'const [kind,status,line,cmd,reason,secs,exit]=process.argv.slice(1); + process.stdout.write(JSON.stringify({kind,status,line:line?Number(line):null,cmd,reason:reason||null,seconds:secs?Number(secs):null,exit:exit===""?null:Number(exit)}))' \ + "$1" "$2" "$3" "$4" "$5" "${6:-}" "${7:-}") + _tt record "$TT_STEPS" "$json" + local mark=$2; [[ $2 == pass ]] && mark=ok + printf '[%s] %s%s%s\n' "$mark" "${3:+L$3 }" "${4%%$'\n'*}" "${5:+ ($5)}" +} + +page() { + TT_PAGE=$1 + TT_SLUG=$(sed 's#^docs/##; s#\.mdx\{0,1\}$##; s#/#__#g' <<< "$1") + mkdir -p "$TT_OUT" + TT_WORK=$(mktemp -d "${TMPDIR:-/tmp}/tt-$TT_SLUG.XXXXXX") + (cd "$TT_ROOT" && node "$TT_LIB_DIR/extract.mjs" "$TT_PAGE") > "$TT_WORK/page.json" || { echo "cannot read $TT_PAGE"; exit 2; } + TT_STEPS=$TT_WORK/steps.jsonl; : > "$TT_STEPS" + [[ -n "$TT_BIN" || "$TT_DRY" == 1 ]] || { echo "openclaw binary not found (set OPENCLAW_BIN)"; exit 2; } + + # Sandbox: the page's ~ is this directory. + export HOME=$TT_WORK/home + export OPENCLAW_STATE_DIR=$HOME/.openclaw + export OPENCLAW_CONFIG_PATH=$HOME/.openclaw/openclaw.json + export OPENCLAW_DISABLE_UPDATE_CHECK=1 NO_COLOR=1 FORCE_COLOR=0 COLUMNS=200 + unset OPENCLAW_GATEWAY_TOKEN OPENCLAW_GATEWAY_PASSWORD ANTHROPIC_API_KEY OPENAI_API_KEY + mkdir -p "$HOME" "$TT_WORK/bin" + [[ -n "$TT_BIN" ]] && ln -sf "$TT_BIN" "$TT_WORK/bin/openclaw" + export PATH=$TT_WORK/bin:$PATH + cd "$HOME" || exit 2 + trap _tt_cleanup EXIT + echo "== $TT_PAGE (OpenClaw $TT_PIN$([[ $TT_DRY == 1 ]] && echo ", dry run"))" +} + +_tt_cleanup() { gateway_stop quiet; } + +# ph name value: fill in page commands +ph() { printf -v "PH_$1" '%s' "$2"; export "PH_$1"; } + +_fill() { + local text=$1 name var + while [[ $text =~ \<([A-Za-z][A-Za-z0-9_]*)\> ]]; do + name=${BASH_REMATCH[1]}; var=PH_$name + if [[ -z "${!var:-}" ]]; then echo "__MISSING__$name"; return 1; fi + text=${text//<$name>/${!var}} + done + printf '%s' "$text" +} + +# _exec kind line label cmd codes expect timeout +_exec() { + local kind=$1 line=$2 label=$3 cmd=$4 codes=$5 expect=$6 t=$7 + TT_N=$((TT_N + 1)) + if [[ "$TT_DRY" == 1 ]]; then _record "$kind" pass "$line" "$label" "dry run" 0 0; return 0; fi + local out=$TT_WORK/out.$TT_N start=$SECONDS rc status=pass reason= + _group "L${line:-?} $label" + printf '$ %s\n' "$cmd" + (cd "$HOME" && timeout "$t" bash -c "$cmd" < /dev/null) > "$out" 2>&1 + rc=$? + head -c 20000 "$out"; echo + _endgroup + if [[ $rc == 124 ]]; then status=fail; reason="timed out after ${t}s" + elif [[ ",$codes," != *",$rc,"* ]]; then status=fail; reason="exit $rc (expected $codes): $(grep -v '^\s*$' "$out" | tail -n 3 | tr '\n' ' ' | cut -c1-300)" + elif [[ -n "$expect" ]] && ! grep -Eq -- "$expect" "$out"; then status=fail; reason="output did not match /$expect/" + fi + TT_LAST_OUT=$out + _record "$kind" "$status" "$line" "$label" "$reason" $((SECONDS - start)) "$rc" + [[ $status == pass ]] +} + +_opts() { + TT_CODES=0; TT_EXPECT=; TT_T=$TT_STEP_TIMEOUT + while [[ $# -gt 1 ]]; do + case $1 in + -x) TT_CODES=$2; shift 2 ;; + -e) TT_EXPECT=$2; shift 2 ;; + -t) TT_T=$2; shift 2 ;; + *) break ;; + esac + done + TT_ARG=$1 +} + +# run [-x 0,1] [-e regex] [-t secs] "" +run() { + _opts "$@" + local hit line src text cmd + if ! hit=$(_tt resolve "$TT_WORK/page.json" "$TT_ARG" "$TT_STEPS" 2>&1); then + _record page fail "" "$TT_ARG" "$hit"; return 1 + fi + { read -r line; read -r src; text=$(cat); } <<< "$hit" + if ! cmd=$(_fill "$text"); then + _record page fail "$line" "$text" "no value for placeholder <${cmd#__MISSING__}>"; return 1 + fi + _exec page "$line" "$text" "$cmd" "$TT_CODES" "$TT_EXPECT" "$TT_T" +} + +# run_as [-x ..] [-e ..] "" "" +# For page steps that need a CI-safe variant (non-interactive flags, the pinned +# version instead of @latest). Recorded against the page line, with both texts. +run_as() { + _opts "$@"; local q=$TT_ARG; shift $(($# - 1)); local cmd=$1 + local hit line src text + if ! hit=$(_tt resolve "$TT_WORK/page.json" "$q" "$TT_STEPS" 2>&1); then + _record page fail "" "$q" "$hit"; return 1 + fi + { read -r line; read -r src; text=$(cat); } <<< "$hit" + _exec page "$line" "$text [CI: $cmd]" "$cmd" "$TT_CODES" "$TT_EXPECT" "$TT_T" +} + +# skip "" "" +skip() { + local hit line src text + if ! hit=$(_tt resolve "$TT_WORK/page.json" "$1" "$TT_STEPS" 2>&1); then + _record page fail "" "$1" "skip target $hit"; return 1 + fi + { read -r line; read -r src; text=$(cat); } <<< "$hit" + _record page skip "$line" "$text" "$2" +} + +# skip_step "" "" +skip_step() { _record page skip "" "$1" "$2"; } + +# setup [-x codes] [-e regex] "