diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000..544e849 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,20 @@ +name: setup +description: Node.js and the pnpm version `packageManager` pins, then the workspace dependencies. + +inputs: + install: + description: Install the workspace dependencies, not needed by jobs that only pack the layer. + default: 'true' + +runs: + using: composite + steps: + - uses: actions/setup-node@v6 + with: + package-manager-cache: false + node-version: '24' + - run: npm i -g --force corepack && corepack enable + shell: bash + - if: inputs.install == 'true' + run: pnpm install --frozen-lockfile + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32a4aab..8517977 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,12 +22,7 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 - with: - package-manager-cache: false - node-version: '24' - - run: npm i -g --force corepack && corepack enable - - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/setup # Lint and unit tests need no generated types, so they run before `prepare` # and fail fast. - run: pnpm lint @@ -51,14 +46,36 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 - with: - package-manager-cache: false - node-version: '24' - - run: npm i -g --force corepack && corepack enable - - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/setup - run: pnpm build env: NITRO_PRESET: vercel VERCEL_BYPASS_TOKEN: ci-placeholder-bypass-token-0000000000 - run: pnpm test:vercel + + # The layer as a site installs it: packed, outside the workspace, built in a fresh app. + pack: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup + with: + install: false + - run: ./scripts/test-pack.sh + + # comark.dev, a site extending the layer, built against this commit instead of the version it pins. + # cms.comark.dev joins once its repository is public, a private one would need a token secret. + consumer: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + - uses: actions/checkout@v6 + with: + repository: comarkdown/comark + path: .consumer + - uses: ./.github/actions/setup + with: + install: false + - run: ./scripts/test-consumer.sh .consumer diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..1a2ae62 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,35 @@ +name: smoke + +# Vercel's GitHub integration reports every deployment, preview and production, through this event. +# Content-only pushes skip the build (Ignored Build Step), so they deploy nothing and run nothing here. +on: deployment_status + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.deployment.id }} + cancel-in-progress: true + +jobs: + smoke: + name: smoke (${{ github.event.deployment.environment }}) + if: github.event.deployment_status.state == 'success' + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + # The tests of the deployed commit, not the default branch's. + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.deployment.sha }} + persist-credentials: false + - uses: ./.github/actions/setup + - run: pnpm test:smoke + env: + SMOKE_URL: ${{ github.event.deployment_status.environment_url || github.event.deployment_status.target_url }} + SMOKE_ENVIRONMENT: ${{ github.event.deployment.environment }} + # Vercel project → Deployment Protection → Protection Bypass for Automation. + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + # Set the repository variable to `true` to also call the assistant, which costs AI Gateway calls. + SMOKE_ASSISTANT: ${{ vars.SMOKE_ASSISTANT }} diff --git a/README.md b/README.md index 85c6318..9181142 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,13 @@ Each skill is a directory with a `SKILL.md` whose frontmatter includes a `descri ```bash pnpm install -pnpm dev # runs the playground -pnpm lint # correctness-only ESLint (no stylistic rules — see eslint.config.mjs) -pnpm test # Vitest over the pure utils -pnpm test:e2e # Vitest against the built playground (`pnpm build` first) -pnpm typecheck # vue-tsc across the layer + playground +pnpm dev # runs the playground +pnpm lint # correctness-only ESLint (no stylistic rules — see eslint.config.mjs) +pnpm test # Vitest over the pure utils +pnpm test:e2e # Vitest against the built playground (`pnpm build` first) +pnpm test:vercel # Vitest over the Vercel output (`NITRO_PRESET=vercel pnpm build` first) +pnpm test:smoke # the e2e suite against a deployment (`SMOKE_URL=https://… pnpm test:smoke`) +pnpm typecheck # vue-tsc across the layer + playground ``` To develop the layer against a consumer app in a sibling checkout: diff --git a/package.json b/package.json index 2b9b41f..b9ae399 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "test": "vitest run --project unit", "test:e2e": "vitest run --project e2e", "test:vercel": "vitest run --project vercel", + "test:smoke": "vitest run --project smoke", "test:watch": "vitest --project unit", "dev:prepare": "nuxt prepare playground" }, diff --git a/scripts/test-consumer.sh b/scripts/test-consumer.sh new file mode 100755 index 0000000..75e9224 --- /dev/null +++ b/scripts/test-consumer.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Builds a site extending the layer against the packed layer, instead of the version it pins. +# Usage: scripts/test-consumer.sh [app-dir] +# - `app-dir` is the Nuxt app inside the site's repository, `docs` by default. +set -euo pipefail + +site="$(cd "${1:?Usage: scripts/test-consumer.sh [app-dir]}" && pwd)" +app="${2:-docs}" +root="$(cd "$(dirname "$0")/.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +echo "› Packing the layer" +pnpm --dir "$root" pack --pack-destination "$work" > /dev/null +tarball="$(ls "$work"/comark-docs-*.tgz)" + +# A copy, so the checkout keeps its own layer version and lockfile. +# `.git` stays: the layer reads the content commit from it. +echo "› Copying the site to $work/site" +rsync -a --exclude node_modules --exclude .nuxt --exclude .output --exclude .vercel --exclude .data "$site/" "$work/site/" +cd "$work/site" + +echo "› Pointing $app at the packed layer" +(cd "$app" && npm pkg set "dependencies.comark-docs=file:$tarball") +# The site's lockfile pins the layer it ships, so it has to move. +pnpm install --no-frozen-lockfile + +echo "› Building $app" +pnpm --dir "$app" run build diff --git a/scripts/test-pack.sh b/scripts/test-pack.sh new file mode 100755 index 0000000..b40fd34 --- /dev/null +++ b/scripts/test-pack.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Packs the layer as a consumer installs it, then builds a fresh copy of test/fixtures/consumer with it. +# - Catches what the workspace hides: a file missing from `files`, a dependency only the workspace provides. +# - `WORKDIR` keeps the app there instead of in a temporary folder. +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd)" + +if [ -n "${WORKDIR:-}" ]; then + work="$WORKDIR" + if [ -e "$work/app" ]; then + echo "$work/app already exists, remove it or pick another WORKDIR" >&2 + exit 1 + fi + mkdir -p "$work" +else + work="$(mktemp -d)" + trap 'rm -rf "$work"' EXIT +fi + +echo "› Packing the layer" +pnpm --dir "$root" pack --pack-destination "$work" > /dev/null +tarball="$(ls "$work"/comark-docs-*.tgz)" + +# Outside the repo: pnpm would otherwise pick up its workspace and overrides. +echo "› Copying the consumer fixture to $work/app" +cp -R "$root/test/fixtures/consumer" "$work/app" +cd "$work/app" + +# A git checkout like a real site, so the build-time content snapshot runs too. +# The empty root commit gives the content commit a parent, as in any non-shallow clone. +git init -q +git -c user.name=ci -c user.email=ci@localhost commit -q --allow-empty -m root +git add -A +git -c user.name=ci -c user.email=ci@localhost commit -qm fixture + +echo "› Installing the packed layer" +npm pkg set "dependencies.comark-docs=file:$tarball" "packageManager=$(npm pkg get packageManager --prefix "$root" | tr -d '"')" +pnpm install + +echo "› Building" +pnpm exec nuxt build diff --git a/test/e2e/mcp.test.ts b/test/e2e/mcp.test.ts index 1e6dc0e..7b312ea 100644 --- a/test/e2e/mcp.test.ts +++ b/test/e2e/mcp.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from 'vitest' -import { baseURL, fetchWithFailure, INTRO_PAGE, NOT_FOUND_SLUG, SITE_URL } from './utils' +import { fetchWithFailure, INTRO_PAGE, NOT_FOUND_SLUG, request, SITE_URL } from './utils' let requestId = 0 /** Calls the MCP server over streamable HTTP, without a session. */ async function mcp(method: string, params?: Record): Promise { - const response = await fetch(new URL('/mcp', baseURL), { + const response = await request('/mcp', { method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream' }, body: JSON.stringify({ jsonrpc: '2.0', id: ++requestId, method, params }), diff --git a/test/e2e/utils.ts b/test/e2e/utils.ts index e047a39..06e8c0a 100644 --- a/test/e2e/utils.ts +++ b/test/e2e/utils.ts @@ -14,9 +14,19 @@ export const INTRO_PAGE = { path: '/getting-started/introduction', title: 'Intro /** A slug no page uses. */ export const NOT_FOUND_SLUG = '__e2e-not-found__' +// Only set against a protected Vercel deployment, see `test/smoke/`. +const bypassSecret = process.env.VERCEL_AUTOMATION_BYPASS_SECRET + +/** Requests a site path, passing Vercel's Deployment Protection when a bypass secret is set. */ +export function request(path: string, init: RequestInit = {}): Promise { + const headers = new Headers(init.headers) + if (bypassSecret) headers.set('x-vercel-protection-bypass', bypassSecret) + return fetch(new URL(path, baseURL), { ...init, headers }) +} + /** Fetches a site path once: use it for expected errors, which the server logs on every request. */ export function fetchPath(path: string, headers?: HeadersInit): Promise { - return fetch(new URL(path, baseURL), { headers }) + return request(path, { headers }) } /** Fetches a site path, explaining a failure with the message Nitro attaches to its JSON errors. */ diff --git a/test/fixtures/consumer/content/1.getting-started/.navigation.yml b/test/fixtures/consumer/content/1.getting-started/.navigation.yml new file mode 100644 index 0000000..bcd2ba1 --- /dev/null +++ b/test/fixtures/consumer/content/1.getting-started/.navigation.yml @@ -0,0 +1 @@ +title: Getting Started diff --git a/test/fixtures/consumer/content/1.getting-started/1.introduction.md b/test/fixtures/consumer/content/1.getting-started/1.introduction.md new file mode 100644 index 0000000..0d52e94 --- /dev/null +++ b/test/fixtures/consumer/content/1.getting-started/1.introduction.md @@ -0,0 +1,6 @@ +--- +title: Introduction +description: What this project does and why it exists. +--- + +Welcome to the docs! diff --git a/test/fixtures/consumer/content/index.md b/test/fixtures/consumer/content/index.md new file mode 100644 index 0000000..210fffe --- /dev/null +++ b/test/fixtures/consumer/content/index.md @@ -0,0 +1,9 @@ +--- +title: Consumer Fixture +description: A site extending the packed comark-docs layer. +navigation: false +--- + +# Consumer Fixture + +Read the [introduction](/getting-started/introduction). diff --git a/test/fixtures/consumer/nuxt.config.ts b/test/fixtures/consumer/nuxt.config.ts new file mode 100644 index 0000000..034da81 --- /dev/null +++ b/test/fixtures/consumer/nuxt.config.ts @@ -0,0 +1,8 @@ +// The setup the installation guide documents, nothing more. +export default defineNuxtConfig({ + extends: ['comark-docs'], + site: { + url: 'https://docs.example.com', + name: 'Consumer Fixture', + }, +}) diff --git a/test/fixtures/consumer/package.json b/test/fixtures/consumer/package.json new file mode 100644 index 0000000..d35ed74 --- /dev/null +++ b/test/fixtures/consumer/package.json @@ -0,0 +1,9 @@ +{ + "name": "comark-docs-consumer", + "private": true, + "type": "module", + "dependencies": { + "comark-docs": "set by scripts/test-pack.sh", + "nuxt": "^4.6.2" + } +} diff --git a/test/fixtures/consumer/pnpm-workspace.yaml b/test/fixtures/consumer/pnpm-workspace.yaml new file mode 100644 index 0000000..8f13118 --- /dev/null +++ b/test/fixtures/consumer/pnpm-workspace.yaml @@ -0,0 +1,15 @@ +# The layer depends on a pkg.pr.new build of nuxt-llms, which pnpm blocks in subdependencies by default. +# Remove once nuxt-llms ships a registry release the layer can depend on instead. +blockExoticSubdeps: false + +# pnpm fails a non-interactive install on unreviewed build scripts, same decisions as the repo root. +allowBuilds: + esbuild: false + vue-demi: false + +minimumReleaseAgeExclude: + - '@comark/nuxt' + - '@comark/vue' + - comark + - comark-content + - nuxt-agent-discovery diff --git a/test/smoke/setup.ts b/test/smoke/setup.ts new file mode 100644 index 0000000..2ef0b85 --- /dev/null +++ b/test/smoke/setup.ts @@ -0,0 +1,10 @@ +import type { TestProject } from 'vitest/node' + +/** Points the e2e suite at a deployment instead of booting the local build. */ +export default function setup(project: TestProject) { + const url = process.env.SMOKE_URL + if (!url) { + throw new Error('No deployment to test: set `SMOKE_URL`, e.g. `SMOKE_URL=https://docs-template.comark.dev pnpm test:smoke`.') + } + project.provide('baseURL', new URL(url).origin) +} diff --git a/test/smoke/vercel.test.ts b/test/smoke/vercel.test.ts new file mode 100644 index 0000000..f025079 --- /dev/null +++ b/test/smoke/vercel.test.ts @@ -0,0 +1,50 @@ +import { setTimeout as sleep } from 'node:timers/promises' +import { describe, expect, it } from 'vitest' +import { INTRO_PAGE, request } from '../e2e/utils' + +const FROM_CACHE = /^(?:HIT|STALE|PRERENDER)$/ + +describe('ISR', () => { + it('serves a repeat request from the cache', async () => { + let cache: string | null = null + // Vercel stores a fresh response about a second after serving it, so an immediate repeat still misses. + for (let attempt = 0; attempt < 10 && !FROM_CACHE.test(cache ?? ''); attempt++) { + if (attempt) await sleep(1000) + const response = await request(INTRO_PAGE.path) + expect(response.status).toBe(200) + await response.text() + cache = response.headers.get('x-vercel-cache') + } + expect(cache).toMatch(FROM_CACHE) + }) +}) + +// The push webhook only targets production. +describe.runIf(process.env.SMOKE_ENVIRONMENT === 'Production')('revalidation webhook', () => { + it('refuses an unsigned call', async () => { + const response = await request('/api/revalidate', { method: 'POST' }) + // A 501 means `WEBHOOK_SECRET` or `VERCEL_BYPASS_TOKEN` is missing from the deployment. + expect(response.status).toBe(401) + }) +}) + +// Every run costs AI Gateway calls, so it is opt-in. +describe.runIf(process.env.SMOKE_ASSISTANT === 'true')('assistant', () => { + it('streams an answer', async () => { + const response = await request('/api/assistant', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + messages: [{ id: 'smoke', role: 'user', parts: [{ type: 'text', text: 'In one sentence, what is this documentation about?' }] }], + }), + signal: AbortSignal.timeout(55_000), + }) + expect(response.status).toBe(200) + expect(response.headers.get('content-type')).toContain('text/event-stream') + + const stream = await response.text() + expect(stream).not.toContain('"type":"error"') + expect(stream).toContain('"type":"text-delta"') + expect(stream).toContain('"type":"finish"') + }) +}) diff --git a/test/vercel/output.test.ts b/test/vercel/output.test.ts index 175534d..9e09990 100644 --- a/test/vercel/output.test.ts +++ b/test/vercel/output.test.ts @@ -57,13 +57,11 @@ function resolveFunction(path: string) { return { dest, expiration } } -function walk(dir: string, files: string[] = []): string[] { - for (const entry of readdirSync(dir, { withFileTypes: true })) { - const path = join(dir, entry.name) - if (entry.isDirectory()) walk(path, files) - else files.push(path) - } - return files +/** Files under `dir`, skipping symlinks such as the ISR functions, which all link to the server function. */ +function filesIn(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true, recursive: true }) + .filter((entry) => entry.isFile()) + .map((entry) => join(entry.parentPath, entry.name)) } /** Nitro inlines each server asset as a raw chunk named after the asset, exporting its contents as a string. */ @@ -124,7 +122,7 @@ describe('ISR', () => { // The push webhook purges pages with it, without one content updates wait for the TTL. it('gives every ISR function the same bypass token', () => { - const configs = walk(functionsDir) + const configs = filesIn(functionsDir) .filter((file) => file.endsWith('.prerender-config.json')) .map((file) => ({ file: relative(functionsDir, file), ...readJSON(file) })) @@ -148,7 +146,7 @@ describe('server function', () => { }) it(`stays under ${MAX_FUNCTION_MB} MB`, () => { - const bytes = walk(serverDir).reduce((total, file) => total + lstatSync(file).size, 0) + const bytes = filesIn(serverDir).reduce((total, file) => total + lstatSync(file).size, 0) expect(bytes / 1024 / 1024).toBeLessThan(MAX_FUNCTION_MB) }) diff --git a/vitest.config.ts b/vitest.config.ts index b0eadf7..431711d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,11 +1,6 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ - // Transform without a tsconfig. oxc would otherwise resolve the root stub, which - // points into `playground/.nuxt/` and so only exists after `nuxt prepare` — a fresh - // clone (and CI, which tests before preparing) ran zero tests. Nothing under test - // needs tsconfig semantics. Absent from Vite's `OxcOptions` type but forwarded to - // `transformSync`, hence the cast. oxc: { tsconfig: false } as never, test: { projects: [ @@ -13,7 +8,7 @@ export default defineConfig({ test: { name: 'unit', include: ['test/**/*.test.ts', 'modules/**/*.test.ts'], - exclude: ['test/e2e/**', 'test/vercel/**'], + exclude: ['test/e2e/**', 'test/vercel/**', 'test/smoke/**'], setupFiles: ['test/setup.ts'], environment: 'node', }, @@ -37,6 +32,19 @@ export default defineConfig({ environment: 'node', }, }, + { + // The e2e suite against a deployment (`SMOKE_URL`), plus what only Vercel can answer. + test: { + name: 'smoke', + include: ['test/e2e/**/*.test.ts', 'test/smoke/**/*.test.ts'], + // Reads the local build output, which a deployment does not have. + exclude: ['test/e2e/bundle.test.ts'], + globalSetup: ['test/smoke/setup.ts'], + environment: 'node', + // Cold starts included. + testTimeout: 60_000, + }, + }, ], }, })