diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2462bba..a45157b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,13 @@ jobs: package-manager-cache: false - name: Install Unix test dependencies run: brew install stow + # fzf-tab as the bootstrap clones it, for the real fzf-tab preview case + # of tests/test_ai_integration.py (this runner has zsh; ubuntu-24.04 has none). + - name: Check out fzf-tab for the real fzf-tab preview test + run: | + read -r url ref < <(awk -F '\t' '$1 == "fzf-tab" { print $3, $4 }' config/bootstrap/git-clones.tsv) + git clone --quiet --depth 1 --branch "$ref" "$url" "$RUNNER_TEMP/fzf-tab" + echo "FZF_TAB_TEST_DIR=$RUNNER_TEMP/fzf-tab" >> "$GITHUB_ENV" - name: Provision pinned configuration parser explicitly run: ./setup-sync.sh - name: Run behavior tests with system Bash and BSD utilities diff --git a/docs/testing.md b/docs/testing.md index 7489991..3984da9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -9,7 +9,7 @@ macOS and Windows runners' homes.) | Job | Mandatory checks | Provisioning | | --- | --- | --- | | `ubuntu-24.04` | `./tests/run.sh --ci`, full `pre-commit run --all-files`, then the `gitleaks-history` secret scan of every ref | Python 3.11, Node 24, GNU Stow; full-depth checkout; only the coding-agent-kit submodule, whose pinned commit must be on its `main`; pre-commit installs its pinned hooks | -| `macos-15` | `/bin/bash ./tests/run.sh --ci` | Python 3.12, Node 24, GNU Stow; only the coding-agent-kit submodule; retain system Bash and BSD utilities | +| `macos-15` | `/bin/bash ./tests/run.sh --ci` | Python 3.12, Node 24, GNU Stow; only the coding-agent-kit submodule; fzf-tab cloned from its `config/bootstrap/git-clones.tsv` row into the runner's temp directory as `FZF_TAB_TEST_DIR`; retain system Bash and BSD utilities | | `windows-2025` | `./tests/run.ps1 -CI` in native PowerShell | Python 3.12 and Node 24; a local `core.symlinks` re-checkout of `common/coding-agent-kit`, then only the coding-agent-kit submodule, required to be LF with resolving links into it; Git and PowerShell come from the runner and are checked explicitly | The Unix entrypoint checks Bash, POSIX sh, Git, Python and Node before starting. @@ -121,6 +121,9 @@ fetches a missing page from GitHub. `FZF_TAB_TEST_DIR` set to an fzf-tab checkout (the bootstrap clones it to `~/.oh-my-zsh/custom/plugins/fzf-tab`) it also drives real fzf-tab in an interactive zsh with a fake fzf. Without that variable the second case skips. +CI's macOS job clones fzf-tab from its `config/bootstrap/git-clones.tsv` row +and sets the variable, so both cases run there; the `ubuntu-24.04` runner has +no zsh, so both skip in the Linux job. The day-zero bootstrap has four suites. `tests/bootstrap-manifest.sh` unit-tests the sourced `lib/bootstrap/{manifest,platform,version}.sh` and runs diff --git a/tests/COVERAGE.md b/tests/COVERAGE.md index 0561f1e..cd40452 100644 --- a/tests/COVERAGE.md +++ b/tests/COVERAGE.md @@ -43,7 +43,7 @@ source-to-test behavior matrix instead. | `wsl-ubuntu` and `lab-ubuntu` zsh and bash overlay rc files | `host-overlays.sh` | the Linuxbrew `brew shellenv` guard under `zsh -f` and `bash --norc --noprofile` in a clean environment: silent with a missing or non-executable brew, applied only when brew is executable; the overlays' conda and mamba prefixes point at missing paths so only the guard is tested | | `tests/e2e/inside.sh`, `tests/e2e/lib/*.sh` (`blocks.sh` policy, `assert.sh` snapshots and audits), `tests/e2e/hosts/*.env` | `e2e-harness.sh` | HUMAN block parser (notes against commands, `%q` paths with spaces kept byte for byte), the `HUMAN steps pending:` reader, every (step id, kind) policy row with the fail rows (planted extra H7-stow line, another host's stow line, two stow lines, `S2-brew-bundle` sudo and judgment, `S4-nvm`, `X-recovery`, unknown ids and kinds), the exact printed shape of every line the harness runs: the digest gate (sha256sum and shasum forms, the run path bound to the digested one, nothing after it; another path, a `;` tail, an extra pipe stage, an `&&` tail, no `--status`, the `/bin/bash` Homebrew form, a bare run line and an un-gated run line beside the gate refused), `mv -n FILE FILE.pre-dotfiles` for one `%q` path under HOME (chained commands, a mismatched or unescaped path, `$(...)`, backticks and `$'...'` refused) and the stow line (one of the three printed PATH prefixes, then the clone's own `stow-all.sh` and the host; a command between them, command substitution, a `..` path or a trailing argument refused), the by-hand fence holding only the two helper definitions, `E2E_ALLOC_ENV` expansion and refusals with the reason printed, the failure-detail reader, the wrappers.log phase and no-sudo audits and the sudo.log window audit against fixture logs; the real `sudo` and `stow` wrappers with two copies on PATH (the macOS runner's layout) log a call once and run the real tool, and give up with 127 instead of exec'ing each other when there is none; `tests/e2e/timeout.pl` (the per-command limit where neither timeout nor gtimeout exists) exits 124 within seconds of its limit, passes a status and a signal death through, gives 127 for a command it cannot run, kills the whole process group, and is what e2e_find_timeout picks only without timeout or gtimeout; a sandboxed dry run of `inside.sh` under `E2E_SANDBOX_HOME` (a home with a space) against a fixture clone whose stub `setup-host.sh`, `doctor.sh` and `stow-all.sh` print realistic plan lines, blocks and exit codes, with stub sudo, chsh, stow and apt-get wrappers writing the wrappers.log format: all steps pass through four apply runs (apt sudo block, gated inspect line, mv -n and stow line), the reminders are skipped once, the second apply applies nothing, the login shell, doctor runs and negatives pass, the timeline frames every phase and the audits pass; the same at a fixture commit whose `doctor.sh` calls sudo outside a human phase fails the wrappers audit and exits 1; the exit-2 refusals (sandbox not under the temp dir or relative, a source checkout or a sudo, chsh or stow outside the temp dir under the sandbox seam, win, a bad rev, a taken home, no seam) run no step. Docker images, the real bootstrap and the sudo.log a real sudo writes are exercised only by the opt-in `tests/e2e/run.sh` | | `tests/e2e/run.ps1`, `tests/e2e/lib/*.ps1` | `e2e-harness.ps1`, `e2e-harness-windows.sh`, `tests/run.ps1` parse gate | every file parses with the PowerShell parser on each run of `tests/run.ps1` (the native Windows CI job included); with the libraries dot-sourced unchanged and stubs for each setup-host.ps1 run, the HW-stow child, the registry PATH rebuild and the snapshots: the HUMAN block parser on CRLF output with winget's blank progress lines, the (step id, kind) policy (HW-stow runs, the five reminders skip, HW-clone and every other pair fail), only the exact `& '\stow-all.ps1' win` line accepted, the real `setup-host.ps1 -PrintManual` blocks against their policy rows, the pending-id, failure-detail and summary readers, HOME snapshots (added, removed, changed, links by target never followed, an unreadable directory, pruned subtrees, the top-level ntuser hives, `E2E_SNAPSHOT_PRUNE`), CryptnetUrlCache and the TokenBroker cache noted (also through a no-write step) and every other HOME write failing, `summary.tsv` as UTF-8 without BOM with LF and tabs, a passing step failed by a dirty clone, doctor TSV counts, `Invoke-E2EProcess` (closed stdin, both streams logged, 124 after the timeout), the `run.ps1` exit-2 refusals (arguments, a host but Windows), apply-loop dry runs (exit 3 then 0, reminders recorded once, HW-stow passing only with the applied state recorded and no WARNING line, a pending HW-clone failing before the stow, an HW-stow block with another line refused, no progress, exit 1 and 2, a timeout, eight exit-3 runs), no-write steps (noted cache entries pass; a HOME write, a TEMP leftover or redone work fail) and `Invoke-E2EFlowWin` step order with dependents skipped after a break; runs on Linux and macOS pwsh too, where the Windows-only checks print SKIP. winget, the real bootstrap and the runner's home are exercised only by the opt-in `bootstrap-e2e.yml` windows job | -| `.stowrc`, `common/(claude\|codex\|coding-agent-kit)/**`, `fzf-tab.zsh`, `.zsh_aliases`, `.bash_aliases`, and the real `settings.json` against the pinned coding-agent-kit | `ai-integration.sh`, `test_ai_integration.py`, `tests/run.ps1` | the fzf-tab account preview, under the context fzf-tab really looks up (`:fzf-tab:complete:TOOL:`), shows the kit's `--account-info` for the `[Codex accounts]` and `[Claude accounts]` groups and only the word for arguments or a name that is no account, never reaching the file preview or a credential, and, with a real fzf-tab checkout in `FZF_TAB_TEST_DIR`, an interactive zsh with a fake fzf runs fzf-tab's own preview command for each candidate, the email or label shown as its description; both real alias files sourced twice reach `codex-as --help` in Bash and Zsh through a link to its package link, which finds the kit checkout and names its `common/coding-agent-kit/coding-agent-kit/docs/account-launchers.md` with no caller kit configuration in reach, real Stow installs the underscore zsh completions from the coding-agent-kit package, ignores AppleDouble files and leaves its checkout unstowed; the real-data overlay extracts the pinned submodule tree, puts the real `settings.json` over its fixture and reruns its 8 `claude-customizations.cjs` tests (on Windows too, through `tests/run.ps1`) | +| `.stowrc`, `common/(claude\|codex\|coding-agent-kit)/**`, `fzf-tab.zsh`, `.zsh_aliases`, `.bash_aliases`, and the real `settings.json` against the pinned coding-agent-kit | `ai-integration.sh`, `test_ai_integration.py`, `tests/run.ps1` | the fzf-tab account preview, under the context fzf-tab really looks up (`:fzf-tab:complete:TOOL:`), shows the kit's `--account-info` for the `[Codex accounts]` and `[Claude accounts]` groups and only the word for arguments or a name that is no account, never reaching the file preview or a credential, and, with a real fzf-tab checkout in `FZF_TAB_TEST_DIR` (CI's macOS job clones one), an interactive zsh with a fake fzf runs fzf-tab's own preview command for each candidate, the email or label shown as its description; both real alias files sourced twice reach `codex-as --help` in Bash and Zsh through a link to its package link, which finds the kit checkout and names its `common/coding-agent-kit/coding-agent-kit/docs/account-launchers.md` with no caller kit configuration in reach, real Stow installs the underscore zsh completions from the coding-agent-kit package, ignores AppleDouble files and leaves its checkout unstowed; the real-data overlay extracts the pinned submodule tree, puts the real `settings.json` over its fixture and reruns its 8 `claude-customizations.cjs` tests (on Windows too, through `tests/run.ps1`) | | `tests/run.sh`, `tests/run.ps1`, the suites' fixture Git, `.pre-commit-config.yaml` and CI | `test-entrypoints.sh`, three native CI jobs | required dependencies fail visibly, local optional skips reported, native Windows job cannot run on Unix, shared backend and AI integration suites required, every entry point unsets the same dotfiles and kit knobs (covering each `DOTFILES_*`/`AWESOME_SKILLS_*` name in any tracked file outside tests, docs, `.github` and Markdown, in shell, PowerShell or Python spelling, and each `CODING_AGENT_KIT_*` or `_CODING_AGENT_KIT_*` name in the kit's code at the recorded gitlink, through the delivery test's `check_kit_knobs`) and a suite passes under the provisioning exports; fixture Git out of a commit hook's repository: the Bash suites with fixture Git in their body (`bootstrap-manifest.sh`, `doctor.sh`, `e2e-harness.sh`, `setup-host.sh`, `unix-installer.sh`) unset `GIT_DIR` and `GIT_INDEX_FILE` and the PowerShell ones (`e2e-harness.ps1`, `windows-installer-controls.ps1`) remove every `GIT_*` variable, the manifest suite passes under a relative `GIT_INDEX_FILE`, and with pwsh the installer controls leave the index, `HEAD` and `core.bare` of a throwaway repository alone under its linked worktree's `GIT_DIR` and `GIT_INDEX_FILE`; `.pre-commit-config.yaml` selection as pre-commit makes it (files, exclude, top-level exclude): the engine hooks on `lib/config_sync.py` and `lib/sync-runtime.sh`, `structured-config-sync` and `codex-config-sync` on the real Codex `config.toml`, the delivery hook on `tests/run.sh`, the bump workflow and its reviewed copy, and the `test-entrypoints` hook on itself, the entry points and `.pre-commit-config.yaml` | Windows updater mocks do not establish actual Task Scheduler execution. The diff --git a/tests/test_ai_integration.py b/tests/test_ai_integration.py index 22f2b28..e62c63a 100644 --- a/tests/test_ai_integration.py +++ b/tests/test_ai_integration.py @@ -11,6 +11,7 @@ from pathlib import Path import select import shutil +import signal import subprocess import sys import tempfile @@ -144,12 +145,21 @@ def test_account_preview_shows_account_info_and_never_reads_a_secret(self): @unittest.skipIf(os.name == "nt", "zsh and fzf-tab") def test_real_fzf_tab_previews_accounts_through_the_kit(self): # Real fzf-tab (FZF_TAB_TEST_DIR names a checkout; the bootstrap clones - # it to $ZSH_CUSTOM/plugins/fzf-tab) in an interactive zsh, with dotfiles' - # zstyles and a fake fzf that runs fzf-tab's own preview command. + # it to $ZSH_CUSTOM/plugins/fzf-tab, CI's macOS job to the runner's temp) + # in an interactive zsh, with dotfiles' zstyles and a fake fzf that runs + # fzf-tab's own preview command. zsh, plugin = shutil.which("zsh"), Path(os.environ.get("FZF_TAB_TEST_DIR", "/nonexistent")) if not zsh or not (plugin / "fzf-tab.plugin.zsh").is_file(): self.skipTest("zsh and an fzf-tab checkout in FZF_TAB_TEST_DIR") import pty # POSIX only: native Windows Python has no termios, so not at module level + # A hang below fails with its traceback, not the CI job's timeout. An alarm, not a + # watchdog thread: forking a multi-threaded process can deadlock the child. + def hung(signum, frame): + signal.alarm(10) # again, should the cleanup of the session hang too + raise TimeoutError("the real fzf-tab case hung") + self.addCleanup(signal.signal, signal.SIGALRM, signal.signal(signal.SIGALRM, hung)) + self.addCleanup(signal.alarm, 0) + signal.alarm(120) self.preview_home() log = self.base / "previews.log" # fzf-tab splits its fzf-command on spaces, so not below the spaced HOME. @@ -166,8 +176,10 @@ def test_real_fzf_tab_previews_accounts_through_the_kit(self): f"autoload -Uz compinit && compinit -u -d '{self.base}/zcompdump'\n" f"source '{plugin}/fzf-tab.plugin.zsh'\nsource '{PREVIEW}'\n" f"zstyle ':fzf-tab:*' fzf-command '{fake}'\nzstyle ':fzf-tab:*' fzf-flags --ansi\nPS1='READY> '\n") + # The caller's PATH behind the launchers, as in every other case: they + # need Python 3.11+, which /usr/bin does not have on a macOS runner. env = self.env | {"ZDOTDIR": str(rc), "SHELL": zsh, "TERM": "xterm-256color", "FAKE_FZF_LOG": str(log), - "TMPPREFIX": str(self.base / "zsh"), "PATH": os.pathsep.join((str(self.command_bin), "/usr/bin", "/bin"))} + "TMPPREFIX": str(self.base / "zsh")} for line, expected in ((b"codex-as \t", ["=== A -- alice@example.com", "A (Codex)", "email alice@example.com", "plan plus", "usage not recorded yet", "=== B", "B (Codex)", "email not recorded yet", @@ -182,22 +194,31 @@ def test_real_fzf_tab_previews_accounts_through_the_kit(self): os.chdir(self.cwd) # No global rc: a distribution zshrc may run its own compinit, which prompts. os.execve(zsh, [zsh, "--no-globalrcs", "-i"], env) + output = b"" try: - output, deadline = b"", time.monotonic() + 20 + deadline = time.monotonic() + 20 while b"READY> " not in output and time.monotonic() < deadline: if select.select([fd], [], [], 0.1)[0]: output += os.read(fd, 4096) os.write(fd, line) while not log.exists() and time.monotonic() < deadline: if select.select([fd], [], [], 0.1)[0]: - os.read(fd, 4096) + output += os.read(fd, 4096) time.sleep(0.5) finally: - os.kill(pid, 9) # only the test's own interactive zsh - os.waitpid(pid, 0) + # The session pty.fork started (its pid is the process group too): only + # the test's own zsh and what it ran. The terminal is closed before the + # wait: on macOS and the BSDs the last close of a terminal with unread + # output can wait for it to drain, and nothing would read it. + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass os.close(fd) - self.assertTrue(log.exists(), "fzf-tab never ran the fake fzf") - self.assertEqual(log.read_text().splitlines(), expected) + os.waitpid(pid, 0) + shown = output.decode(errors="replace")[-2000:] + self.assertTrue(log.exists(), "fzf-tab never ran the fake fzf; the terminal showed:\n" + shown) + self.assertEqual(log.read_text().splitlines(), expected, shown) @unittest.skipIf(os.name == "nt", "GNU Stow") def test_real_stow_installs_underscore_completions_and_ignores_appledouble(self):