From 18858cb2d9444d0926943cae0874ae4fe2f0b697 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 4 Sep 2026 15:26:54 -0700 Subject: [PATCH 1/2] docs(readme): add intro video and make the README agent-agnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the YouTube intro video and rework the README so it reads as a harness-agnostic framework rather than a Claude Code tool: - New tagline: "Playwright for coding agents". - Title drops "Claude Code skills" for "their skills". - Intro names all four harnesses and states that changing harness is one field (agent.type) with tasks/criteria/scoring unchanged. - Quick Start prerequisites become a per-agent runtime table (claude-code, codex, antigravity, opencode) instead of the Claude CLI only. - Task Definition section explains swapping agent.type and links Run-Limit Parity. - Known limits: bring your own agent runtime as well as model credentials; mention OpenRouter for open-weight models via OpenCode. - Acknowledgments credit every harness the framework drives. The plugin section stays Claude Code specific — it is, factually — but says so and notes the suites it authors run on every harness. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016v9L8gXbq4MzH6V9JZPqcb --- README.md | 89 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1273b57c..6b93a993 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Coder Eval — evaluate & benchmark AI coding agents and Claude Code skills +# Coder Eval — evaluate and benchmark AI coding agents and their skills [![PyPI](https://img.shields.io/pypi/v/coder-eval.svg)](https://pypi.org/project/coder-eval/) [![GitHub Marketplace](https://img.shields.io/badge/marketplace-coder__eval-2ea44f.svg)](https://github.com/marketplace/actions/coder_eval) @@ -8,16 +8,23 @@ [![Python 3.13+](https://img.shields.io/badge/python-3.13%2B-blue.svg)](https://www.python.org/downloads/) [![CI](https://github.com/UiPath/coder_eval/actions/workflows/pr-checks.yml/badge.svg)](https://github.com/UiPath/coder_eval/actions/workflows/pr-checks.yml) -**Coder Eval** (`pip install coder-eval` / `uv tool install coder-eval`) is an open-source framework for -**evaluating and benchmarking AI coding agents and their skills** — built for CLI -and skill builders — with sandboxing, reproducibility, and data-driven analysis. -It runs a real agent (**Claude Code**, **Codex**, **Google Antigravity / -Gemini**, or **OpenCode**) in a sandbox against declarative YAML tasks, then scores the files and -commands it actually produced. Not an "agentic coding" benchmark: it measures how -effective your CLI and skills are when used by coding agents. +

+ Playwright for coding agents — one declarative test file, any agent + runtime, a real sandbox, and a pass/fail gate in CI. +

-Reach for it when you want to **test whether a Claude Code skill triggers**, -**A/B-test Claude Code vs. Codex vs. Gemini vs. OpenCode** (or model vs. model, +**Coder Eval** (`pip install coder-eval` / `uv tool install coder-eval`) is an +open-source, **agent-agnostic** framework for **evaluating and benchmarking AI coding +agents and their skills** — built for CLI and skill builders — with sandboxing, +reproducibility, and data-driven analysis. It runs a real agent — **Claude Code**, +**OpenAI Codex**, **Google Antigravity (Gemini)**, or **OpenCode** — in a sandbox +against declarative YAML tasks, then scores the files and commands it actually +produced. Changing harness is one field (`agent.type`); the tasks, criteria, scoring, +telemetry, and reports stay the same. Not an "agentic coding" benchmark: it measures +how effective your CLI and skills are when used by coding agents. + +Reach for it when you want to **test whether a skill triggers** in the agent you ship +for, **A/B-test Claude Code vs. Codex vs. Gemini vs. OpenCode** (or model vs. model, prompt vs. prompt), or **gate CI on coding-agent quality**. Unlike fixed datasets (SWE-bench, SkillsBench) that rank models on a shared leaderboard, Coder Eval evaluates the tasks, skills, and workflows *you* ship — with weighted 0.0–1.0 criteria, a @@ -33,15 +40,26 @@ telemetry. See [How it compares](https://coder-eval.com/docs/comparison). - **Sandboxed execution** in isolated environments with resource limits - **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds - **Many criterion types** — from file checks to code similarity and LLM-graded rubrics -- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), and OpenCode today, extensible via a plugin SPI +- **Agent-agnostic by design** — Claude Code, OpenAI Codex, Antigravity (Gemini), and OpenCode today; add your own harness through the plugin SPI - **Experiment layer** — A/B agent configs (models, tools, prompts) side-by-side - **Full telemetry** — every tool call, token counts, and cost, with real-time streaming +## Watch the intro + +

+ + Video: Coder Eval — UiPath open-source framework to test AI coding agents + +

+ +▶ **[Coder Eval: UiPath open-source framework to test AI Coding Agents](https://www.youtube.com/watch?v=Iyq-5m1CnuI)** +— what the framework does, and how a run works end to end. + ## What you can do with it - **Benchmark coding agents** — score an agent across a suite of tasks with weighted scoring and pass/fail thresholds - **Compare models & configs** — A/B-test Claude vs. Codex vs. Gemini vs. OpenCode, model vs. model, tool-on vs. tool-off, prompt vs. prompt -- **Evaluate skills** — verify an agent actually engages a target skill (`skill_triggered`) and score skill-driven suites (SkillsBench-style) +- **Evaluate skills** — verify an agent actually engages a target skill (`skill_triggered`) and score skill-driven suites (SkillsBench-style), on whichever harness your users run - **Keep skills up to date in CI** — re-validate your skills on every change or on a schedule; catch silent regressions when models, prompts, or the skills themselves drift - **Gate CI on agent quality** — run the suite in GitHub Actions and fail the build on regressions - **Bring your own dataset** — fan one task out over many rows for larger benchmark suites @@ -53,9 +71,19 @@ telemetry. See [How it compares](https://coder-eval.com/docs/comparison). ## Quick Start -**Prerequisites:** Python 3.13+, [uv](https://docs.astral.sh/uv/) 0.8+, and the -[Claude CLI](https://docs.anthropic.com/claude/docs/claude-code) (`brew install claude`). -Developed on macOS; CI runs on Linux. +**Prerequisites:** Python 3.13+, [uv](https://docs.astral.sh/uv/) 0.8+, and **the +runtime of at least one coding agent**. Coder Eval installs the harness adapters, never +the agents themselves — pick the one you want to evaluate: + +| Agent | `agent.type` | Runtime to install | Guide | +| --- | --- | --- | --- | +| Claude Code (default) | `claude-code` | `brew install claude` | [Claude Code](docs/agents/CLAUDE_CODE.md) | +| OpenAI Codex | `codex` | `pip install 'coder-eval[codex]'` | [Codex](docs/agents/CODEX.md) | +| Google Antigravity (Gemini) | `antigravity` | `pip install 'coder-eval[antigravity]'` | [Antigravity](docs/agents/ANTIGRAVITY.md) | +| OpenCode (open-weight models) | `opencode` | `npm install -g opencode-ai` | [OpenCode](docs/agents/OPENCODE.md) | + +The examples below use the default `claude-code` agent. Developed on macOS; CI runs on +Linux. ```bash git clone https://github.com/UiPath/coder_eval.git @@ -98,9 +126,10 @@ the full setup. ## Use inside Claude Code -This repo is also a **Claude Code plugin marketplace**, so the whole loop — -scaffold a suite, author a task, check whether a skill triggers, read the -results — runs inside the agent: +Coder Eval evaluates any of the supported agents, and it also ships an authoring +front-end for one of them: this repo is a **Claude Code plugin marketplace**, so the +whole loop — scaffold a suite, author a task, check whether a skill triggers, read the +results — runs inside Claude Code. The suites you author this way run on every harness: ``` /plugin marketplace add UiPath/coder_eval @@ -275,9 +304,13 @@ success_criteria: description: "Script must execute successfully" ``` -Tasks can omit the `agent` section entirely — defaults resolve from the experiment -layer (`experiments/default.yaml`). For the full schema and every criterion type, -see the [Task Definition Guide](docs/TASK_DEFINITION_GUIDE.md). +`agent.type` is the only harness-specific line: swap it for `codex`, `antigravity`, or +`opencode` — or override it per run with `coder-eval run … -D agent.type=opencode` — and +the same criteria score the same way. Tasks can omit the `agent` section entirely — +defaults resolve from the experiment layer (`experiments/default.yaml`). For the full +schema and every criterion type, see the +[Task Definition Guide](docs/TASK_DEFINITION_GUIDE.md); for what each `run_limits` field +means on each harness, see [Run-Limit Parity](docs/agents/HARNESS_PARITY.md). > **Tip:** With the [Claude Code plugin](docs/PLUGIN.md) installed, use > `/coder-eval:task` to scaffold a task from a natural-language description, and @@ -301,8 +334,10 @@ extension points (new criteria, new agents). - **Tasks execute real code** — run untrusted tasks only under the container driver (see [Docker Isolation](docs/DOCKER_ISOLATION.md)); the `tempdir` driver is not a security boundary. -- **Bring your own model credentials** — Anthropic, Bedrock, or Gemini keys; Coder Eval - does not proxy or supply model access. +- **Bring your own agent runtime and model credentials** — Coder Eval installs neither + the coding-agent CLIs nor model access. Supply the runtime (see + [Quick Start](#quick-start)) and the keys it needs — Anthropic, Bedrock, OpenAI, + Gemini, or an OpenRouter key for open-weight models via OpenCode. - **Python 3.13+ only.** ## Support & security @@ -318,6 +353,8 @@ extension points (new criteria, new agents). ## Acknowledgments -Built with the [Claude Agent SDK](https://github.com/anthropics/claude-agent-sdk), -[Pydantic](https://pydantic.dev/), [Typer](https://typer.tiangolo.com/), and -[Rich](https://rich.readthedocs.io/). +Built with [Pydantic](https://pydantic.dev/), [Typer](https://typer.tiangolo.com/), +and [Rich](https://rich.readthedocs.io/), on top of the harnesses it drives — the +[Claude Agent SDK](https://github.com/anthropics/claude-agent-sdk), the +[Codex SDK](https://github.com/openai/codex), [Google Antigravity](https://antigravity.google/), +and [OpenCode](https://opencode.ai). From a28bbad0d011f7400184fceae9e0a111db7cfbc0 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 4 Sep 2026 15:30:45 -0700 Subject: [PATCH 2/2] docs(stub): make the Pages stub and package metadata agent-agnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the GitHub Pages stub in line with the reworked README: drop "Claude Code skills" from the title, name all four harnesses (OpenCode was missing entirely), and lead with the "Playwright for coding agents" tagline in both the meta description and the visible lead paragraph. The old lead also read "agents and their Claude Code skills", which parsed wrong even for a Claude-only framing. The stub's own comment says its description mirrors the package metadata, so pyproject's description and keywords gain OpenCode too — otherwise that comment stops being true the moment the stub changes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016v9L8gXbq4MzH6V9JZPqcb --- .github/pages-stub/index.html | 11 ++++++----- pyproject.toml | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/pages-stub/index.html b/.github/pages-stub/index.html index 738657ec..4d3677df 100644 --- a/.github/pages-stub/index.html +++ b/.github/pages-stub/index.html @@ -3,7 +3,7 @@ - Coder Eval — evaluate AI coding agents and Claude Code skills + Coder Eval — evaluate AI coding agents and their skills

Coder Eval

- An open-source framework for evaluating and benchmarking AI coding agents and their Claude - Code skills: it runs a real agent — Claude Code, Codex, or Gemini — in a sandbox against - declarative YAML tasks, then scores the files and commands the agent actually produced. + Playwright for coding agents. An open-source, agent-agnostic framework for + evaluating and benchmarking AI coding agents and their skills: it runs a real agent — Claude + Code, Codex, Antigravity (Gemini), or OpenCode — in a sandbox against declarative YAML + tasks, then scores the files and commands the agent actually produced.

The documentation has moved to diff --git a/pyproject.toml b/pyproject.toml index 5eea61f9..3b22030f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "coder-eval" version = "0.11.6" -description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity) with sandboxed, reproducible YAML task suites." +description = "Evaluate, benchmark, and A/B-test AI coding agents (Claude Code, Codex, Gemini/Antigravity, OpenCode) with sandboxed, reproducible YAML task suites." readme = "README.md" license = "Apache-2.0" requires-python = ">=3.13" @@ -9,7 +9,7 @@ authors = [{ name = "UiPath", email = "coder-eval@uipath.com" }] keywords = [ "ai", "llm", "agent", "coding-agent", "evaluation", "eval", "evals", "benchmark", "swe-bench", "claude", "claude-code", "codex", "anthropic", - "gemini", "antigravity", "sandbox", "code-generation", "agent-evaluation", + "gemini", "antigravity", "opencode", "sandbox", "code-generation", "agent-evaluation", "llm-evaluation", "llm-eval", "ai-evaluation", "skills-evaluation", "claude-skills", "claude-code-skills", "agent-skills", "skillsbench", "agent-testing", "llmops",