diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4640ca8..9837189 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -23,7 +23,7 @@ "name": "galaxy-skills", "source": "./plugins/galaxy-skills", "description": "Curated Galaxy skills for running analyses -- connection, MCP tool surface, collections, user-defined tools, reports, reproducibility. Mirrors the skills/ tree of galaxyproject/galaxy-skills.", - "version": "2026.9.17", + "version": "2026.11.16", "category": "development", "keywords": [ "galaxy", @@ -36,7 +36,7 @@ "name": "galaxy-dev-skills", "source": "./plugins/galaxy-dev-skills", "description": "Galaxy development skills -- tool wrappers, Nextflow conversion, ToolShed revisions, track hubs, Hub posts. Mirrors the dev-skills/ tree of galaxyproject/galaxy-skills.", - "version": "2026.9.17", + "version": "2026.11.16", "category": "development", "keywords": [ "galaxy", @@ -49,7 +49,7 @@ "name": "foundry-skills", "source": "./plugins/foundry-skills", "description": "Galaxy Workflow Foundry cast skills (paper/Nextflow/CWL to validated Galaxy workflows) mirrored from galaxyproject/foundry.", - "version": "2026.9.17", + "version": "2026.11.16", "category": "development", "keywords": [ "galaxy", diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index b959a21..2f140bb 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -24,7 +24,7 @@ "name": "galaxy-skills", "source": "plugins/galaxy-skills", "description": "Curated Galaxy skills for running analyses -- connection, MCP tool surface, collections, user-defined tools, reports, reproducibility. Mirrors the skills/ tree of galaxyproject/galaxy-skills.", - "version": "2026.9.17", + "version": "2026.11.16", "category": "Developer Tools", "keywords": [ "galaxy", @@ -37,7 +37,7 @@ "name": "galaxy-dev-skills", "source": "plugins/galaxy-dev-skills", "description": "Galaxy development skills -- tool wrappers, Nextflow conversion, ToolShed revisions, track hubs, Hub posts. Mirrors the dev-skills/ tree of galaxyproject/galaxy-skills.", - "version": "2026.9.17", + "version": "2026.11.16", "category": "Developer Tools", "keywords": [ "galaxy", @@ -51,7 +51,7 @@ "name": "foundry-skills", "source": "plugins/foundry-skills", "description": "Galaxy Workflow Foundry cast skills mirrored from galaxyproject/foundry.", - "version": "2026.9.17", + "version": "2026.11.16", "category": "Developer Tools", "keywords": [ "galaxy", diff --git a/package.json b/package.json index 69f5fbb..84e0523 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@galaxyproject/agentic-plugins", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Galaxy MCP server configuration and curated Galaxy skills for Pi and other AI coding agents", "keywords": [ "pi-package", diff --git a/plugins/foundry-skills/.claude-plugin/plugin.json b/plugins/foundry-skills/.claude-plugin/plugin.json index 1300aed..6397fb9 100644 --- a/plugins/foundry-skills/.claude-plugin/plugin.json +++ b/plugins/foundry-skills/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "foundry-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Galaxy Workflow Foundry cast skills: schema-validated, corpus-grounded steps for turning papers, Nextflow pipelines and CWL workflows into validated Galaxy workflows. Mirror of galaxyproject/foundry (casts/claude/skills).", "author": { "name": "The Galaxy Project", diff --git a/plugins/foundry-skills/.codex-plugin/plugin.json b/plugins/foundry-skills/.codex-plugin/plugin.json index 4863424..a283f29 100644 --- a/plugins/foundry-skills/.codex-plugin/plugin.json +++ b/plugins/foundry-skills/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "foundry-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Galaxy Workflow Foundry cast skills: schema-validated, corpus-grounded steps for turning papers, Nextflow pipelines and CWL workflows into validated Galaxy workflows. Mirror of galaxyproject/foundry (casts/claude/skills).", "author": { "name": "The Galaxy Project", diff --git a/plugins/foundry-skills/.cursor-plugin/plugin.json b/plugins/foundry-skills/.cursor-plugin/plugin.json index 35a6fa8..d181d64 100644 --- a/plugins/foundry-skills/.cursor-plugin/plugin.json +++ b/plugins/foundry-skills/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "foundry-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Galaxy Workflow Foundry cast skills: schema-validated, corpus-grounded steps for turning papers, Nextflow pipelines and CWL workflows into validated Galaxy workflows. Mirror of galaxyproject/foundry (casts/claude/skills).", "author": { "name": "The Galaxy Project" diff --git a/plugins/foundry-skills/UPSTREAM.json b/plugins/foundry-skills/UPSTREAM.json index 693abbb..70da545 100644 --- a/plugins/foundry-skills/UPSTREAM.json +++ b/plugins/foundry-skills/UPSTREAM.json @@ -1,7 +1,7 @@ { "repository": "https://github.com/galaxyproject/foundry", "ref": "main", - "commit": "5aac370fffa778adabd42053dbb952dd5a77c1c7", + "commit": "161947ef7cd5c61b647aa3f49397ae2c04183009", "path": "casts/claude/skills", - "synced_at": "2026-09-17T20:49:33Z" + "synced_at": "2026-11-16T06:42:23Z" } diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/SKILL.md b/plugins/foundry-skills/skills/advance-galaxy-draft-step/SKILL.md index c1637bd..8f325e0 100644 --- a/plugins/foundry-skills/skills/advance-galaxy-draft-step/SKILL.md +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/SKILL.md @@ -13,8 +13,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs -- Read artifact `galaxy-workflow-draft`. Schema: galaxy-workflow-draft. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. gxformat2 draft (see galaxy-workflow-draft-format) mutated in-place across iterations; topology is fully concrete, individual tool steps may still carry `TODO_*` sentinels and `_plan_*` planning fields. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `galaxy-workflow-draft`. Schema: galaxy-workflow-draft. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-template`. gxformat2 draft (see galaxy-workflow-draft-format) mutated in-place across iterations; topology is fully concrete, individual tool steps may still carry `TODO_*` sentinels and `_plan_*` planning fields. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -24,19 +24,19 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache`. +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. Check: `galaxy-tool-cache --help | grep -q summarize`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli ## Load Upfront - `references/cli/draft-next-step.json`: CLI command reference packaged as a sidecar. Deterministically pick the next drafty step (or report no remaining work). The orchestrator owns the loop oracle so the harness reduces to `while draft: invoke skill`. Use when: at the start of every iteration, before any per-step work. -- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Read the ledger after each step implementation for a newly appended blocking entry, count the open blocking entries the convergence gate reads, and maintain the topology_repair escalation budget the loop's termination guard depends on. +- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Read the ledger for an open blocking entry against the picked step, or any open blocking entry once no drafty step is left, count the open blocking entries the convergence gate reads, and maintain the topology_repair escalation budget the loop's termination guard depends on. - `references/schemas/galaxy-workflow-draft.schema.json`: Schema file copied verbatim into the bundle. In/out contract: the draft this Mold reads and mutates one step per iteration conforms to galaxy-workflow-draft. Cast bundles the JSON Schema alongside the draft-validate CLI checks. ## Load On Demand @@ -45,8 +45,11 @@ Follow the procedure below and use the artifact/reference sections as the runtim - `references/cli/draft-extract.json`: CLI command reference packaged as a sidecar. At loop endstate, extract the concrete gxformat2 workflow from the fully-concretized draft — drop drafty steps, strip `_plan_*` fields, promote `class` to `GalaxyWorkflow` — and write it as the runnable `galaxy-workflow.gxwf.yml`. Use when: draft-next-step reports `draft: false` (no remaining drafty steps). - `references/cli/draft-validate.json`: CLI command reference packaged as a sidecar. Validate the draft and its concrete subset with --concrete --strict-state --json; skipped tool-state checks must fail the gate. Use when: after implementing or modifying the chosen step in the draft. - `references/cli/list.json`: CLI command reference packaged as a sidecar. Read a stock tool's cached version when the step plan has no version pin. Use when: in step 2, when the chosen step's tool is a bare/stock id and its concrete version isn't already known from a step-plan pin. +- `references/cli/summarize.json`: CLI command reference packaged as a sidecar. Regenerate the galaxy-tool-summary for a pin reused from a sibling step, from the populated cache, without re-running discovery. Use when: in step 3, when step 2 reused a sibling's `tool_id`/`tool_version` and no summary file for that pin is at hand. - `references/notes/galaxy-tool-job-failure-reference.md`: Research note copied verbatim into the bundle. Classify draft-validate diagnostics against wrapper-defined runtime failure semantics so the iteration routes back to the right authoring surface (implementation vs. wrapper choice). Use when: draft-validate fails after a step has been implemented, or when a selected wrapper has explicit failure semantics that may surface at runtime. +- `references/notes/galaxy-user-tool-workflow-binding.md`: Research note copied verbatim into the bundle. Pass an authored `GalaxyUserTool` straight to implementation. Use when: step 2 falls through to author-galaxy-tool-wrapper, or when the draft already contains a step whose `run:` is a `GalaxyUserTool`. - `references/schemas/galaxy-tool-summary.schema.json`: Schema file copied verbatim into the bundle. Bind the chosen step against the deterministic tool summary manifest emitted by summarize-galaxy-tool — read `parsed_tool` for ports/datatypes and `input_schemas.workflow_step_linked` for valid step `state` shape. Use when: after a wrapper has been resolved for the chosen step and before implementing it. +- `references/schemas/parsed-tool.schema.json`: Schema file copied verbatim into the bundle. Field-level shape of the `ParsedTool` payload under the summary's `parsed_tool`, which galaxy-tool-summary leaves as a placeholder: port names, datatypes, and a conditional's `whens[]` (`discriminator`, `is_default_when`) in declaration order. Use when: reading a field of `parsed_tool`, including a conditional's case index or default case. ## Validation @@ -56,39 +59,39 @@ Follow the procedure below and use the artifact/reference sections as the runtim Orchestrator skill for the per-step Galaxy authoring loop. One invocation advances the gxformat2 draft by **one** step: pick → resolve a wrapper → summarize the wrapper → implement the step → validate. The harness loop reduces to `while (gxwf draft-next-step ).draft: invoke skill`. -This skill is **single-entry, single-exit**: it owns the loop oracle (draft-next-step) and the per-step validator (draft-validate `--concrete`). Iterations terminate when the draft has no remaining drafty steps; on that terminal call the skill extracts the concrete `galaxy-workflow.gxwf.yml` (via draft-extract) — that promoted-class workflow, not the `-draft` file, is what downstream skills test and run — and the harness then drops out of the loop and proceeds to terminal validation via validate-galaxy-workflow. +This skill is **single-entry, single-exit**: it owns the loop oracle (draft-next-step) and the per-step validator (draft-validate `--concrete`). Iterations terminate when the draft has no remaining drafty steps; the call that leaves no drafty step (or a call that finds none) extracts the concrete `galaxy-workflow.gxwf.yml` (via draft-extract) — that promoted-class workflow, not the `-draft` file, is what downstream skills test and run — and the harness then drops out of the loop and proceeds to terminal validation via validate-galaxy-workflow. ### Sequence Choose a writable tool-cache directory for the run. Pass the same `--cache-dir ` to cache commands, wrapper summarization, and validation. -1. **Pick.** Run draft-next-step. If `draft: false`, the loop is done: run draft-extract to emit the concrete `galaxy-workflow.gxwf.yml` (drafty steps dropped, `_plan_*` stripped, `class` promoted to `GalaxyWorkflow`), then return. Otherwise carry the chosen step id forward. +1. **Pick.** Run draft-next-step. If `draft: false`, the loop is done: apply step 5 to any `open` blocking entry, then run draft-extract to emit the concrete `galaxy-workflow.gxwf.yml` (drafty steps dropped, `_plan_*` stripped, `class` promoted to `GalaxyWorkflow`), then return. Otherwise carry the chosen step id forward. If an `open` blocking entry against it already records the wrapper as confirmed absent, the author branch as ruled out, a found wrapper that can't serve the step, or why its wired inputs can't supply the step, go to step 5. Tool Shed misses and a tool version missing from a cache don't record that. 2. **Resolve a wrapper.** First check for an existing pin: - - **Already resolved in this draft** — find a concrete step with the `tool_id` named by the chosen step's `_plan_*` context or identity pin. Reuse its `tool_id` and `tool_version`, skipping discover-shed-tool for Tool Shed wrappers or cache/version lookup for stock tools. Continue to step 3. + - **Already resolved in this draft** — find a concrete step with the `tool_id` named by the chosen step's `_plan_*` context or identity pin. Reuse its `tool_id` and `tool_version`, skipping discover-shed-tool for Tool Shed wrappers or cache/version lookup for stock tools. An embedded user-defined tool has no `tool_id`: match a sibling whose `run:` carries the authored definition the plan calls for, by its `id` and `version`, and reuse that definition instead of authoring it again. Continue to step 3. Otherwise, split on whether the step's tool is a **built-in / stock** Galaxy tool — a bare id with no `owner/repo` path (`Filter1`, `sort1`, `Cut1`, `Show beginning1`, collection ops, `__APPLY_RULES__`): - **Built-in / stock** — the bare id *is* the wrapper identity; it does **not** route through discover-shed-tool (Tool Shed search) or author-galaxy-tool-wrapper. Only its concrete version needs resolving: the shed serves stock tools by bare id, but its TRS version-list endpoint can't auto-resolve the version, so read it from a populated cache via `galaxy-tool-cache list` or take a known pin from the step plan — never hand-guess a stock version. summarize-galaxy-tool then performs the bare-id `add`/`summarize` with that explicit `--tool-version`. - **Tool Shed wrapper** — branch on whether the template already pinned wrapper identity (see the tiers in galaxy-workflow-draft-format): - **Identity-pinned** — `tool_id` is concrete and `tool_version` is `TODO`. Treat the pin as a strong seed: confirm it via discover-shed-tool and resolve the changeset, correcting the `tool_id` only if discovery contradicts the pin (a pinned id is high-confidence template evidence, not a guess to re-derive from scratch). - - **Deferred** — `tool_id` is `TODO`. Search fresh: run discover-shed-tool against the step's `_plan_*` context. + - **Deferred** — `tool_id` is `TODO`. Search fresh: run discover-shed-tool against the step's `_plan_*` context, naming the source executable when `_plan_context` gives one. - Either way, if no acceptable shed candidate emerges, fall through to author-galaxy-tool-wrapper. + Either way, fall through to author-galaxy-tool-wrapper only on `miss`. Confirm a `weak` candidate and continue with it; one that confirmation shows doesn't run the named work counts as a `miss`. A found wrapper that can't serve the step goes to repair through implement's ledger entry (step 5). If the resolved pin is absent from the cache, run add ` --tool-version --cache-dir ` before summarization. In `@galaxy-tool-util/cli` 1.10.0, validation also fetches and caches missing metadata unless `--offline` is set; offline validation requires a populated cache. -3. **Summarize the wrapper.** If step 2 reused a sibling's pin, reuse the cached galaxy-tool-summary for that `tool_id`/`tool_version` pair. Otherwise, invoke summarize-galaxy-tool to produce the summary. -4. **Implement.** Invoke implement-galaxy-tool-step with the summary and the draft; it resolves the chosen step's remaining `TODO_*` / `_plan_*` slots into a concrete `tool_id` (confirming or correcting any pinned identity), `tool_version`, `state`, and wrapper-determined port names. -5. **Check computability.** Inspect the open-requirements-ledger for a new `open` blocking entry implement-galaxy-tool-step appended against this step. draft-validate cannot catch this: the connection graph knows ports connect, not what they carry, so the draft validates green even though the step can't run. If such an entry is present, escalate to repair-galaxy-draft-topology for a bounded repair (insert a producer/sub-path or honestly narrow the output), then update the ledger's `topology_repair` budget as the ledger note directs — each escalation must strictly reduce the open blocking-entry count, under a hard cap, and surrender rather than retry once the cap is reached. Then return — the next iteration resumes the loop, realizing any draft-tier steps the repair inserted. With no new blocking entry, continue to validation. -6. **Validate.** Run draft-validate ` --concrete --strict-state --json --cache-dir `. `--strict-state` makes skipped tool-state checks fail validation; draft structure and topology checks still run. If metadata is unavailable, resolve the cache or fetch error and retry. Return on exit 0; route other failures using the JSON diagnostics and the rules below. +3. **Summarize the wrapper.** If step 2 reused a sibling's pin, reuse that pin's galaxy-tool-summary: use the file an earlier iteration wrote, or regenerate it from the cache with `galaxy-tool-cache` summarize ` --tool-version --cache-dir --output `. If step 2 fell through to author-galaxy-tool-wrapper or reused an embedded definition, skip summarization: the authored `galaxy-user-tool.yml` is not in any toolbox or cache, and its own `inputs` and `outputs` are the port contract. Otherwise, invoke summarize-galaxy-tool to produce the summary. +4. **Implement.** Invoke implement-galaxy-tool-step with the summary, or the authored `galaxy-user-tool.yml`, and the draft. It resolves the chosen step's remaining `TODO_*` / `_plan_*` slots into a concrete `tool_id` (confirming or correcting any pinned identity), `tool_version`, `state`, and wrapper-determined port names. For an authored tool it embeds the definition under the step's `run:` instead, as galaxy-user-tool-workflow-binding describes. +5. **Check the ledger.** Inspect the open-requirements-ledger for an `open` blocking entry against this step: one that implement-galaxy-tool-step appended, or one an earlier call or skill raised. Once no drafty step is left, any `open` blocking entry counts, including one on a realized step or with no `step`. draft-validate cannot catch a computability gap: the connection graph knows ports connect, not what they carry, so the draft validates green even though the step can't run. If such an entry is present, escalate to repair-galaxy-draft-topology for a bounded structural edit, then update the ledger's `topology_repair` budget as the ledger note directs — each repair must leave fewer open blocking entries than it found, under a hard cap, and surrender rather than retry once the cap is reached. Then run draft-next-step: if it reports `draft: false`, the repair removed the last drafty step, so continue at step 6, which validates and extracts. Otherwise return — the next iteration resumes the loop, realizing any draft-tier steps the repair inserted. With no such entry, continue to validation. +6. **Validate.** Run draft-validate ` --concrete --strict-state --json --cache-dir `. `--strict-state` makes skipped tool-state checks fail validation; draft structure and topology checks still run. If metadata is unavailable, resolve the cache or fetch error and retry. On exit 0, run draft-next-step again: if it now reports `draft: false`, no drafty step is left, so apply step 5 to any `open` blocking entry, then run draft-extract to emit `galaxy-workflow.gxwf.yml` before returning, because a `while draft` harness never invokes the skill again. Route other failures using the JSON diagnostics and the rules below. ### Failure routing `draft-validate --concrete --strict-state --json` failures, after resolving metadata availability, fall into three buckets: - **Local to the just-implemented step** (sentinel violation, wrong port name, malformed `state`) — re-enter implement-galaxy-tool-step with the diagnostic. -- **Wrapper-choice mismatch** (selected wrapper cannot satisfy the step's `_plan_*` contract — wrong datatype, missing parameter, incompatible collection shape) — back out to step 2 and pick a different wrapper, either via discover-shed-tool with refined criteria or by escalating to author-galaxy-tool-wrapper. -- **Earlier-step defect surfaced by the growing concrete projection** (e.g. a connection that looked fine in isolation breaks once a downstream step pulls a previously-deferred port into scope) — flag to the user. The orchestrator does not unwind prior iterations on its own; cross-step rework belongs at the harness level. *Open question: at what threshold should this skill attempt to re-enter implement-galaxy-tool-step for an earlier step versus always escalating?* +- **Wrapper-choice mismatch** (selected wrapper cannot satisfy the step's `_plan_*` contract — wrong datatype, missing parameter, incompatible collection shape) — back out to step 2 and pick a different wrapper via discover-shed-tool with refined criteria. When no wrapper that runs the step's work satisfies the contract and it lacks only a parameter, drop that setting and append a plain entry. When it can't produce a declared output, append a blocking entry naming the wrapper and escalate as in step 5. +- **Earlier-step defect surfaced by the growing concrete projection** (e.g. a connection that looked fine in isolation breaks once a downstream step pulls a previously-deferred port into scope) — when a structural edit closes it, append a blocking entry naming the defect and escalate as in step 5. Otherwise flag it to the user. The orchestrator does not unwind prior iterations on its own. -These are red-`draft-validate` buckets. The fourth escalation path — a step output uncomputable from its wired inputs — is **not** one of them: the draft validates green there, so it is detected from the ledger in step 5 above, not from a validation failure. +These are red-`draft-validate` buckets. The fourth escalation path — a step output uncomputable from its wired inputs, or one the chosen wrapper can't produce — is **not** one of them: the draft validates green there, so it is detected from the ledger in step 5 above, not from a validation failure. Consult galaxy-tool-job-failure-reference when the wrapper has explicit failure semantics that affect routing — strict-shell behavior, dynamic outputs, or non-default stdio rules can present as wrapper-choice mismatches even when the static shape validates. diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/_provenance.json b/plugins/foundry-skills/skills/advance-galaxy-draft-step/_provenance.json index 5d9c0c7..2a69aef 100644 --- a/plugins/foundry-skills/skills/advance-galaxy-draft-step/_provenance.json +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "advance-galaxy-draft-step", "path": "content/molds/advance-galaxy-draft-step/index.md", - "revision": 7, - "content_hash": "ae19f7e5f5ac9434e31302efd88bb5f5db484caf0145951dd27dfe19247f2ea9", - "commit": "215b0a4c80c752a9eef2801d46accc4acce88af2" + "revision": 12, + "content_hash": "12f242ddbbfee7d9fe4ef8b6a2d701beceb71e4885cd19872c51041b04e08b27", + "commit": "db8801f2f0a31746e35cf399dd96cdbb8aa36dcf" }, - "cast_at": "2026-09-17T11:40:06.540Z", + "cast_at": "2026-11-10T10:22:51.277Z", "refs": [ { "kind": "cli-command", @@ -90,6 +90,22 @@ "dst_hash": "92d336667531c34180db6f75b9fdf87261923c1e85567dce3b21a6d87d9543bd", "source": "deterministic" }, + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[summarize]]", + "src": "content/cli/galaxy-tool-cache/summarize.md", + "dst": "references/cli/summarize.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Regenerate the [[galaxy-tool-summary]] for a pin reused from a sibling step, from the populated cache, without re-running discovery.", + "trigger": "In step 3, when step 2 reused a sibling's `tool_id`/`tool_version` and no summary file for that pin is at hand.", + "verification": "Cast the skill, reuse a sibling pin on an IWC-derived draft, confirm summarize --output writes a manifest that validates against galaxy-tool-summary.", + "src_hash": "96099b98e52244ac882066511a8ffa87c1fbbd6fb1c7acc98de3cca822f98277", + "dst_hash": "b51364ad4d415f6047e3cf3d3052dea1598cd2197595be66697c9777918b2de7", + "source": "deterministic" + }, { "kind": "research", "mode": "verbatim", @@ -101,8 +117,24 @@ "evidence": "corpus-observed", "purpose": "Classify [[draft-validate]] diagnostics against wrapper-defined runtime failure semantics so the iteration routes back to the right authoring surface (implementation vs. wrapper choice).", "trigger": "When draft-validate fails after a step has been implemented, or when a selected wrapper has explicit failure semantics that may surface at runtime.", - "src_hash": "bf6530fca24f40780836e92d8d263f789038a6e0a722351ab8e8c9aa42646e2c", - "dst_hash": "bf6530fca24f40780836e92d8d263f789038a6e0a722351ab8e8c9aa42646e2c", + "src_hash": "188511486605e05d6d6ffb555b9c31287a1b6bd94c1f67dd8e483ce258415bfa", + "dst_hash": "188511486605e05d6d6ffb555b9c31287a1b6bd94c1f67dd8e483ce258415bfa", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-user-tool-workflow-binding]]", + "src": "content/research/galaxy-user-tool-workflow-binding/index.md", + "dst": "references/notes/galaxy-user-tool-workflow-binding.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Pass an authored `GalaxyUserTool` straight to implementation.", + "trigger": "When step 2 falls through to [[author-galaxy-tool-wrapper]], or when the draft already contains a step whose `run:` is a `GalaxyUserTool`.", + "verification": "Promote after a loop run reaches `draft: false` with an authored tool embedded, and the extracted workflow imports into Galaxy with the tool resolved.", + "src_hash": "cfdf62f982349c02cdfa5f3214f3feb9f54cf664b880e438294e5748f9002902", + "dst_hash": "cfdf62f982349c02cdfa5f3214f3feb9f54cf664b880e438294e5748f9002902", "source": "deterministic" }, { @@ -114,10 +146,10 @@ "used_at": "runtime", "load": "upfront", "evidence": "hypothesis", - "purpose": "Read the ledger after each step implementation for a newly appended blocking entry, count the open blocking entries the convergence gate reads, and maintain the topology_repair escalation budget the loop's termination guard depends on.", - "verification": "Promote after a run where a blocking entry appended by implement is detected here and drives an escalation that strictly reduces the open count.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "purpose": "Read the ledger for an open blocking entry against the picked step, or any open blocking entry once no drafty step is left, count the open blocking entries the convergence gate reads, and maintain the topology_repair escalation budget the loop's termination guard depends on.", + "verification": "Promote after a run where a blocking entry appended by implement is detected here and drives an escalation that leaves fewer open blocking entries than it found.", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -147,8 +179,26 @@ "purpose": "In/out contract: the draft this Mold reads and mutates one step per iteration conforms to [[galaxy-workflow-draft]]. Cast bundles the JSON Schema alongside the [[draft-validate]] CLI checks.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", - "dst_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[parsed-tool]]", + "src": "package://@galaxy-tool-util/schema#parsedToolSchema", + "dst": "references/schemas/parsed-tool.schema.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Field-level shape of the `ParsedTool` payload under the summary's `parsed_tool`, which [[galaxy-tool-summary]] leaves as a placeholder: port names, datatypes, and a conditional's `whens[]` (`discriminator`, `is_default_when`) in declaration order.", + "trigger": "When reading a field of `parsed_tool`, including a conditional's case index or default case.", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "3966d0c425e561891113f5c71a2d36a42c00a78cb744bb8970c577b46abcff9b", + "dst_hash": "3966d0c425e561891113f5c71a2d36a42c00a78cb744bb8970c577b46abcff9b", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } @@ -187,7 +237,8 @@ "freeform-summary-to-galaxy-template", "implement-galaxy-tool-step", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-template" ] }, { @@ -210,7 +261,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/_required_tools.json b/plugins/foundry-skills/skills/advance-galaxy-draft-step/_required_tools.json index 4308170..b2c2491 100644 --- a/plugins/foundry-skills/skills/advance-galaxy-draft-step/_required_tools.json +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/_required_tools.json @@ -3,24 +3,25 @@ "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": [ "galaxy-tool-cache add", - "galaxy-tool-cache list" + "galaxy-tool-cache list", + "galaxy-tool-cache summarize" ] }, { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/cli/summarize.json b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/cli/summarize.json new file mode 100644 index 0000000..cf6ccef --- /dev/null +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/cli/summarize.json @@ -0,0 +1,50 @@ +{ + "type": "cli-command", + "tool": "galaxy-tool-cache", + "command": "summarize", + "summary": "Emit a deterministic galaxy-tool-summary manifest (cache provenance + embedded ParsedTool + generated input JSON Schemas) for a cached tool.", + "source_path": "content/cli/galaxy-tool-cache/summarize.md", + "source_revision": 2, + "package": "@galaxy-tool-util/cli", + "description": "Emit a deterministic summary manifest for a cached Galaxy tool", + "synopsis": "galaxy-tool-cache summarize [options] ", + "args": [ + { + "raw": "tool_id", + "name": "tool_id", + "required": true, + "variadic": false, + "description": "Tool ID" + } + ], + "options": [ + { + "flags": "--tool-version ", + "name": "toolVersion", + "description": "Tool version", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--output ", + "name": "output", + "description": "Output file (default: stdout)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--cache-dir ", + "name": "cacheDir", + "description": "Cache directory", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + } + ], + "body": "# `galaxy-tool-cache summarize`\n\nEmit a deterministic summary manifest for a tool already in the cache. This is the command [[summarize-galaxy-tool]] invokes: it does not hand-author the manifest, it runs `summarize` against a cache populated for the pin (see `add`). The cache entry must exist — `summarize` does not re-fetch; cache the pin with `add` first.\n\n`` plus `--tool-version` select the cached entry; they must match the pin used at `add` time and the cache key, or the wrong wrapper is summarized. `` accepts a full Tool Shed path, a TRS id, or a **bare/stock id** for built-in tools (`Show beginning1`, `Cut1`, …) — the same id form used at `add` time. A cache miss reports `Tool not found in cache: . Run 'galaxy-tool-cache add' first.`\n\n## Output\n\nA single JSON document conforming to [[galaxy-tool-summary]]. Top-level fields: `schema_version`, `tool_id`, `tool_version`, `cache_key`, `source`, `artifacts`, `parsed_tool`, `input_schemas`, `warnings`. The `parsed_tool` subtree is the upstream [[parsed-tool]] payload verbatim; `input_schemas.workflow_step` / `input_schemas.workflow_step_linked` carry generated JSON Schemas describing the tool's inputs at workflow-step authoring time. Default sink is stdout; `--output` writes a file.\n\n## Examples\n\n```bash\ngalaxy-tool-cache summarize toolshed.g2.bx.psu.edu/repos/iuc/staramr/staramr_search \\\n --tool-version 0.11.0+galaxy3 --cache-dir ~/.cache/gxwf \\\n --output galaxy-tool-summary.json\n\n# Stock/built-in tool by bare id (must be added with the same pin first)\ngalaxy-tool-cache summarize \"Show beginning1\" --tool-version 1.0.2 --cache-dir ~/.cache/gxwf\n```\n\n## Gotchas\n\n- Deterministic by design: same cached pin → byte-identical manifest. Re-run after re-`add`ing if the wrapper revision changed.\n- Wrapper facts not yet exposed by the upstream `ParsedTool` (requirements, containers, stdio) surface additively as Galaxy extends `ParsedTool`; absence is a coverage gap, not a wrapper without those declarations — cross-check against raw XML when the manifest's `warnings` flag a lossy surface.\n- For just the input JSON Schema (not the full manifest), use the `schema` subcommand instead; `summarize` already embeds it under `input_schemas`." +} diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/galaxy-tool-job-failure-reference.md b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/galaxy-tool-job-failure-reference.md index 0630071..9934a99 100644 --- a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/galaxy-tool-job-failure-reference.md +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/galaxy-tool-job-failure-reference.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-workflow-invocation-failure-reference]]" - "[[planemo-workflow-test-architecture]]" @@ -15,123 +15,130 @@ related_molds: - "[[implement-galaxy-tool-step]]" - "[[debug-galaxy-workflow-output]]" sources: - - "~/projects/repositories/galaxy/lib/galaxy/tools/__init__.py" - - "~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/xml.py" - - "~/projects/repositories/galaxy/lib/galaxy/tool_util/output_checker.py" - - "~/projects/repositories/galaxy/lib/galaxy/jobs" - - "~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/xml.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/stdio.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/output_checker.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/command_factory.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/runners/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/model/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/schema.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/api/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/services/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/jobs.py" summary: "Reference for Galaxy tool stdio rules, job failure detection, job states, and job API failure surfaces." --- -# Galaxy Tool And Job Failure Reference +# Galaxy tool and job failure reference -This is reference material, not a debug recipe. Use it to understand what Galaxy can know about a failed tool job and which API surfaces preserve that evidence. +A failed Galaxy job preserves execution evidence separately from workflow scheduling and output assertions. Start with its state, exit code, structured messages, and separate tool and runner streams. A red downstream dataset or stderr text alone does not identify the original failure. -## Model +[[galaxy-workflow-invocation-failure-reference]] covers scheduling and dependency messages. [[planemo-workflow-test-architecture]] locates the structured Planemo artifacts, and [[debug-galaxy-workflow-output]] uses this evidence to classify a failure before proposing repairs. -Galaxy tool failure handling is layered: +## Job evidence -- The tool wrapper defines expected failure semantics through `detect_errors`, ``, exit-code checks, regex checks, and command strictness. -- The job runner executes the command and captures exit code plus tool/job stdout and stderr streams. -- Galaxy evaluates configured failure rules and records structured `job_messages`. -- The job reaches a terminal state, output datasets may become `error`, and dependent jobs may pause or fail later. -- Workflow invocation APIs summarize those jobs, but job APIs preserve the most detailed tool-level evidence. +Full job detail includes execution identity, state, exit code, structured messages, dependencies, and separate tool and runner streams. The combined stream fields provide compatibility views of those streams. -## Tool Wrapper Failure Controls +| Field | Evidence | +|---|---| +| `id`, `tool_id`, `tool_version`, `state` | Identify the execution and the wrapper version whose failure rules apply. | +| `exit_code` | Process exit code recorded by the runner. A missing value is not evidence of exit code zero. | +| `tool_stdout`, `tool_stderr` | Captured streams from the tool command. | +| `job_stdout`, `job_stderr` | Captured streams from the job wrapper or runner context. | +| `stdout`, `stderr` | Combined compatibility views of tool and job streams. They lose the stream provenance. | +| `job_messages` | Structured records from stdio checks or output discovery. Empty messages do not prove success. | +| `dependencies` | Job dependencies included with full detail. | -Important wrapper controls: +Prefer separate streams for diagnosis. A runner failure can occur before the tool executes, and output processing can fail after the command exits successfully. -| Control | Meaning | -|---|---| -| `detect_errors="default"` | For non-legacy XML tools without explicit ``, Galaxy defaults to non-zero exit-code failure detection. | -| `detect_errors="exit_code"` | Adds fatal checks for non-zero exit codes, plus optional OOM exit-code handling. | -| `detect_errors="aggressive"` | Adds non-zero exit-code checks plus broad stdout/stderr regexes for OOM and generic error text. | -| `` | Adds explicit exit-code ranges or values with levels such as `fatal`, `warning`, or `fatal_oom`. | -| `` | Adds case-insensitive stdout/stderr regex checks. | -| `` | Controls shell strictness; newer tool profiles default to `set -e` behavior. | +## Job and output dataset states -Current Galaxy source parses these rules while loading the tool, then stores exit-code and regex rules on the tool object. XML parser behavior lives under `~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/xml.py`; concrete stdio presets live under `~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/stdio.py`. +| Job state | Interpretation | +|---|---| +| `new`, `waiting`, `upload`, `queued`, `running`, `resubmitted` | Execution is pending or in progress. A timeout while polling these states needs timing and runner evidence. | +| `ok` | Galaxy accepted the execution. Check the expected outputs and assertions separately. | +| `error`, `failed` | Failure states. Inspect full detail and the producing job's outputs. | +| `paused` | Execution is blocked or paused. Inspect upstream inputs and pause information before blaming the tool command. | +| `skipped` | Execution was skipped, for example by a workflow condition. It is not itself a tool failure. | +| `stop`, `stopped`, `deleting`, `deleted` | Stopping or deletion states. Determine whether cancellation or deletion explains the missing result. | -## Rule Ordering And Levels +The API value for a stop in progress is `stop`, not `stopping`. Consumers use different terminal-state sets. Do not infer tool success from a workflow completion record, or infer permanent failure merely because polling stopped. -Galaxy evaluates explicit and preset rules in a defined order: +Output datasets have their own states, including `ok`, `error`, `paused`, `failed_metadata`, `deferred`, and `discarded`. `failed_metadata` identifies a metadata problem, while `deferred` denotes data awaiting materialization. Dataset deletion and purging are separate flags. For a red downstream output, trace its producing job and upstream datasets instead of treating job and dataset states as interchangeable. For mapped jobs, retain the failed member's job ID and collection element identifier. See [[galaxy-collection-semantics]]. -- Exit-code rules are evaluated before regex rules. -- Regexes search stdout and/or stderr case-insensitively. -- A fatal rule stops later checks. -- Warnings, log messages, and QC messages can produce `job_messages` without failing the job. +## Wrapper failure rules -Useful levels: +Galaxy loads an XML wrapper's failure rules from `` and ``, then checks the captured tool exit code and streams. Inspect the installed wrapper and its `profile` before interpreting a non-zero exit or stderr text. -| Level | Effect | +| Control | Behavior | |---|---| -| `log` | Records informational message; does not fail the job. | -| `qc` | Records QC message; does not fail the job. | -| `warning` | Records warning; does not fail the job. | -| `fatal` | Fails the job as a generic tool error. | -| `fatal_oom` | Fails as out-of-memory; runner behavior can use this for OOM handling/resubmission. | +| Omitted `detect_errors` or `detect_errors="default"` | Adds non-zero exit-code checks for profiles other than legacy `16.01` only when there is no `` element. An explicit `` disables this implicit preset, even if it contains no rules. | +| `detect_errors="exit_code"` | Adds fatal checks for negative and positive non-zero exit codes. Optional `oom_exit_code` adds an OOM-specific check before those ranges. | +| `detect_errors="aggressive"` | Adds non-zero exit checks and case-insensitive checks on both streams for OOM text, `exception:`, and `error:`. | +| `` | Defines explicit values or inclusive ranges and their severity. Open-ended ranges such as `1:` are supported. | +| `` | Searches selected tool streams case-insensitively. With no source specified, both stdout and stderr are searched. | +| `` | Requests `set -e` in the generated tool script. The default is true for profiles `20.09` and newer, false for older profiles. | -Low-confidence caveat: OOM resubmission behavior is runner/destination configuration dependent. The wrapper and output checker can classify OOM, but retry policy is not solely a wrapper property. +Explicit stdio rules are prepended to rules from an explicitly selected `exit_code` or `aggressive` preset. Adding a warning for an exit code does not override a later fatal preset rule matching that code. -## Job And Dataset States +When **no exit-code or regex rules exist**, the output checker falls back to treating any nonempty tool stderr as an error. In this fallback, a non-zero exit code with empty stderr does not itself trigger failure. Conversely, with configured rules, ordinary stderr text is not automatically fatal. -Job states relevant to workflow tests include: +Shell strictness and stdio checks serve different purposes. `set -e` can stop a multi-command script before later commands hide an earlier error, but follows the shell's exceptions and does not add `pipefail`. It does not guarantee that every component of a pipeline succeeded. Galaxy's command factory also bypasses script creation when the configured shell is `none`. -- In progress or queued: `new`, `queued`, `running`, `waiting`, `upload`, `resubmitted`. -- Success: `ok`. -- Failure or terminal problem: `error`, `failed`, `paused`, `stopped`, `deleted`, `skipped` depending context. +## Rule ordering and structured messages -Dataset states relevant to downstream failures include `ok`, `error`, `paused`, `failed_metadata`, `deferred`, and `discarded`. +Exit-code checks run before regex checks. Within each group, explicit rules precede presets. Regex rules check stderr before stdout when both are selected. -For workflow debugging, do not collapse job state and dataset state. A job can fail, its outputs can become error datasets, and a downstream workflow step can later fail because it consumes those datasets. +| XML level | Numeric `error_level` | Effect | +|---|---:|---| +| `log` | 1 | Records an informational message. | +| `qc` | 1.1 | Records a quality-control message. | +| `warning` | 2 | Records a warning. | +| `fatal` | 3 | Classifies a generic tool error. | +| `fatal_oom` | 4 | Classifies an out-of-memory error and stops further checking. | -## Stream And Message Fields +Log, QC, and warning matches do not fail the job by themselves. A generic `fatal` match does **not** stop all later checks. A later OOM match can upgrade the classification. The checker tracks the maximum severity, and level 4 ends checking. OOM classification can inform runner resubmission, but retry policy depends on the runner and destination configuration. -Galaxy distinguishes tool streams from job/runner streams: +Structured `job_messages` retain the matching rule evidence: -| Field | Meaning | +| Message `type` | Additional fields | |---|---| -| `tool_stdout` | stdout from the executed tool command. | -| `tool_stderr` | stderr from the executed tool command. | -| `job_stdout` | stdout from job wrapper/runner context. | -| `job_stderr` | stderr from job wrapper/runner context. | -| `stdout` | Combined stdout compatibility view. | -| `stderr` | Combined stderr compatibility view. | -| `job_messages` | Structured failure/warning messages produced by stdio and output checking. | -| `exit_code` | Process exit code observed by the runner. | +| `exit_code` | `exit_code`, `code_desc`, `desc`, `error_level`. | +| `regex` | `stream`, `match`, `code_desc`, `desc`, `error_level`. | +| `max_discovered_files` | `desc`, `error_level` for output-discovery failures, including too many files or an output name that is too long. | -Prefer `job_messages` plus separate streams for reference-quality failure interpretation. Combined `stdout` and `stderr` are useful for humans but lose provenance. +Regex `match` text is truncated after 256 characters when longer. `code_desc` is the rule's description, while `desc` includes the detected severity and explanation. Preserve the raw message rather than reducing it to “stderr contained an error.” Fallback stderr failure and runner errors need not produce stdio messages. -## API Surfaces +## Job API reference -Useful job APIs from Galaxy source under `~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/jobs.py`: +Paths are relative to the server URL and use encoded API IDs. -| API | Use | +| Method and path | Result | |---|---| -| `GET /api/jobs` | Filter jobs by state, tool id, workflow id, invocation id, history id, etc. | -| `GET /api/jobs/{job_id}` | Basic job detail. | -| `GET /api/jobs/{job_id}?full=true` | Full job detail, including streams and `job_messages`. | +| `GET /api/jobs` | Lists accessible jobs with filters including state, tool ID, history ID, workflow ID, and invocation ID. | +| `GET /api/jobs/{job_id}` | Basic job detail, including state, tool identity, and exit code. | +| `GET /api/jobs/{job_id}?full=true` | Adds separate and combined streams, `job_messages`, and dependencies. | | `GET /api/jobs/{job_id}/stdout` | Combined stdout as plain text. | | `GET /api/jobs/{job_id}/stderr` | Combined stderr as plain text. | -| `GET /api/jobs/{job_id}/console_output` | Live or stored tool stdout/stderr, useful while a job is running. | -| `GET /api/jobs/{job_id}/inputs` | Input datasets for the job. | -| `GET /api/jobs/{job_id}/outputs` | Output datasets and output collections. | -| `GET /api/jobs/{job_id}/metrics` | Job metrics, if available. | -| `GET /api/jobs/{job_id}/common_problems` | Known simple problems such as empty or duplicate inputs. | +| `GET /api/jobs/{job_id}/console_output` | Tool console output, subject to the requirements below. | +| `GET /api/jobs/{job_id}/inputs` | Job input datasets. | +| `GET /api/jobs/{job_id}/outputs` | Job output datasets and collections. | +| `GET /api/jobs/{job_id}/metrics` | Job metrics subject to access policy and availability. | +| `GET /api/jobs/{job_id}/common_problems` | Simple checks such as empty or duplicate inputs. These are clues, not a complete diagnosis. | -Admin-only or admin-enriched fields can include command line, traceback, destination, runner, handler, external id, and destination parameters. Do not assume a normal workflow-testing API key can retrieve everything. +`console_output` requires four query parameters: `stdout_position`, `stdout_length`, `stderr_position`, and `stderr_length`. They define the starting offsets and lengths of the requested stream ranges. -## Durable Reference Use +The destination must enable `live_tool_output_reporting`, including when retrieving a finished job through this endpoint. While running, Galaxy reads the requested ranges from tool stream files. Otherwise it returns stored tool streams. Full job detail exposes stored failure evidence independently of live reporting. -This note should inform generated skills when they need to preserve or inspect tool-level failure evidence: +Full detail does not grant administrator access. System details such as traceback, runner, destination, handler, and external job ID can require an administrator. Command-line exposure also depends on server configuration, and full-detail `job_metrics` are added for administrators. A normal workflow-testing key may therefore lack some server or runner evidence. -- A concrete workflow step should preserve tool id, version, input labels, output labels, datatype and collection shape so a later job failure can be traced back to the authoring decision. -- A runtime failure should not be classified from stderr alone if `job_messages`, exit code, and wrapper stdio rules are available. -- A workflow invocation failure may be caused by an upstream job, but invocation APIs are not a substitute for full job detail. +## Diagnostic pitfalls and verification limits -## Low-Confidence Or Version-Sensitive Points +- **Stderr is evidence, not a verdict.** Warnings can be harmless under configured stdio rules. An aggressive regex can also classify text as fatal despite exit code zero. Compare streams with the installed wrapper's rules. +- **Exit code zero is not scientific success.** Output collection, metadata, and assertions can still fail. Preserve output states and Planemo's `output_problems` alongside job records. +- **A paused consumer may hide an upstream failure.** Follow dependencies and producing jobs before changing the consumer's parameters. +- **No messages is not a clean bill of health.** A runner, fallback stderr check, or output-processing failure can leave no matching stdio record. Keep all available streams and state evidence. +- **Resubmission changes the context.** Record runner and destination details when available and distinguish an OOM classification from a configured retry decision. -- Tool profile defaults changed over time. Prefer current Galaxy source and wrapper profile over old generic rules. -- Some docs describe failure messages as stream text; current code preserves structured `job_messages` separately. -- OOM handling depends on job runner and destination configuration. -- Planemo may print or summarize job detail differently from raw Galaxy APIs; see [[planemo-workflow-test-architecture]]. +The parser, checker, state definitions, and API contracts were inspected in the pinned Galaxy source. No live run verified every runner's stream capture, OOM resubmission, or access configuration. Check the deployed server and installed wrapper before applying version-sensitive conclusions. diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/galaxy-user-tool-workflow-binding.md b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/galaxy-user-tool-workflow-binding.md new file mode 100644 index 0000000..4a5bc55 --- /dev/null +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/galaxy-user-tool-workflow-binding.md @@ -0,0 +1,91 @@ +--- +type: research +title: "Binding a workflow step to a user-defined tool" +tags: + - target/galaxy +status: draft +created: 2026-09-24 +revised: 2026-10-23 +revision: 4 +related_notes: + - "[[galaxy-user-tool-authoring-help]]" + - "[[galaxy-user-tool-authoring]]" + - "[[galaxy-workflow-draft-format]]" + - "[[gxformat2-schema]]" +related_molds: + - "[[author-galaxy-tool-wrapper]]" + - "[[implement-galaxy-tool-step]]" + - "[[advance-galaxy-draft-step]]" +sources: + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/client/src/components/Tool/authoringHelp.yml" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy/managers/tools.py" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy_test/workflow/inline_user_defined_tool.gxwf.yml" +summary: "How a workflow step uses an authored GalaxyUserTool: embedded under run:, never by tool_uuid, tool_id or content_id; registering via /api/unprivileged_tools." +--- + +A gxformat2 step that uses an authored `GalaxyUserTool` carries the whole tool definition under its `run:` key. That is the only way the Foundry binds a step to a user-defined tool. + +A step never names a user-defined tool by `tool_uuid`, `tool_id`, or `content_id` alone. The tool is not in a server's toolbox, so a `tool_id` does not resolve to it, even when the string matches the tool's `id`. A `uuid` identifies one registered copy of the tool, for its owner, on one server. Galaxy's workflow editor and its exports write any such references themselves, and Galaxy rejects a `tool_uuid` step without an embedded definition whose `tool_id` names a different tool. + +The rule is derived from the `workflows` section ("Using a tool in a workflow") of Galaxy's authoring help, vendored in [[galaxy-user-tool-authoring-help]]. The Galaxy-side facts below are pinned to the same commit. The gxwf behavior was checked with `@galaxy-tool-util/cli` 1.14.0, the version the Foundry pins. + +## Embedding the definition in a gxformat2 step + +The embedded form makes the workflow self-contained. When a user imports it, Galaxy registers a private copy of the tool for that user. That requires a server with `enable_beta_tool_formats` set and a user holding the `Custom Tool Execution` role, the same conditions as registering the tool directly. Converting the workflow to native `.ga` with gxformat2 carries the definition as `tool_representation`, with no `tool_id`, `tool_version`, or `tool_uuid`, and Galaxy imports that form the same way. Galaxy's own `.ga` export also includes the definition as `tool_representation`. Nothing has to be registered first, and nothing has to be injected into the converted file afterwards. + +To bind a step: + +- Put the complete `GalaxyUserTool` document under `run:`, unchanged. Its `id` and `version` identify the tool. +- Leave out `tool_id`, `tool_version`, and `tool_shed_repository`. They describe toolbox tools. Remove any `TODO` sentinel the template left in those fields rather than filling it. +- Key `in:` by the names of the definition's `data` and `collection` inputs. +- List every output a downstream step or a workflow output uses under `out:`, by the definition's output `name`. +- Set values for non-data inputs in `state`, keyed by input `name`. + +```yaml +steps: + head: + run: + class: GalaxyUserTool + id: head_lines + version: 0.1.0 + name: Head lines + container: quay.io/biocontainers/python:3.13 + shell_command: head -n '$(inputs.n_lines)' '$(inputs.data_input.path)' > out.txt + inputs: + - name: n_lines + type: integer + value: 10 + - name: data_input + type: data + format: [txt] + outputs: + - name: out + type: data + format: txt + from_work_dir: out.txt + in: + data_input: text_file + out: + - id: out + state: + n_lines: 5 +``` + +gxformat2's converter turns that step into a native tool step whose `tool_state` contains `n_lines: 5`, with the definition in `tool_representation`. Galaxy's own test workflow `inline_user_defined_tool.gxwf.yml` exercises the same form end to end, without `out:` or `state`. + +## Validating a draft with an embedded tool + +gxwf treats a `GalaxyUserTool` under `run:` as a tool step. [[draft-validate]] with and without `--concrete`, [[draft-next-step]], [[draft-extract]], and `gxwf validate` on the extracted workflow all run as they do for toolbox steps, and `draft-extract` keeps workflow outputs sourced from the embedded step even without an `out:` list. `gxwf validate-tool-source ` validates the definition itself when it is saved as a separate file. + +## Registering a tool on a server + +A step that embeds its tool does not need it registered. Registering is for running the tool by itself, outside a workflow, or for binding a native `.ga` step by `tool_uuid`. + +- Register with `POST /api/unprivileged_tools`, body `{"representation": }`. It needs the same `enable_beta_tool_formats` setting and `Custom Tool Execution` role as importing an embedded step, and returns the registered copy's `uuid`. The `api` section of [[galaxy-user-tool-authoring-help]] lists the other endpoints. +- Do not use `POST /api/dynamic_tools`. It is admin-only and returns 403 for an ordinary user. +- A returned `uuid` means Galaxy accepted the definition, not that the tool runs. Run it on test data before reporting it as working. + +## Open questions + +- Whether a Planemo-managed Galaxy runs a workflow with an embedded tool has not been tested. It would need `enable_beta_tool_formats` on that server and the `Custom Tool Execution` role for the test user. diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/schemas/galaxy-workflow-draft.schema.json index 3a39f43..dd3a94a 100644 --- a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/schemas/galaxy-workflow-draft.schema.json +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/schemas/galaxy-workflow-draft.schema.json @@ -75,7 +75,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_version": { "anyOf": [ @@ -108,7 +108,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "out": { "anyOf": [ @@ -309,7 +309,7 @@ ] }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "release": { "anyOf": [ @@ -412,22 +412,28 @@ "items": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" } ] }, @@ -437,22 +443,28 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" }, { "type": "string" @@ -927,7 +939,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "label": { "anyOf": [ @@ -1040,7 +1052,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_state": { "anyOf": [ @@ -1856,7 +1868,9 @@ "text", "File", "data", - "collection" + "collection", + "color", + "directory_uri" ], "type": "string" } @@ -1952,10 +1966,10 @@ ] }, "restrictions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "suggestions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "type": { "enum": [ @@ -1963,6 +1977,9 @@ "string" ], "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" } }, "required": [ @@ -1971,6 +1988,96 @@ "type": "object" }, "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { "additionalProperties": false, "properties": { "default": { @@ -2016,7 +2123,7 @@ "required": [], "type": "object" }, - "Shared10": { + "Shared11": { "additionalProperties": false, "properties": { "default": { @@ -2124,7 +2231,7 @@ ], "type": "object" }, - "Shared11": { + "Shared12": { "additionalProperties": false, "properties": { "default": { @@ -2230,7 +2337,7 @@ "required": [], "type": "object" }, - "Shared12": { + "Shared13": { "additionalProperties": false, "properties": { "default": { @@ -2337,7 +2444,7 @@ ], "type": "object" }, - "Shared13": { + "Shared14": { "additionalProperties": false, "properties": { "default": { @@ -2424,7 +2531,143 @@ ], "type": "object" }, - "Shared14": { + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { "anyOf": [ { "items": { @@ -2457,7 +2700,7 @@ } ] }, - "Shared15": { + "Shared18": { "anyOf": [ { "type": "null" @@ -2488,11 +2731,11 @@ } ] }, - "Shared16": { + "Shared19": { "anyOf": [ { "items": { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, "type": "array" }, @@ -2500,7 +2743,7 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, { "type": "string" @@ -2522,7 +2765,7 @@ } ] }, - "Shared17": { + "Shared20": { "anyOf": [ { "type": "null" @@ -2610,7 +2853,7 @@ "$ref": "#/$defs/Shared1" }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "uuid": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/schemas/parsed-tool.schema.json b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/schemas/parsed-tool.schema.json new file mode 100644 index 0000000..52bda10 --- /dev/null +++ b/plugins/foundry-skills/skills/advance-galaxy-draft-step/references/schemas/parsed-tool.schema.json @@ -0,0 +1,930 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "id", + "version", + "name", + "description", + "inputs", + "outputs", + "citations", + "license", + "profile", + "edam_operations", + "edam_topics", + "xrefs" + ], + "properties": { + "id": { + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "name": { + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "inputs": { + "type": "array", + "items": { + "type": "object" + } + }, + "outputs": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type", + "format", + "format_source", + "metadata_source", + "discover_datasets", + "from_work_dir", + "precreate_directory" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "data" + ] + }, + "format": { + "type": "string" + }, + "format_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "metadata_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "discover_datasets": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path", + "sort_key", + "sort_comp", + "sort_reverse", + "pattern" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "pattern" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + }, + "sort_key": { + "type": "string", + "enum": [ + "filename", + "name", + "designation", + "dbkey" + ] + }, + "sort_comp": { + "type": "string", + "enum": [ + "lexical", + "numeric" + ] + }, + "sort_reverse": { + "type": "boolean" + }, + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "tool_provided_metadata" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "from_work_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "precreate_directory": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "collection" + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_from_rules": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "structured_like": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "discover_datasets": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path", + "sort_key", + "sort_comp", + "sort_reverse", + "pattern" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "pattern" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + }, + "sort_key": { + "type": "string", + "enum": [ + "filename", + "name", + "designation", + "dbkey" + ] + }, + "sort_comp": { + "type": "string", + "enum": [ + "lexical", + "numeric" + ] + }, + "sort_reverse": { + "type": "boolean" + }, + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "tool_provided_metadata" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "structure": { + "type": "object", + "required": [ + "collection_type", + "collection_type_source", + "collection_type_from_rules", + "structured_like", + "discover_datasets" + ], + "properties": { + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_from_rules": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "structured_like": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "discover_datasets": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path", + "sort_key", + "sort_comp", + "sort_reverse", + "pattern" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "pattern" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + }, + "sort_key": { + "type": "string", + "enum": [ + "filename", + "name", + "designation", + "dbkey" + ] + }, + "sort_comp": { + "type": "string", + "enum": [ + "lexical", + "numeric" + ] + }, + "sort_reverse": { + "type": "boolean" + }, + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "tool_provided_metadata" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "integer" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "float" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "boolean" + ] + } + }, + "additionalProperties": false + } + ] + } + }, + "citations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "type", + "content" + ], + "properties": { + "type": { + "type": "string" + }, + "content": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "license": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "edam_operations": { + "type": "array", + "items": { + "type": "string" + } + }, + "edam_topics": { + "type": "array", + "items": { + "type": "string" + } + }, + "xrefs": { + "type": "array", + "items": { + "type": "object", + "required": [ + "value", + "type" + ], + "properties": { + "value": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "help": { + "anyOf": [ + { + "type": "object", + "required": [ + "format", + "content" + ], + "properties": { + "format": { + "type": "string" + }, + "content": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false +} diff --git a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/SKILL.md b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/SKILL.md index f84316e..ae646cc 100644 --- a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/SKILL.md +++ b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/SKILL.md @@ -15,7 +15,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `starting-galaxy-workflow`. Produced by `summarize-galaxy-workflow`. The normalized concrete gxformat2 workflow being modified, emitted by summarize-galaxy-workflow (passthrough when already gxformat2, the `.ga`→gxformat2 conversion otherwise). The substrate edits apply to — untouched regions must survive byte-for-byte. - Read artifact `galaxy-workflow-changeset`. Produced by `interview-to-galaxy-workflow-changeset`. Reviewed, step-anchored change-set from interview-to-galaxy-workflow-changeset; the edit intents to realize. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -24,8 +24,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli @@ -51,7 +51,7 @@ Apply a reviewed change-set to a concrete Galaxy workflow and emit a `galaxy-wor ### What to apply directly vs. defer -- **Direct edits (applied inline, no drafty step).** `remove-step`, `rewire`, `relabel` / `annotate`, `change-parameter`, `add-input` / `remove-input`, `add-output` / `expose-output`. These need no tool resolution: edit the concrete step/input/output in place and rewire consumers as the change-set dictates. A change-set of purely direct edits yields a draft with `draft: false` — an immediately concrete workflow, no loop iterations. +- **Direct edits (applied inline, no drafty step).** `remove-step`, `rewire`, `relabel` / `annotate`, `change-parameter`, `add-input` / `remove-input`, `add-output` / `expose-output`, `remove-output`. These need no tool resolution: edit the concrete step/input/output in place and rewire consumers as the change-set dictates. A change-set of purely direct edits yields a draft with `draft: false` — an immediately concrete workflow, no loop iterations. - **Deferred edits (injected as drafty steps).** `add-step` and `replace-tool` (when the replacement is a different tool or an unpinned version). Add the step at draft tier with `tool_id: TODO` (or Identity-pinned when the change-set pinned an id), and record the operation, source evidence, and constraints in `_plan_state` / `_plan_context` / `_plan_in` / `_plan_out` so advance-galaxy-draft-step can resolve a wrapper and concretize it. ### Preserve untouched regions diff --git a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_provenance.json b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_provenance.json index 0c414a0..0f96aad 100644 --- a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_provenance.json +++ b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "apply-galaxy-workflow-changeset", "path": "content/molds/apply-galaxy-workflow-changeset/index.md", - "revision": 3, - "content_hash": "82d8e6e440fc48fdb4e7c5a5e25591ecc8e8a248ea03cc950df5096256656b62", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 4, + "content_hash": "f589ff84acc67e2172a57ce26ae44b865786d5d739615309b0085c323c0389f0", + "commit": "db8801f2f0a31746e35cf399dd96cdbb8aa36dcf" }, - "cast_at": "2026-09-13T20:58:26.175Z", + "cast_at": "2026-11-10T10:22:51.298Z", "refs": [ { "kind": "cli-command", @@ -37,8 +37,8 @@ "evidence": "hypothesis", "purpose": "Represent the whole workflow as a draft: carry untouched steps at Resolved tier, inject tool-introducing/replacing edits as drafty steps with _plan_state / _plan_context / _plan_in / _plan_out.", "verification": "Promote after a downstream advance-galaxy-draft-step run drains the injected drafty steps without round-tripping back through the change-set.", - "src_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", - "dst_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", + "src_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", + "dst_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", "source": "deterministic" }, { @@ -52,8 +52,8 @@ "evidence": "hypothesis", "purpose": "Close the open entries the change-set discharges, and append a blocking entry for an edited region whose output the rewired inputs cannot supply, so an edit-introduced gap is recorded rather than discovered downstream.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -68,8 +68,8 @@ "purpose": "Output contract: the emitted gxformat2 draft conforms to [[galaxy-workflow-draft]]. Cast bundles the JSON Schema so the skill carries its output shape alongside [[draft-validate]] checks.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", - "dst_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } @@ -125,7 +125,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_required_tools.json b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_required_tools.json index 54fd1c9..18f1089 100644 --- a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_required_tools.json +++ b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/_required_tools.json @@ -3,9 +3,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", diff --git a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/galaxy-workflow-draft-format.md b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/galaxy-workflow-draft-format.md index 2a66f1a..9fc6e51 100644 --- a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/galaxy-workflow-draft-format.md +++ b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/galaxy-workflow-draft-format.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-06 -revised: 2026-05-10 -revision: 2 +revised: 2026-11-10 +revision: 6 related_notes: - "[[gxformat2-schema]]" - "[[galaxy-data-flow-draft-contract]]" @@ -17,6 +17,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[implement-galaxy-tool-step]]" - "[[advance-galaxy-draft-step]]" @@ -117,14 +118,14 @@ steps: ## Topology repair escalation -The template Mold settles topology, but it can miss a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. +The template Mold settles topology, but a settled draft can still be unable to give a correct run. The common case is a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. When one surfaces, the per-step loop does not author topology to fix it. It **escalates**: -- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. -- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and re-wires the affected region: one producer step, a small sub-path, or honest narrowing of the output. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. +- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs, or that the chosen wrapper can't produce it. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. Earlier Molds and the loop raise blocking entries too. +- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and makes the smallest structural edits that close them. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. -This is **repair**, distinct from **settling**: it is narrow (one named region), reactive (only a detected computability gap triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each escalation must strictly reduce that count, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. +This is **repair**, distinct from **settling**: it is narrow (the entries bound it), reactive (an open blocking entry triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each repair must leave fewer than it found, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. ## Open work diff --git a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/schemas/galaxy-workflow-draft.schema.json index 3a39f43..dd3a94a 100644 --- a/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/schemas/galaxy-workflow-draft.schema.json +++ b/plugins/foundry-skills/skills/apply-galaxy-workflow-changeset/references/schemas/galaxy-workflow-draft.schema.json @@ -75,7 +75,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_version": { "anyOf": [ @@ -108,7 +108,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "out": { "anyOf": [ @@ -309,7 +309,7 @@ ] }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "release": { "anyOf": [ @@ -412,22 +412,28 @@ "items": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" } ] }, @@ -437,22 +443,28 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" }, { "type": "string" @@ -927,7 +939,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "label": { "anyOf": [ @@ -1040,7 +1052,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_state": { "anyOf": [ @@ -1856,7 +1868,9 @@ "text", "File", "data", - "collection" + "collection", + "color", + "directory_uri" ], "type": "string" } @@ -1952,10 +1966,10 @@ ] }, "restrictions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "suggestions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "type": { "enum": [ @@ -1963,6 +1977,9 @@ "string" ], "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" } }, "required": [ @@ -1971,6 +1988,96 @@ "type": "object" }, "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { "additionalProperties": false, "properties": { "default": { @@ -2016,7 +2123,7 @@ "required": [], "type": "object" }, - "Shared10": { + "Shared11": { "additionalProperties": false, "properties": { "default": { @@ -2124,7 +2231,7 @@ ], "type": "object" }, - "Shared11": { + "Shared12": { "additionalProperties": false, "properties": { "default": { @@ -2230,7 +2337,7 @@ "required": [], "type": "object" }, - "Shared12": { + "Shared13": { "additionalProperties": false, "properties": { "default": { @@ -2337,7 +2444,7 @@ ], "type": "object" }, - "Shared13": { + "Shared14": { "additionalProperties": false, "properties": { "default": { @@ -2424,7 +2531,143 @@ ], "type": "object" }, - "Shared14": { + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { "anyOf": [ { "items": { @@ -2457,7 +2700,7 @@ } ] }, - "Shared15": { + "Shared18": { "anyOf": [ { "type": "null" @@ -2488,11 +2731,11 @@ } ] }, - "Shared16": { + "Shared19": { "anyOf": [ { "items": { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, "type": "array" }, @@ -2500,7 +2743,7 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, { "type": "string" @@ -2522,7 +2765,7 @@ } ] }, - "Shared17": { + "Shared20": { "anyOf": [ { "type": "null" @@ -2610,7 +2853,7 @@ "$ref": "#/$defs/Shared1" }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "uuid": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/SKILL.md b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/SKILL.md index 30ef984..4025ebe 100644 --- a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/SKILL.md +++ b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/SKILL.md @@ -1,6 +1,6 @@ --- name: author-galaxy-tool-wrapper -description: "Author a new Galaxy user-defined tool YAML definition when discovery yields nothing acceptable." +description: "Author a new Galaxy user-defined tool YAML definition when Tool Shed discovery misses." --- # author-galaxy-tool-wrapper @@ -9,11 +9,12 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## When To Use -- Author a new Galaxy user-defined tool YAML definition when discovery yields nothing acceptable. +- Author a new Galaxy user-defined tool YAML definition when Tool Shed discovery misses. ## Inputs - Read artifact `summary-nextflow`. Schema: summary-nextflow. Produced by `summarize-nextflow`. Source pipeline summary from summarize-nextflow; provides process command, inputs, outputs, and container or conda evidence for UDT authoring. +- Read artifact `galaxy-tool-pin`. Schema: galaxy-tool-discovery. Produced by `discover-shed-tool`. The discover-shed-tool result for the step. ## Outputs @@ -33,6 +34,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - `references/notes/component-nextflow-containers-and-envs.md`: Research note copied verbatim into the bundle. Map Nextflow container/conda evidence to Galaxy UDT container and package provenance. Use when: a missing Galaxy user-defined tool must be authored from a Nextflow process with container or conda directives. - `references/notes/component-nextflow-containers-and-envs.yml`: Companion file copied verbatim into the bundle. Sibling of `references/notes/component-nextflow-containers-and-envs.md`; read it where that note directs. - `references/notes/galaxy-user-tool-critique.md`: Research note copied verbatim into the bundle. Clarity and idiomaticity criteria, and the text-versus-structural test for each proposed fix. Use when: after the drafted `GalaxyUserTool` passes structural validation and before emitting `galaxy-user-tool.yml`. +- `references/notes/galaxy-user-tool-workflow-binding.md`: Research note copied verbatim into the bundle. Register an authored tool through `/api/unprivileged_tools`, not the admin-only `/api/dynamic_tools`, when it must run on a server outside a workflow step. Use when: the caller asks to register, upload, or run the authored tool on a Galaxy server instead of embedding it in a workflow step. ## Validation @@ -40,7 +42,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Procedure -Author a Galaxy user-defined tool (UDT) YAML definition when discover-shed-tool cannot produce an acceptable existing Tool Shed wrapper. This skill owns the author-on-fallthrough path only; do not duplicate existing wrappers unless discovery evidence is weak or missing. +Author a Galaxy user-defined tool (UDT) YAML definition when discover-shed-tool finds no wrapper that runs the step's work. This skill owns the author-on-miss path only. The output is a single `GalaxyUserTool` YAML document, not Galaxy XML. Preserve source evidence and unresolved assumptions in the emitted artifact or companion notes rather than inventing command flags, containers, datatypes, or package names. @@ -51,13 +53,12 @@ Use the source summary and branch fallthrough context to identify: - The process or abstract step that needs a tool. - Command intent, required inputs, expected outputs, and test fixture evidence. - Container, Bioconda, Conda, or environment evidence from the source pipeline. -- Why Tool Shed discovery did not supply an acceptable existing wrapper. ### Procedure #### 1. Confirm authoring is justified -Read the discovery result before authoring. Continue only when discovery returned `miss`, or when the selected hit was weak enough that the harness explicitly fell through. +Read the discover-shed-tool result before writing any part of the UDT. Author only when its `status` is `miss`, or when it is `weak` and the caller's confirmation showed the candidate doesn't run the step's work. Otherwise a wrapper exists: stop without emitting `galaxy-user-tool.yml` and return the candidate to the caller. Authoring a UDT for a step an existing wrapper can run is a failure. If the missing tool need is under-specified, ask for clarification or emit an unresolved assumption. Do not create a plausible-looking UDT from absent command or container evidence. @@ -97,7 +98,9 @@ Apply every concrete clarity or idiomaticity issue. Sort each fix into text-leve Write `galaxy-user-tool.yml` with the validated and critic-reviewed `GalaxyUserTool` definition. -The artifact should be ready for the downstream harness or Galaxy runtime to load as a user-defined tool. Include unresolved assumptions only when they are truthful and actionable; otherwise fail instead of emitting a misleading tool. +The artifact should be ready for the downstream harness or Galaxy runtime to load as a user-defined tool. Include unresolved assumptions only when they are truthful and actionable. Otherwise fail instead of emitting a misleading tool. + +The file is handed on as written. implement-galaxy-tool-step embeds it under the workflow step's `run:`, so it does not need registering with a Galaxy server first, and its `id` and `version` are what identify the tool in the workflow. To register or run it on a server by itself, follow galaxy-user-tool-workflow-binding. ### Non-goals diff --git a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_provenance.json b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_provenance.json index 0cd7403..b2702ca 100644 --- a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_provenance.json +++ b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "author-galaxy-tool-wrapper", "path": "content/molds/author-galaxy-tool-wrapper/index.md", - "revision": 5, - "content_hash": "cced8f068cd1ce81fefad23a62fdb4d52b3ce805b2231f64a7fe37f258bc1e09", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "revision": 8, + "content_hash": "77446daafedbcc6754902533a0280d2f0418d1bb8f92e042562404dbc75c9023", + "commit": "53d1b93e584ed09195044c9e688ed835997b2eeb" }, - "cast_at": "2026-09-15T23:16:18.298Z", + "cast_at": "2026-11-11T16:56:50.837Z", "cast_date": "2026-05-08", "cast_revision": 1, "cast_history": [ @@ -61,8 +61,8 @@ "load": "upfront", "evidence": "corpus-observed", "purpose": "Field contract, expression syntax, script placement, and package inference for a `GalaxyUserTool` definition.", - "src_hash": "365fad455369945a34bda537c8dbf409a0966f0f88da9fe78b61cff32d5b0d0d", - "dst_hash": "365fad455369945a34bda537c8dbf409a0966f0f88da9fe78b61cff32d5b0d0d", + "src_hash": "94fbbfe2fe4a12379b72609a6360ca4c3f360246ba74e4fe3442bb8705fb9e6e", + "dst_hash": "94fbbfe2fe4a12379b72609a6360ca4c3f360246ba74e4fe3442bb8705fb9e6e", "source": "deterministic" }, { @@ -80,6 +80,21 @@ "dst_hash": "94b047cc268b0f788e8d596b9e809b4764c68b94e8bd4f819c2f9af59d559a11", "source": "deterministic" }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-user-tool-workflow-binding]]", + "src": "content/research/galaxy-user-tool-workflow-binding/index.md", + "dst": "references/notes/galaxy-user-tool-workflow-binding.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Register an authored tool through `/api/unprivileged_tools`, not the admin-only `/api/dynamic_tools`, when it must run on a server outside a workflow step.", + "trigger": "When the caller asks to register, upload, or run the authored tool on a Galaxy server instead of embedding it in a workflow step.", + "src_hash": "cfdf62f982349c02cdfa5f3214f3feb9f54cf664b880e438294e5748f9002902", + "dst_hash": "cfdf62f982349c02cdfa5f3214f3feb9f54cf664b880e438294e5748f9002902", + "source": "deterministic" + }, { "kind": "schema", "mode": "verbatim", @@ -91,8 +106,8 @@ "evidence": "corpus-observed", "purpose": "Read process tool, container, conda, inputs, outputs, script summary, and test fixture evidence from the source pipeline summary.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], @@ -113,6 +128,14 @@ "producers": [ "summarize-nextflow" ] + }, + { + "id": "galaxy-tool-pin", + "description": "The [[discover-shed-tool]] result for the step.", + "inherited_schema": "[[galaxy-tool-discovery]]", + "producers": [ + "discover-shed-tool" + ] } ] } diff --git a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_verify.json b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_verify.json index 667ae3b..c218f55 100644 --- a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_verify.json +++ b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/_verify.json @@ -1,6 +1,18 @@ { "verify_schema_version": 1, "entries": [ + { + "artifact_id": "galaxy-tool-pin", + "direction": "input", + "kind": "json", + "default_filename": "galaxy-tool-pin.json", + "schema": "[[galaxy-tool-discovery]]", + "validator_bin": "foundry", + "args": [ + "validate-galaxy-tool-discovery", + "{artifact_path}" + ] + }, { "artifact_id": "summary-nextflow", "direction": "input", @@ -10,6 +22,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/notes/galaxy-user-tool-authoring.md b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/notes/galaxy-user-tool-authoring.md index 011b19a..77ba805 100644 --- a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/notes/galaxy-user-tool-authoring.md +++ b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/notes/galaxy-user-tool-authoring.md @@ -5,21 +5,26 @@ tags: - target/galaxy status: draft created: 2026-08-24 -revised: 2026-08-24 -revision: 1 +revised: 2026-11-10 +revision: 2 component: tool_util_models related_notes: - "[[component-nextflow-containers-and-envs]]" - "[[galaxy-user-tool-critique]]" + - "[[galaxy-user-tool-authoring-help]]" sources: - "https://github.com/galaxyproject/galaxy/blob/4d235b615e60bdb8c7e7d9ada100245068c8e4d9/lib/galaxy/agents/prompts/custom_tool_structured.md" - "https://github.com/galaxyproject/galaxy/blob/4d235b615e60bdb8c7e7d9ada100245068c8e4d9/lib/galaxy/agents/prompts/custom_tool_container_critic.md" -summary: "What validates in a GalaxyUserTool definition — fields, expression syntax, script placement, package inference — derived from Galaxy's own generator prompts." + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/client/src/components/Tool/authoringHelp.yml" + - "https://github.com/galaxyproject/galaxy/blob/7c18ab768ac112bddf485cfcb5c074a83165c591/lib/galaxy/jobs/command_factory.py" + - "https://github.com/galaxyproject/galaxy/blob/7c18ab768ac112bddf485cfcb5c074a83165c591/lib/galaxy/tool_util/deps/requirements.py" + - "https://github.com/common-workflow-language/cwl-utils/blob/v0.41/cwl_utils/expression.py" +summary: "What validates in a GalaxyUserTool definition — fields, expression syntax, script placement, package inference — from Galaxy's prompts, authoring help and code." --- The rules a `GalaxyUserTool` YAML document has to satisfy, and the decisions worth making while writing one. -Derived from two vendored Galaxy prompts — [[custom-tool-structured]] and [[custom-tool-container-critic]] — rather than restating them. Those prompts are written for Galaxy's in-process agent, where a grammar constrains generation and a deployment-side step may rewrite the container. Neither holds in a cast, so the harness divergences are named in the last section instead of being inherited silently. +Derived from two vendored Galaxy prompts — [[custom-tool-structured]] and [[custom-tool-container-critic]] — rather than restating them. Those prompts are written for Galaxy's in-process agent, where a grammar constrains generation and a deployment-side step may rewrite the container. Neither holds in a cast, so the harness divergences are named in the last section instead of being inherited silently. The escaping and shell rules in §3 come from Galaxy's own authoring help ([[galaxy-user-tool-authoring-help]]), its job script code and the cwl-utils expression evaluator it uses, because the prompts do not cover them. Sections 6 and 7 are target-neutral: they are about running a script and naming its dependencies, not about Galaxy. They are kept self-contained so a CWL-side note can lift them without a rewrite. @@ -49,7 +54,7 @@ help: ## 3. Expression syntax in `shell_command` -Expressions are JavaScript, spliced into the command verbatim. +Expressions are JavaScript, and their results are spliced into the command verbatim, without shell quoting. - Single file path: `$(inputs.param_name.path)` - Scalar value (text, integer, float, boolean): `$(inputs.param_name)` @@ -60,7 +65,20 @@ Expressions are JavaScript, spliced into the command verbatim. A bare `.join(" ")` leaves paths unquoted, and the result is spliced verbatim, so unquoted paths with spaces break the command. - **`inputs.param_name[].path` is not valid.** The empty `[]` is a JavaScript syntax error that surfaces only when the job is built. Indexing itself is fine — `inputs.some_repeat[0].x` works. - **`$(outputs.param_name.path)` is not valid at all.** Outputs are captured by `from_work_dir` or `discover_datasets`, never referenced in the command. -- Escape shell variables that are not Galaxy expressions: `\$(date)`. + +**Only `$(` and `${` are expression delimiters.** The rules below follow the `expressions-shell` section of [[galaxy-user-tool-authoring-help]] and the `interpolate` function of cwl-utils, which Galaxy uses to evaluate expressions. They apply equally to every `configfiles` entry's `content`. + +| Written | Reaches the shell as | +| --- | --- | +| `$(date +%s)`, `${f%.txt}`, `${VAR:-default}` | Nothing. Galaxy evaluates it as JavaScript, and the job fails with an expression error. | +| `$(( 1 + 2 ))` | `3`. Valid JavaScript is substituted without any error, changing the command's meaning. | +| `\$(date +%s)`, `\${f%.txt}` | `$(date +%s)`, `${f%.txt}`. Escape a shell command substitution or braced variable this way. | +| `$GALAXY_SLOTS`, `$name`, `$1` | Unchanged. A `$` not followed by `(` or `{` is not an expression, so it needs no escape. | +| `\$name` | `\$name`, backslash included. The shell does not expand it to the variable's value. | + +So escape the dollar only in front of `(` or `{`, and prefer unbraced variables (`"$f"` rather than `"${f}"`) where the shell allows it. Escape processing only runs on a command or file that contains `$(` or `${` somewhere, even an escaped `\$(`. There, `\\` also collapses to a single `\`, so a literal backslash pair is written `\\\\`. + +**Expect the command to run under `/bin/sh` with `set -e`.** Galaxy writes the evaluated `shell_command` into a job script with `set -e`, which it enables for every user-defined tool. When Galaxy runs the container itself (Docker or Singularity on a local or cluster runner), the script's interpreter is the container's `/bin/sh`. Runners that launch the container themselves, such as Kubernetes, use the destination's shell instead, `/bin/bash` by default. Write the script for `/bin/sh` unless it calls `bash` explicitly. In many images `/bin/sh` is a minimal shell such as dash, which may lack bash features such as `set -o pipefail`, arrays and `[[ ]]`. Where `set -o pipefail` is rejected, the job fails at once (exit status 2 under dash). When a script needs bash, put it in a `configfiles` entry and run it with `bash script.sh` (§6), after checking that the image provides `bash`. **Name matching is the highest-frequency failure.** Every `inputs.NAME` in `shell_command` must exactly match a declared input's `name`. Never reference an input that is not declared. diff --git a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/notes/galaxy-user-tool-workflow-binding.md b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/notes/galaxy-user-tool-workflow-binding.md new file mode 100644 index 0000000..4a5bc55 --- /dev/null +++ b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/notes/galaxy-user-tool-workflow-binding.md @@ -0,0 +1,91 @@ +--- +type: research +title: "Binding a workflow step to a user-defined tool" +tags: + - target/galaxy +status: draft +created: 2026-09-24 +revised: 2026-10-23 +revision: 4 +related_notes: + - "[[galaxy-user-tool-authoring-help]]" + - "[[galaxy-user-tool-authoring]]" + - "[[galaxy-workflow-draft-format]]" + - "[[gxformat2-schema]]" +related_molds: + - "[[author-galaxy-tool-wrapper]]" + - "[[implement-galaxy-tool-step]]" + - "[[advance-galaxy-draft-step]]" +sources: + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/client/src/components/Tool/authoringHelp.yml" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy/managers/tools.py" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy_test/workflow/inline_user_defined_tool.gxwf.yml" +summary: "How a workflow step uses an authored GalaxyUserTool: embedded under run:, never by tool_uuid, tool_id or content_id; registering via /api/unprivileged_tools." +--- + +A gxformat2 step that uses an authored `GalaxyUserTool` carries the whole tool definition under its `run:` key. That is the only way the Foundry binds a step to a user-defined tool. + +A step never names a user-defined tool by `tool_uuid`, `tool_id`, or `content_id` alone. The tool is not in a server's toolbox, so a `tool_id` does not resolve to it, even when the string matches the tool's `id`. A `uuid` identifies one registered copy of the tool, for its owner, on one server. Galaxy's workflow editor and its exports write any such references themselves, and Galaxy rejects a `tool_uuid` step without an embedded definition whose `tool_id` names a different tool. + +The rule is derived from the `workflows` section ("Using a tool in a workflow") of Galaxy's authoring help, vendored in [[galaxy-user-tool-authoring-help]]. The Galaxy-side facts below are pinned to the same commit. The gxwf behavior was checked with `@galaxy-tool-util/cli` 1.14.0, the version the Foundry pins. + +## Embedding the definition in a gxformat2 step + +The embedded form makes the workflow self-contained. When a user imports it, Galaxy registers a private copy of the tool for that user. That requires a server with `enable_beta_tool_formats` set and a user holding the `Custom Tool Execution` role, the same conditions as registering the tool directly. Converting the workflow to native `.ga` with gxformat2 carries the definition as `tool_representation`, with no `tool_id`, `tool_version`, or `tool_uuid`, and Galaxy imports that form the same way. Galaxy's own `.ga` export also includes the definition as `tool_representation`. Nothing has to be registered first, and nothing has to be injected into the converted file afterwards. + +To bind a step: + +- Put the complete `GalaxyUserTool` document under `run:`, unchanged. Its `id` and `version` identify the tool. +- Leave out `tool_id`, `tool_version`, and `tool_shed_repository`. They describe toolbox tools. Remove any `TODO` sentinel the template left in those fields rather than filling it. +- Key `in:` by the names of the definition's `data` and `collection` inputs. +- List every output a downstream step or a workflow output uses under `out:`, by the definition's output `name`. +- Set values for non-data inputs in `state`, keyed by input `name`. + +```yaml +steps: + head: + run: + class: GalaxyUserTool + id: head_lines + version: 0.1.0 + name: Head lines + container: quay.io/biocontainers/python:3.13 + shell_command: head -n '$(inputs.n_lines)' '$(inputs.data_input.path)' > out.txt + inputs: + - name: n_lines + type: integer + value: 10 + - name: data_input + type: data + format: [txt] + outputs: + - name: out + type: data + format: txt + from_work_dir: out.txt + in: + data_input: text_file + out: + - id: out + state: + n_lines: 5 +``` + +gxformat2's converter turns that step into a native tool step whose `tool_state` contains `n_lines: 5`, with the definition in `tool_representation`. Galaxy's own test workflow `inline_user_defined_tool.gxwf.yml` exercises the same form end to end, without `out:` or `state`. + +## Validating a draft with an embedded tool + +gxwf treats a `GalaxyUserTool` under `run:` as a tool step. [[draft-validate]] with and without `--concrete`, [[draft-next-step]], [[draft-extract]], and `gxwf validate` on the extracted workflow all run as they do for toolbox steps, and `draft-extract` keeps workflow outputs sourced from the embedded step even without an `out:` list. `gxwf validate-tool-source ` validates the definition itself when it is saved as a separate file. + +## Registering a tool on a server + +A step that embeds its tool does not need it registered. Registering is for running the tool by itself, outside a workflow, or for binding a native `.ga` step by `tool_uuid`. + +- Register with `POST /api/unprivileged_tools`, body `{"representation": }`. It needs the same `enable_beta_tool_formats` setting and `Custom Tool Execution` role as importing an embedded step, and returns the registered copy's `uuid`. The `api` section of [[galaxy-user-tool-authoring-help]] lists the other endpoints. +- Do not use `POST /api/dynamic_tools`. It is admin-only and returns 403 for an ordinary user. +- A returned `uuid` means Galaxy accepted the definition, not that the tool runs. Run it on test data before reporting it as working. + +## Open questions + +- Whether a Planemo-managed Galaxy runs a workflow with an embedded tool has not been tested. It would need `enable_beta_tool_formats` on that server and the `Custom Tool Execution` role for the test user. diff --git a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/author-galaxy-tool-wrapper/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/SKILL.md b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/SKILL.md index 8163ddd..43595f7 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/SKILL.md +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/SKILL.md @@ -40,11 +40,12 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. +- Before starting, check that the installed validator carries the galaxy-workflow-test-plan schema this bundle was cast with: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. ## Procedure -Produce a Galaxy workflow test plan for the update pipeline: carry the existing workflow's tests forward as a **regression baseline** and augment them for the change-set's behavioral deltas. The output is a reviewable YAML handoff conforming to galaxy-workflow-test-plan, not a concrete `tests-format` file — implement-galaxy-workflow-test authors the final `*-tests.yml` from it. This skill is the update pipeline's analogue of nextflow-test-to-galaxy-test-plan / freeform-summary-to-galaxy-test-plan: the dedicated test-plan producer every Galaxy-targeting pipeline places before the implement step. +Produce a Galaxy workflow test plan for the update pipeline: carry the existing workflow's tests forward as a **regression baseline** and augment them for the change-set's behavioral deltas. The output is a reviewable YAML handoff conforming to galaxy-workflow-test-plan, not a concrete `tests-format` file — implement-galaxy-workflow-test authors the final `*-tests.yml` from it. This skill is the update pipeline's analogue of nextflow-test-to-galaxy-test-plan / freeform-summary-to-galaxy-test-plan / workflow-brief-to-galaxy-test-plan: the dedicated test-plan producer every Galaxy-targeting pipeline places before the implement step. ### Translate-and-augment, not synthesize-from-scratch diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_provenance.json b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_provenance.json index 123ee71..12df2dd 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_provenance.json +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "changeset-to-galaxy-test-plan", "path": "content/molds/changeset-to-galaxy-test-plan/index.md", - "revision": 2, - "content_hash": "1a5f10ecf90158c4fdaf292055ca73dcd6dd39e7253fe417b62020c3c4ec8be9", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 4, + "content_hash": "1b8362487f534389c4e581cc2cb54e21fc004bb1a06d818b4dff1ba34bdc2423", + "commit": "a59d6025d612ce7013b09fa1ccb58f60507ccc6a" }, - "cast_at": "2026-09-13T20:58:26.449Z", + "cast_at": "2026-10-26T16:18:38.156Z", "refs": [ { "kind": "research", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Decide which change-set-exposed outputs and promoted checkpoints make meaningful new assertions, and how much a baseline assertion must loosen when an edit intentionally changes an output.", "trigger": "When adding assertions for a newly exposed output or a new step, or reconciling a baseline assertion an edit invalidates.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -36,8 +36,8 @@ "evidence": "corpus-observed", "purpose": "Distinguish accepted IWC-style test shortcuts from assertion smells when loosening a baseline assertion or synthesizing a new one.", "trigger": "When considering existence-only, size-only, image-dimension, or tolerant output checks, or recording an omission.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -51,8 +51,8 @@ "evidence": "corpus-observed", "purpose": "Record job-input fixtures for change-set-added inputs (remote-URL-first locations, hashes, collection shapes) as fixture provenance, reusing the baseline's existing fixtures unchanged.", "trigger": "When a change-set adds a workflow input that needs new test data, or when recording provenance for a new fixture.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -66,8 +66,8 @@ "evidence": "corpus-observed", "purpose": "Pick the assertion family and tolerance for each change-set-driven expected output by output type.", "trigger": "When turning a change-set behavioral delta into assertion intent and a tolerance.", - "src_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", - "dst_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", "source": "deterministic" }, { @@ -81,8 +81,8 @@ "evidence": "corpus-observed", "purpose": "Keep the plan addressable by stable labels and artifacts Planemo can connect back to invocations, jobs, and outputs.", "trigger": "When recording the labels and checkpoints the downstream test must address.", - "src_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", - "dst_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", + "src_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "dst_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", "source": "deterministic" }, { @@ -97,8 +97,8 @@ "purpose": "Output contract: the emitted plan conforms to [[galaxy-workflow-test-plan]]. Cast bundles the JSON Schema so the skill carries its output shape; validate with `foundry validate-galaxy-workflow-test-plan`.", "verification": "Cast the skill, run on a summary with existing tests plus a change-set, confirm the emitted YAML validates and downstream [[implement-galaxy-workflow-test]] consumes it without re-reading the change-set.", "license": "MIT", - "src_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", - "dst_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", + "src_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "dst_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "source": "deterministic" }, { @@ -129,8 +129,8 @@ "trigger": "When naming an assertion family or compare operator for a carried-forward or change-set-driven expected output.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", - "dst_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", + "src_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "dst_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_verify.json b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_verify.json index 4dba49e..221b389 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_verify.json +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-galaxy-workflow", + "--schema-sha256", + "55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3", "{artifact_path}" ] }, @@ -22,6 +24,8 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-workflow-test-plan", + "--schema-sha256", + "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md index 6d0fef7..bd1160e 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-11 -revision: 6 +revised: 2026-10-21 +revision: 10 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,231 +16,109 @@ related_notes: - "[[validate-tests]]" - "[[iwc-tabular-operations-survey]]" - "[[galaxy-discover-datasets]]" -summary: "Decision and idiom guide for picking planemo workflow-test assertions: which family per output type, how to size tolerances, when to validate." +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." --- -# Planemo asserts: idiom and decision guide +# Choosing Planemo workflow-test assertions -Companion to [[iwc-test-data-conventions]] (input shapes), [[galaxy-workflow-testability-design]] (workflow structure before test YAML exists), and [[iwc-shortcuts-anti-patterns]] (what's accepted vs smell). This note is forward-looking: when authoring a new `-tests.yml`, which assertion family fits which output, and what the recommended tolerances and operators are. +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. -The **vocabulary itself is not restated here** — every assertion's parameter list, types, defaults, required fields, and Python docstring is rendered from the test-format JSON Schema at [[tests-format]]. Assertion names below deep-link into that page (e.g. [[tests-format#has_text_model|has_text]] jumps straight to that `$def`). +## 1. Choose by the property, then the output type -## 1. Choose by output type +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | -The single most useful decision table. Pick the row that matches the file format the workflow emits; default to the recommended assertion family. +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. -| Output type | Default assertion family | Why | Fallback | -|---|---|---|---| -| **Plain text reports / logs** (FastQC summary, MultiQC text section) | [[tests-format#has_text_model|has_text]] (substring on a known stable token) + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Substring is robust across versions; line count catches truncation | [[tests-format#has_text_matching_model|has_text_matching]] (regex) when token text changes per run | -| **HTML reports** (MultiQC HTML, custom dashboards) | [[tests-format#has_text_model|has_text]] against stable section names | HTML embeds timestamps and asset hashes; byte-diff is hopeless | [[tests-format#has_size_model|has_size]] + large `delta` as last resort | -| **Tabular** (TSV, CSV, BED-like) | [[tests-format#has_n_columns_model|has_n_columns]] + [[tests-format#has_text_model|has_text]] for headers + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Schema-shape + content probes; tolerant to row reordering | [[tests-format#has_line_matching_model|has_line_matching]] for a known canonical row | -| **VCF** | `compare: diff` with `lines_diff: 6` | The `lines_diff: 6` constant matches the typical VCF header preamble that embeds `##fileDate=` and `##source=` | [[tests-format#has_text_matching_model|has_text_matching]] for known variants | -| **BAM** | [[tests-format#has_size_model|has_size]] + [[tests-format#has_archive_member_model|has_archive_member]] (BAM is a gzipped block format) | Headers contain `@PG` lines with command lines and Galaxy job IDs; never byte-diff | Convert to SAM in a downstream step and assert on text | -| **FASTA** (deterministic — assemblies, consensus) | `file:` exact comparison or [[tests-format#has_text_model|has_text]] for known sequence | Output is byte-stable when the upstream tool is deterministic | [[tests-format#has_n_lines_model|has_n_lines]] if line wrapping varies | -| **FASTA** (non-deterministic — RepeatModeler libraries) | `compare: sim_size` with large `delta:` | Family content varies run-to-run | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | -| **FASTQ** (rare as workflow output) | [[tests-format#has_n_lines_model|has_n_lines]] (must be multiple of 4) | Quality scores are read-id-dependent | `compare: sim_size` | -| **JSON** (deterministic — config dumps, params) | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] / [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] | Surgical: assert on the property you care about, ignore the rest | `compare: diff` if the JSON is fully canonical | -| **JSON** (stochastic — HyPhy stats, MCMC results) | `has_text: text: "{"` (existence-only) | Embedded floats break any structural assertion; see [[iwc-shortcuts-anti-patterns]] §1 | [[tests-format#has_h5_keys_model|has_h5_keys]]-equivalent for top-level structural keys | -| **HDF5 / AnnData** | [[tests-format#has_h5_keys_model|has_h5_keys]] + [[tests-format#has_h5_attribute_model|has_h5_attribute]] for known structure | Asserts shape, not values — values are float-noisy | [[tests-format#has_size_model|has_size]] with `delta:` | -| **XML** | [[tests-format#is_valid_xml_model|is_valid_xml]] + [[tests-format#has_element_with_path_model|has_element_with_path]] + `element_text_is`/`element_text_matches` | Surgical structural probes; byte-diff fails on whitespace/attribute-order | [[tests-format#has_n_elements_with_path_model|has_n_elements_with_path]] for cardinality | -| **PNG / image plots** | [[tests-format#has_image_width_model|has_image_width]] + [[tests-format#has_image_height_model|has_image_height]] + [[tests-format#has_size_model|has_size]] with `delta:` (5–10%) | Catches "rendered something with the right dimensions"; corpus norm | [[tests-format#has_image_center_of_mass_model|has_image_center_of_mass]] if mask/segmentation outputs are deterministic | -| **TIFF / multipage images** | [[tests-format#has_image_frames_model|has_image_frames]] + [[tests-format#has_image_channels_model|has_image_channels]] + [[tests-format#has_size_model|has_size]] with `delta:` | Same logic, channel/frame structure matters | — | -| **Archives** (zip, tar.gz) | `has_archive_member: path: "regex"` with nested `asserts:` | Asserts on a specific member; archive timestamps never byte-stable | [[tests-format#has_size_model|has_size]] with `delta:` | -| **GFF / GTF** | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` + [[tests-format#has_text_model|has_text]] for known feature | Tools reorder freely; hard to byte-diff | `has_n_columns: 9` (GFF spec) | -| **Cool / HiC matrices** | `compare: sim_size` with multi-MB `delta:` | Binary, run-to-run variance | [[tests-format#has_archive_member_model|has_archive_member]] for HDF5 component | +## 2. Compare operators and expected artifacts -When in doubt: start with [[tests-format#has_size_model|has_size]] + `delta_frac: 0.1`. It catches the catastrophic failure mode (empty / 10x bigger output). Then add a content probe. +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. -### 1a. If the assertion is too weak, revisit the workflow output +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. -Assertion choice sometimes reveals a workflow-design problem. If the only available output can support only a size check or image-dimension smoke test, check whether the workflow should expose a stronger checkpoint before settling for the weak assertion. +## 3. Set tolerances from the failure boundary -- Scanpy's plot outputs use size and image-dimension assertions, but the same workflow also exposes AnnData HDF5 checkpoints and a cluster-count table (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end uses coarse size bands for coverage/mapped-read outputs, but expression/count tables get stronger regex and exact-line checks (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. -Rule: if a translated workflow exposes only weakly assertable final reports, consult [[galaxy-workflow-testability-design]] and consider promoting a table/text/HDF5 checkpoint before writing the final assertions. +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. -## 2. The `compare:` operators +## 4. Text assertions and repeated checks -Top-level on a `file:` output assertion. Vocabulary, in decreasing strictness: - -- **`diff`** (default). Byte-for-byte equality with optional `lines_diff:` tolerance for a fixed number of header lines. Use only when the upstream tool is deterministic on fixed inputs *and* the output has no embedded timestamps, command lines, version banners, or hash-ordered Python-dict-style keys. -- **`re_match` / `re_match_multiline`**. Each line of the expected fixture is a regex that must match the corresponding output line. Useful when a few fields per row are timestamped but the rest is canonical. Rare in the corpus. -- **`contains`**. The expected fixture is a substring of the output. Cheap; weak. Prefer `asserts: has_text` for new code unless you genuinely have a multi-line block to assert as a whole. -- **`sim_size`**. Output file size matches the fixture's size within `delta:` (bytes) or `delta_frac:` (fraction). Use when the output is necessarily non-deterministic but its rough size is reproducible (RepeatModeler libraries, HiC matrices, Bayesian sampler outputs). - -Picking `lines_diff:`: count the mutable header lines in the output format. VCF: ~6 (`##fileformat`, `##fileDate`, `##source`, `##reference`, contig/info lines vary). SAM/text headers: count `@HD`/`@PG`/`@CO` lines. Set `lines_diff:` to the count exactly — looser values mask real diffs. - -## 3. Tolerance picking - -**`delta:`** is bytes (for [[tests-format#has_size_model|has_size]] and `compare: sim_size`) or absolute count (for [[tests-format#has_n_lines_model|has_n_lines]], [[tests-format#has_n_columns_model|has_n_columns]], [[tests-format#has_image_width_model|has_image_width]], etc.). Suffix multipliers documented in the schema — `1K`, `1M`, `1G` work. - -**`delta_frac:`** is a fraction (0.1 = 10%). Use when expected size scales with input volume. Three IWC tests use it (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`); the rest use absolute `delta:`. - -**Picking magnitudes** (from corpus survey in [[iwc-shortcuts-anti-patterns]] §2): - -- Image dimensions: `delta: 25–30` pixels (5% of typical matplotlib defaults). -- Image file size: `delta: 5K–60K` (5–10% of file size). -- Small text reports: `delta: 1K–10K`. -- HTML reports: `delta: 25K–100K`. -- BAM files: `delta: 1M–10M`. -- RepeatModeler / Bayesian sampler outputs: `delta: 30K–90M` (extreme, but justified by the underlying nondeterminism). - -**Heuristic for new outputs:** `delta_frac: 0.1` is a defensible default. Tighten if the output proves more deterministic than expected. - -## 4. Text family — [[tests-format#has_text_model|has_text]] vs [[tests-format#has_text_matching_model|has_text_matching]] vs [[tests-format#has_line_model|has_line]] vs [[tests-format#has_line_matching_model|has_line_matching]] vs [[tests-format#has_n_lines_model|has_n_lines]] - -All five are common; choose by what you're verifying. - -- **[[tests-format#has_text_model|has_text]]** — output contains the substring `text:`. Anywhere in the output, any number of times. Add `n:` / `min:` / `max:` to constrain occurrence count. Add `delta:` to allow slack on the count. -- **[[tests-format#has_text_matching_model|has_text_matching]]** — output matches the regex `expression:`. Use sparingly; prefer literal [[tests-format#has_text_model|has_text]] when you can. -- **[[tests-format#has_line_model|has_line]]** — output has at least one *line* matching `line:` exactly. Use when line boundaries matter (e.g. asserting on a specific row in a table). -- **[[tests-format#has_line_matching_model|has_line_matching]]** — same but with regex. -- **[[tests-format#has_n_lines_model|has_n_lines]]** — assert the line count is `n:` ± `delta:`. - -A common combination in IWC: `has_n_lines: n: 100, delta: 5` + `has_text: text: "expected_token"` — line-count sanity-check plus a content marker. This catches both truncation and content drift in one assertion pair. - -`negate: true` is supported on every assertion. Used for the "this output should NOT contain X" case. - -## 5. Collection output assertions (`element_tests:`) - -For Galaxy collection outputs, the test format keys element assertions by element identifier: +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. ```yaml -my_collection_output: - element_tests: - sample_1: +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: asserts: - has_text: - text: "expected" - sample_2: - file: test-data/expected_sample_2.txt -``` - -Optional `attributes:` at the collection level can assert on the produced collection shape: - -```yaml -my_collection_output: - attributes: {collection_type: list:list} - element_tests: - ... + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 ``` -Nested collections: outer `element_tests:` keyed by outer identifier; inner uses `elements:` (note plural, no `_tests` suffix on the inner). See [[iwc-test-data-conventions]] §2f for the live example. - -For a list-of-files where every element should pass the same minimal check, the existence-probe pattern (`has_text: "{"` for JSON; `has_size: min: 100` for any non-empty binary) is widely used and accepted in IWC. - -## 6. The validate-against-workflow inner loop - -A `-tests.yml` file can be **structurally invalid** in two distinct ways: - -1. **Schema-invalid** — wrong field names, wrong nesting, wrong types. Caught by the test-format JSON Schema. -2. **Workflow-incoherent** — schema-valid YAML, but the input/output labels don't match the actual workflow. Renaming an output in the `.ga` and forgetting to update its sibling `-tests.yml` produces this case. Planemo will surface it as an "output not found" error at test-runtime, but only after a full workflow run. - -The `@galaxy-tool-util/schema` npm package ships **two** validators that catch both cases statically — no Galaxy or Planemo invocation needed: +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. -- **`validateTestsFile(yaml)`** — runs the file against `tests.schema.json` (AJV). Reports schema violations with paths. -- **`checkTestsAgainstWorkflow(workflow, tests)`** — cross-checks a `.ga` / format2 workflow against a tests file: missing input labels, missing output labels, type incompatibilities (e.g. test supplies a `File` for a parameter typed `int`). +## 5. Collection and structured-output examples -Both are pure-JS, take milliseconds, and have no Galaxy dependency. Wire them into the inner authoring loop: +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" ``` -edit -tests.yml - → validateTestsFile() # schema gate - → checkTestsAgainstWorkflow(.ga, tests) # coherence gate - → planemo workflow_test_on_invocation # assertion gate (no full re-run) - → planemo test # full integration (slow) -``` - -The first two gates short-circuit cheap mistakes before a slow planemo run. They are the static-validation equivalent of `gxwf` for tests, and the `implement-galaxy-workflow-test` mold should reference them as its primary inner-loop tooling. Source: galaxy-tool-util-ts package, `src/test-format/index.ts` exports. - -## 7. Authoring loop — generation, then refinement - -Reviewer convention is to **generate** the initial `-tests.yml` rather than hand-write it. Two planemo subcommands cover this: - -- **`planemo workflow_test_init --from_invocation `** ([[planemo-workflow_test_init]]) — given a successful Galaxy invocation ID, emit a `-tests.yml` with a `job:` block that captures all inputs (with SHA-1 hashes) and an `outputs:` block with `file:` references to the actual outputs (downloaded into `test-data/`). Hand-tighten the assertions afterward. -- **`planemo workflow_test_on_invocation `** ([[planemo-workflow_test_on_invocation]]) — re-evaluate an edited `-tests.yml` against a saved invocation without re-running the workflow. The fast inner loop for assertion iteration; complements the static gates in §6. - -Together these cut the assertion-iteration cost dramatically. An agent should: - -1. Run the workflow once on usegalaxy.* (or local) to get a known-good invocation. -2. `--from_invocation` to bootstrap the test file. -3. Replace the autogenerated `file:` exact-comparison assertions with assertion-family-appropriate alternatives per §1. -4. [[planemo-workflow_test_on_invocation]] after each edit; full [[planemo-test]] at the end. - -## 8. What the schema gives you for free - -When the test-format schema lands as a Foundry-rendered note, the agent can consult any assertion's `$def` directly for: parameter types, defaults, required fields, the `that` discriminator constant, and the original Python docstring (carried through as `description`). This note does **not** restate that vocabulary — it complements it with the corpus-grounded *which-and-when*. - -What's still missing from the schema and worth keeping in research notes: - -- This decision table (§1) — output-type → assertion family. -- Tolerance magnitudes (§3) — corpus-derived defaults. -- The `validateTestsFile` / `checkTestsAgainstWorkflow` integration story (§6). -- Anti-pattern flags — see [[iwc-shortcuts-anti-patterns]]. -## 9. Common combinations (recipes) +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. -Six recipes worth memorizing. +## 6. Validate labels and schema before execution -**Stable text report (FastQC summary, simple stats).** -```yaml -my_report: - asserts: - has_n_lines: { n: 12, delta: 2 } - has_text: { text: "Total Sequences" } -``` - -**MultiQC HTML report.** -```yaml -multiqc_report: - asserts: - has_text: { text: "Filtered Reads" } - has_text: { text: "FastQC" } -``` +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: -**VCF (pinned tool, fixed reference).** -```yaml -called_variants: - file: test-data/expected.vcf - compare: diff - lines_diff: 6 +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga ``` -**Stochastic JSON (HyPhy-style).** -```yaml -hyphy_meme: - element_tests: - geneA: { asserts: { has_text: { text: "{" } } } - geneB: { asserts: { has_text: { text: "{" } } } -``` +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. -**Matplotlib plot.** -```yaml -umap_plot: - asserts: - has_size: { size: 68416, delta: 6000 } - has_image_width: { width: 601, delta: 30 } - has_image_height: { height: 429, delta: 25 } -``` +## 7. Create and refine the test -**AnnData (HDF5).** -```yaml -clustered_anndata: - asserts: - has_h5_keys: { keys: "obs/louvain" } - has_h5_keys: { keys: "var/highly_variable" } - has_h5_keys: { keys: "uns/rank_genes_groups" } - has_size: { size: 12000000, delta: 1500000 } -``` +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. -## 10. Cross-references +## 8. Source of the exact syntax -- [[galaxy-workflow-testability-design]] — decide which workflow outputs and checkpoints to expose before choosing assertions. -- [[iwc-test-data-conventions]] — input-side conventions (job inputs, collection shapes, `hashes:`, CVMFS). -- [[iwc-shortcuts-anti-patterns]] — accepted-vs-smell catalog and corpus prevalence; this note's mirror image. -- Test-format schema (`@galaxy-tool-util/schema` npm package) — authoritative vocabulary; will be vendored into a Foundry-rendered schema note. See `content/meta/casting.md` for the casting story. -- Planemo test-format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- [[galaxy-xsd]] — Galaxy XSD assertion source of truth, vendored from upstream. -- Tightening of the schema and Pydantic source: [galaxyproject/galaxy#22566](https://github.com/galaxyproject/galaxy/pull/22566). -- TS schema sync into npm: [jmchilton/galaxy-tool-util-ts#75](https://github.com/jmchilton/galaxy-tool-util-ts/pull/75). +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md index 59b3c25..72583a5 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-11 -revision: 3 +revised: 2026-10-26 +revision: 4 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[galaxy-tool-job-failure-reference]]" @@ -23,6 +23,8 @@ sources: - "~/projects/repositories/planemo/planemo/galaxy/activity.py" - "~/projects/repositories/planemo/planemo/galaxy/invocations" - "~/projects/repositories/planemo/planemo/galaxy/config.py" + - "~/projects/repositories/planemo/planemo/galaxy/serve.py" + - "~/projects/repositories/planemo/planemo/engine/galaxy.py" summary: "Reference for Planemo workflow test/run architecture, Galaxy modes, API polling, and noisy failure boundaries." --- @@ -94,6 +96,20 @@ Important operations: For workflows, Planemo invokes Galaxy using input labels/names. Stable generated labels are therefore important for both test execution and debugging. +### Workflow Import Failures + +Planemo imports the workflow while it brings Galaxy up for the run (`config.install_workflows()`, called from `_serve()` in `planemo/galaxy/serve.py` for managed Galaxy and from `ExternalGalaxyEngine.ensure_runnables_served` in `planemo/engine/galaxy.py` for external Galaxy, as of planemo 0.75.47). For gxformat2 it converts the workflow to a native dict client-side first, so Galaxy's own gxformat2 conversion never runs. That is outside the per-test `execute()` wrapper that turns exceptions into an `ErrorRunResponse`. When Galaxy rejects the import request, BioBlend raises `ConnectionError`, the test command exits non-zero, and Planemo writes neither the `--test_output_json` report nor the HTML report. No workflow, history, or invocation id exists. + +The evidence then lives in three places: + +| Evidence | What it holds | +|---|---| +| Upload response | HTTP status and body. A 400 usually names the failing field or validation rule. A 500 body rarely explains the cause. | +| Galaxy server log | The traceback for the failing request, the only place a 500's cause appears. Managed Galaxy streams its log into the Planemo terminal output and keeps it on disk. External Galaxy keeps it on the server. | +| Planemo terminal log | The BioBlend exception and the point in the run where it was raised. | + +Re-importing the same converted dict as the same user on the same server reproduces the failure without the rest of the Planemo run, which makes it the cheap loop for isolating a trigger. + Workflow testability design guidance lives in [[galaxy-workflow-testability-design]]. This architecture note only records why Planemo makes labels and workflow-level outputs operationally important. ## Structured Artifacts @@ -121,6 +137,7 @@ Planemo transforms and summarizes Galaxy failure information. These are likely i - Polling may summarize “at least one job is in error” while detailed job evidence is printed elsewhere or stored separately. - Missing outputs become assertion/output problems, which can conflate label mismatch, workflow output omission, optional output absence, failed invocation, or output download issue. - Output download failures may be logged but not always propagated as the most specific failure. +- A rejected workflow import aborts the run before any structured report is written, so terminal and server logs are the only record. - External Galaxy without an admin key may defer missing-tool problems until runtime. - `allow_tool_state_corrections=True` can let Galaxy adjust tool state during invocation, which is useful but can mask definition drift. - `--no_wait` is not suitable for debug conclusions because invocation creation can succeed before jobs fail. @@ -135,7 +152,7 @@ For [[run-workflow-test]]: For [[debug-galaxy-workflow-output]]: -- Start from structured Planemo output. +- Start from structured Planemo output. When there is none because the workflow import failed, start from the upload response and the Galaxy server log. - For missing-output failures, check label drift and omitted workflow-level outputs before assuming tool failure. - If the failure is job-level, inspect Galaxy job APIs described in [[galaxy-tool-job-failure-reference]]. - If the failure is invocation-level, inspect Galaxy invocation APIs described in [[galaxy-workflow-invocation-failure-reference]]. diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json index 1539281..82e4529 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json @@ -358,7 +358,7 @@ "properties": { "family": { "type": "string", - "description": "tests-format assertion family or compare operator this intent maps to, by name, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys, file, compare:diff, compare:sim_size. Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." + "description": "tests-format assertion family, compare operator, or output-attribute field this intent maps to, by name. Assertion families go inside the output's `asserts:`, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys. Compare operators and output-attribute fields sit beside `asserts:`: file, compare:diff, compare:sim_size, element_count (collection outputs), count (nested sub-collections), ftype, checksum, metadata. Do not use collection_type (state it in ExpectedOutput.collection_shape) or element_tests (a non-null element_identifier implies it). Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." }, "intent": { "type": "string", @@ -390,15 +390,16 @@ "string", "null" ], - "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions." + "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions. Address an element of a nested collection by joining identifiers outermost first with `/`, e.g. WT_REP1/forward in a list:paired." }, "evidence": { "type": "string", "enum": [ "test-evidence", + "fixture-derived", "intent" ], - "description": "Whether this assertion was translated from upstream test evidence or synthesized from intent." + "description": "Basis of the expected value: test-evidence means translated from an upstream assertion, checksum, or expected file; fixture-derived means measured from upstream test fixture bytes where no upstream assertion exists (e.g. snapshot-only nf-tests); intent means synthesized from workflow intent, wrapper XML, or step state with no upstream anchor." }, "confidence": { "type": "string", diff --git a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/tests-format.schema.json b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/tests-format.schema.json index 8de90e7..99d551f 100644 --- a/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/tests-format.schema.json +++ b/plugins/foundry-skills/skills/changeset-to-galaxy-test-plan/references/schemas/tests-format.schema.json @@ -1110,6 +1110,18 @@ "default": "Collection", "title": "Class" }, + "count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Count" + }, "element_tests": { "anyOf": [ { @@ -1171,6 +1183,30 @@ ], "default": null, "title": "Elements" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Maximum Count" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum Count" } }, "title": "TestCollectionCollectionElementAssertions", @@ -1366,6 +1402,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestCollectionDatasetElementAssertions", @@ -1679,6 +1728,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestDataOutputAssertions", diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/SKILL.md b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/SKILL.md index 0b5b575..b87b307 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/SKILL.md +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/SKILL.md @@ -17,20 +17,22 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `nextflow-galaxy-data-flow`. Produced by `nextflow-summary-to-galaxy-data-flow`. Galaxy data-flow brief from nextflow-summary-to-galaxy-data-flow when running the NEXTFLOW → GALAXY pipeline. - Read artifact `cwl-galaxy-interface`. Produced by `cwl-summary-to-galaxy-interface`. Galaxy interface brief from cwl-summary-to-galaxy-interface when running the CWL → GALAXY pipeline. - Read artifact `cwl-galaxy-data-flow`. Produced by `cwl-summary-to-galaxy-data-flow`. Galaxy data-flow brief from cwl-summary-to-galaxy-data-flow when running the CWL → GALAXY pipeline. -- Read artifact `freeform-galaxy-interface`. Produced by `freeform-summary-to-galaxy-interface`. Galaxy interface brief from freeform-summary-to-galaxy-interface when running the PAPER → GALAXY or INTERVIEW → GALAXY pipelines. -- Read artifact `freeform-galaxy-data-flow`. Produced by `freeform-summary-to-galaxy-data-flow`. Galaxy data-flow brief from freeform-summary-to-galaxy-data-flow when running the PAPER → GALAXY or INTERVIEW → GALAXY pipelines. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `freeform-galaxy-interface`. Produced by `freeform-summary-to-galaxy-interface`. Galaxy interface brief from freeform-summary-to-galaxy-interface when running the direct PAPER → GALAXY or INTERVIEW → GALAXY pipeline. +- Read artifact `freeform-galaxy-data-flow`. Produced by `freeform-summary-to-galaxy-data-flow`. Galaxy data-flow brief from freeform-summary-to-galaxy-data-flow when running the direct PAPER → GALAXY or INTERVIEW → GALAXY pipeline. +- Read artifact `workflow-brief-galaxy-interface`. Produced by `workflow-brief-to-galaxy-interface`. Galaxy interface brief from workflow-brief-to-galaxy-interface when running a Workflow-Brief-driven Galaxy path. +- Read artifact `workflow-brief-galaxy-data-flow`. Produced by `workflow-brief-to-galaxy-data-flow`. Galaxy data-flow brief from workflow-brief-to-galaxy-data-flow when running a Workflow-Brief-driven Galaxy path. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs -- Write artifact `iwc-comparison-notes` as `iwc-comparison-notes.md`. Format: `markdown`. Structural diff against the nearest IWC exemplar(s); guidance for the downstream *-summary-to-galaxy-template Mold before per-step authoring. Carries an inline, bounded gxformat2 excerpt of the nearest exemplar's relevant subgraph under a labeled section, cross-referencing the iwc-exemplar-gxformat2 sibling file. +- Write artifact `iwc-comparison-notes` as `iwc-comparison-notes.md`. Format: `markdown`. Structural diff against the nearest IWC exemplar(s); guidance for the downstream Galaxy template Mold before per-step authoring. Carries an inline, bounded gxformat2 excerpt of the nearest exemplar's relevant subgraph under a labeled section, cross-referencing the iwc-exemplar-gxformat2 sibling file. - Write artifact `iwc-exemplar-gxformat2` as `iwc-exemplar.gxwf.yml`. Format: `yaml`. Cleaned gxformat2 conversion (via convert --to format2 --compact) of the nearest IWC exemplar's relevant subgraph — the concrete idiom the downstream template draft pattern-matches against. Bounded to the relevant subgraph, not the whole workflow. Absent when no nearest exemplar is found. - Write artifact `open-requirements-ledger` as `open-requirements.ledger.yml`. Format: `yaml`. Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact. ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli @@ -51,7 +53,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Procedure -Find the nearest IWC exemplar workflow(s) for the upstream Galaxy design briefs and emit a structural diff that guides the downstream `*-summary-to-galaxy-template` skill before per-step authoring effort is spent. +Find the nearest IWC exemplar workflow(s) for the upstream Galaxy design briefs and emit a structural diff that guides the downstream Galaxy template skill before per-step authoring effort is spent. This skill is the corpus-first check in Galaxy-targeting pipelines. It runs after the source-specific interface and data-flow briefs and before the gxformat2 template skill. Discovery, ranking, and comparison are one action — there is no separate retrieval skill. @@ -86,10 +88,10 @@ Domain comes first so a structurally similar workflow in the wrong science area Each finding should name the authoring surface most likely to own the fix: -- Template/data-flow issue: missing node, wrong collection shape, wrong branch, placeholder too vague — surfaced for the downstream `*-summary-to-galaxy-template` skill to apply. +- Template/data-flow issue: missing node, wrong collection shape, wrong branch, placeholder too vague — surfaced for the downstream Galaxy template skill to apply. - Pattern issue: recurring Galaxy idiom should become or update a pattern page. - Tool-step issue: exact wrapper or parameterization will be handled later in the per-step loop. -- Test issue: defer to `*-test-to-galaxy-test-plan` or `implement-galaxy-workflow-test`. +- Test issue: defer to the pipeline's test-plan skill or `implement-galaxy-workflow-test`. Do not block downstream authoring on low-confidence exemplar mismatches. Report them as review guidance for the template skill and the user. diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_provenance.json b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_provenance.json index 608920d..3c15073 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_provenance.json +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "compare-against-iwc-exemplar", "path": "content/molds/compare-against-iwc-exemplar/index.md", - "revision": 10, - "content_hash": "5b198e9b87cf554cb01aceaa14a5aaa3ab24c6a35afc35dab7d09ce154cc3a77", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 12, + "content_hash": "84abb427cbcbae5616ae22e4a7fa803f14350605c246638575b3a32c5ad9c6fb", + "commit": "94ce4ea20107680b467d03e8889b63689774e325" }, - "cast_at": "2026-09-13T20:58:26.504Z", + "cast_at": "2026-11-10T10:05:12.274Z", "cast_date": "2026-05-07", "cast_revision": 3, "cast_history": [ @@ -40,8 +40,8 @@ "evidence": "corpus-observed", "purpose": "Normalize fetched IWC workflows into a consistent representation for structural comparison, and surface the nearest exemplar forward as a cleaned gxformat2 view (sibling file plus inline excerpt) for the downstream template Mold.", "trigger": "After fetching a candidate IWC workflow file and before structural comparison; and again on the nearest exemplar after ranking to emit the iwc-exemplar-gxformat2 view.", - "src_hash": "1bb19d20b4a39b1708d8f5346a0294616cdb9df83ba5255c4a90d97030d13b83", - "dst_hash": "1cdf49f10f77039af6efc5b82fe79720bb18e50d1f57d653670697f7262c7640", + "src_hash": "bfbf46667e803c9ef8b176e2f01a2dbf5b0cd013bfdbf5182280803e736b3fc7", + "dst_hash": "ff19b97f01a45938a25ac68c6865903c313c94d939a5d5084250cb2191f9355c", "source": "deterministic" }, { @@ -56,8 +56,8 @@ "purpose": "Compare against the design briefs' abstract intent without turning exemplar comparison into tool resolution.", "trigger": "When deciding whether to compare abstract data-flow shape, interface structure, or speculative implementation details.", "verification": "Promote after exemplar comparison flags structural issues without resolving concrete tool metadata.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -71,8 +71,8 @@ "evidence": "corpus-observed", "purpose": "Flag proposed shortcuts that are accepted in IWC versus shortcuts that should be treated as smells.", "trigger": "When the design briefs propose tests, assertions, labels, or expected-output comparisons.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -86,8 +86,8 @@ "evidence": "corpus-observed", "purpose": "Compare proposed test-data placement and fixture shapes against IWC conventions.", "trigger": "When the design briefs hint at workflow tests or input fixture organization.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -101,8 +101,8 @@ "evidence": "hypothesis", "purpose": "Close the open entries a corpus exemplar answers — a collection idiom, a label convention, a structural shape the corpus already settles — and append the ones no exemplar covers, so a structural divergence with no corpus precedent reaches the template tier as a recorded obligation.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" } ], @@ -112,7 +112,7 @@ "id": "iwc-comparison-notes", "kind": "markdown", "default_filename": "iwc-comparison-notes.md", - "description": "Structural diff against the nearest IWC exemplar(s); guidance for the downstream *-summary-to-galaxy-template Mold before per-step authoring. Carries an inline, bounded gxformat2 excerpt of the nearest exemplar's relevant subgraph under a labeled section, cross-referencing the iwc-exemplar-gxformat2 sibling file." + "description": "Structural diff against the nearest IWC exemplar(s); guidance for the downstream Galaxy template Mold before per-step authoring. Carries an inline, bounded gxformat2 excerpt of the nearest exemplar's relevant subgraph under a labeled section, cross-referencing the iwc-exemplar-gxformat2 sibling file." }, { "id": "iwc-exemplar-gxformat2", @@ -158,18 +158,32 @@ }, { "id": "freeform-galaxy-interface", - "description": "Galaxy interface brief from [[freeform-summary-to-galaxy-interface]] when running the PAPER → GALAXY or INTERVIEW → GALAXY pipelines.", + "description": "Galaxy interface brief from [[freeform-summary-to-galaxy-interface]] when running the direct PAPER → GALAXY or INTERVIEW → GALAXY pipeline.", "producers": [ "freeform-summary-to-galaxy-interface" ] }, { "id": "freeform-galaxy-data-flow", - "description": "Galaxy data-flow brief from [[freeform-summary-to-galaxy-data-flow]] when running the PAPER → GALAXY or INTERVIEW → GALAXY pipelines.", + "description": "Galaxy data-flow brief from [[freeform-summary-to-galaxy-data-flow]] when running the direct PAPER → GALAXY or INTERVIEW → GALAXY pipeline.", "producers": [ "freeform-summary-to-galaxy-data-flow" ] }, + { + "id": "workflow-brief-galaxy-interface", + "description": "Galaxy interface brief from [[workflow-brief-to-galaxy-interface]] when running a Workflow-Brief-driven Galaxy path.", + "producers": [ + "workflow-brief-to-galaxy-interface" + ] + }, + { + "id": "workflow-brief-galaxy-data-flow", + "description": "Galaxy data-flow brief from [[workflow-brief-to-galaxy-data-flow]] when running a Workflow-Brief-driven Galaxy path.", + "producers": [ + "workflow-brief-to-galaxy-data-flow" + ] + }, { "id": "open-requirements-ledger", "description": "Carried obligations ledger [[open-requirements-ledger]]: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one.", @@ -190,7 +204,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_required_tools.json b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_required_tools.json index 2ba8541..6a6405f 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_required_tools.json +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/_required_tools.json @@ -3,9 +3,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/cli/convert.json b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/cli/convert.json index 0f4e597..8769ba9 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/cli/convert.json +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/cli/convert.json @@ -4,7 +4,7 @@ "command": "convert", "summary": "Convert a Galaxy workflow between native (.ga) and format2 (.gxwf.yml) representations.", "source_path": "content/cli/gxwf/convert.md", - "source_revision": 2, + "source_revision": 3, "package": "@galaxy-tool-util/cli", "description": "Convert between native (.ga) and format2 (.gxwf.yml) formats", "synopsis": "gxwf convert [options] ", @@ -72,15 +72,23 @@ { "flags": "--stateful", "name": "stateful", - "description": "Use cached tool definitions for schema-aware state re-encoding", + "description": "Force schema-aware state re-encoding even if the tool cache is empty", "takesArgument": false, "optionalArgument": false, "negatable": false }, + { + "flags": "--no-stateful", + "name": "stateful", + "description": "Skip schema-aware state re-encoding; copy tool_state through verbatim", + "takesArgument": false, + "optionalArgument": false, + "negatable": true + }, { "flags": "--cache-dir ", "name": "cacheDir", - "description": "Tool cache directory (for --stateful)", + "description": "Tool cache directory (default: schema-aware re-encoding when it is populated)", "takesArgument": true, "argumentPlaceholder": "", "optionalArgument": false, @@ -119,5 +127,5 @@ "negatable": false } ], - "body": "# `gxwf convert`\n\nConvert a Galaxy workflow between the native `.ga` JSON and the `.gxwf.yml` format2 representation. Use this to normalize fetched IWC workflows into a consistent comparison representation.\n\n## Output\n\nDefault output is the converted workflow on stdout. With `--output`, the result is written to a file. JSON output is selected with `--json` (or `--to native`); YAML output is selected with `--yaml` (or `--to format2`).\n\n## Examples\n\n```bash\ngxwf convert workflow.ga --to format2 --output workflow.gxwf.yml\ngxwf convert workflow.ga --to format2 --compact\ngxwf convert workflow.gxwf.yml --to native --output workflow.ga\n```\n\n## Gotchas\n\n- Default output is stdout; pipe or pass `--output` when persisting.\n- `--compact` drops node position metadata; useful for structural diffs and skeleton generation.\n- `--stateful` requires a populated tool cache (see `galaxy-tool-cache`); without it, tool-state stays as fetched." + "body": "# `gxwf convert`\n\nConvert a Galaxy workflow between the native `.ga` JSON and the `.gxwf.yml` format2 representation. Use this to normalize fetched IWC workflows into a consistent comparison representation.\n\n## Output\n\nDefault output is the converted workflow on stdout. With `--output`, the result is written to a file. JSON output is selected with `--json` (or `--to native`); YAML output is selected with `--yaml` (or `--to format2`).\n\n## Examples\n\n```bash\ngxwf convert workflow.ga --to format2 --output workflow.gxwf.yml\ngxwf convert workflow.ga --to format2 --compact\ngxwf convert workflow.gxwf.yml --to native --output workflow.ga\n```\n\n## Gotchas\n\n- Default output is stdout; pipe or pass `--output` when persisting.\n- `--compact` drops node position metadata; useful for structural diffs and skeleton generation.\n- With a populated tool cache, conversion re-encodes tool state using cached definitions by default. Use `--no-stateful` to copy tool state through verbatim, or `--stateful` to force re-encoding even when the cache is empty. See [[galaxy-tool-cache]].\n- Version `1.13.1` preserves numeric input `min` and `max` as native Galaxy range validators and recovers them on export." } diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/compare-against-iwc-exemplar/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/SKILL.md b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/SKILL.md index 67672c5..96ebff2 100644 --- a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/SKILL.md +++ b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/SKILL.md @@ -23,8 +23,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ Bundled reference: `references/cli/planemo.md`. diff --git a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_provenance.json b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_provenance.json index c786cc0..f0c9e8f 100644 --- a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_provenance.json +++ b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/convert-nfcore-module-to-galaxy-tool/index.md", "revision": 11, "content_hash": "35e945843889b66b1e87b894db30670959431d4eb871cce5e74b541b7f527098", - "commit": "ddf54ebebb28dad664e9e95e57f6ed4d6f157ef3" + "commit": "a25b85b301234170473cd3512ed0bf7619e1cd46" }, - "cast_at": "2026-09-16T15:17:06.396Z", + "cast_at": "2026-11-09T17:08:37.182Z", "cast_date": "2026-07-20", "cast_revision": 2, "cast_history": [ @@ -36,7 +36,7 @@ "purpose": "Reference for `planemo lint` flags and output classification; first gate in the convergence loop.", "trigger": "Step 10.1 — after every // emission.", "verification": "Cast and confirm the cast skill consults the manpage's exit-code and error-shape sections, not free-text guesses.", - "src_hash": "d129ae09924ec895ad6350184f5e364b5bc4972f4ce5103badc71a4bc4196593", + "src_hash": "b1a49ad5c89b9b4d259ef69d70bdc350ebcb02145ce1d81b0eb35b385eaf3db7", "dst_hash": "d3251d1d2925f375c0446213b1e1abf3abc13fd6d1d860641fe55bf6332c4072", "source": "deterministic" }, @@ -52,8 +52,8 @@ "purpose": "Reference for `planemo test --test_output_json` invocation, exit codes, and the JSON report path.", "trigger": "Step 10.2 — after lint clears.", "verification": "Cast and confirm the cast skill always runs with --test_output_json and reads from the JSON, not stdout.", - "src_hash": "fec237a064746d57c800f4f55c96bfc54ed375debda518649ad66e47a1d27c8f", - "dst_hash": "42790a9b0ab08912dda7119da3dfd00dfd791531467ae633747a50248aadc9cb", + "src_hash": "27518720eaa8fff04966bfb121ed0e63e3daaaa0a7c32f831bac2559be38cc9e", + "dst_hash": "7f978461e4eed6fc6e11afb6625665079c0563b80468022a40b8cc78508c16e5", "source": "deterministic" }, { @@ -68,8 +68,8 @@ "purpose": "Install metadata for the planemo CLI invoked by the convergence loop.", "trigger": "Always — the cast skill needs planemo on PATH before running lint/test.", "verification": "Cast and confirm the bundle's _required_tools.json carries the pinned planemo install command.", - "src_hash": "12e8cac3ffb4ab7f007ced7a8d4f01e6b3785d06a55926496bbacc0a8f3fc5c5", - "dst_hash": "12e8cac3ffb4ab7f007ced7a8d4f01e6b3785d06a55926496bbacc0a8f3fc5c5", + "src_hash": "7a149ff6f1cc396dd72bbdd7d0b93acecb12e1b86663c6119d4c0815b411fe94", + "dst_hash": "7a149ff6f1cc396dd72bbdd7d0b93acecb12e1b86663c6119d4c0815b411fe94", "source": "deterministic" }, { @@ -116,8 +116,8 @@ "purpose": "Reference for nf-core module conventions: meta.yml shape, modules.json, environment.yml posture, test layout, container directive idioms.", "trigger": "When parsing meta.yml, environment.yml, or main.nf and a convention is unclear; when populating _provenance.yml.", "verification": "Cast and verify _provenance.yml.nfcore_source.git_sha resolution and meta.yml DOI extraction work for fastp.", - "src_hash": "f62c4bf50c3c42dcbd4e6283dc44e21571ff91a9e219daf74330cc65174d58fd", - "dst_hash": "f62c4bf50c3c42dcbd4e6283dc44e21571ff91a9e219daf74330cc65174d58fd", + "src_hash": "c3f26f9027b0e7a59fc21cae9ae82aea0c79d5bd5a1a71854a98b28944003c63", + "dst_hash": "c3f26f9027b0e7a59fc21cae9ae82aea0c79d5bd5a1a71854a98b28944003c63", "source": "deterministic" }, { diff --git a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_required_tools.json b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_required_tools.json index 9fd8b96..bd420e4 100644 --- a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_required_tools.json +++ b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/_required_tools.json @@ -3,9 +3,9 @@ "tool": "planemo", "origin": "pypi", "package": "planemo", - "package_version": "0.75.47", + "package_version": "0.75.48", "invoke": "planemo", - "invoke_fallback": "uvx --from planemo==0.75.47 planemo", + "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "reference": "references/cli/planemo.md", diff --git a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo-test.json b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo-test.json index 194534e..13477aa 100644 --- a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo-test.json +++ b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo-test.json @@ -5,5 +5,5 @@ "summary": "Run specified tool or workflow tests within Galaxy.", "source_path": "content/cli/planemo/planemo-test.md", "source_revision": 1, - "body": "\n\n# `planemo test`\n\nRun specified tool or workflow tests within Galaxy.\n\n## Synopsis\n\n```text\nplanemo test [OPTIONS] TOOL_PATH\n```\n\n\n## Arguments\n\n| Argument | Type | Required | Help |\n|---|---|---|---|\n| TOOL_PATH | path | — | — |\n\n## Options\n\n| Option | Type | Default | Required | Help |\n|---|---|---|---|---|\n| --failed, --lf | flag | false | — | Re-run only failed tests from the previous run. Reads from --failed_json (or --test_output_json if not set) to determine which tests failed. |\n| --failed_json | path | — | — | JSON file from a previous planemo test run to read failed test IDs from when using --failed/--lf. Defaults to --test_output_json. |\n| --test_index | integer | [] | — | Index(es) of specific test(s) to run (1-based). Can be specified multiple times (e.g., --test_index 1 --test_index 3) to run specific tests. If not specified, all tests are run. |\n| --polling_backoff | integer | 0 | — | Poll resources with an increasing interval between requests. Useful when testing against remote and/or production instances to limit generated traffic. |\n| --galaxy_root | directory | — | — | Root of development galaxy directory to execute command with. |\n| --galaxy_python_version | choice | — | — | Python version to start Galaxy under |\n| --extra_tools | path | — | — | Extra tool sources to include in Galaxy's tool panel (file or directory). These will not be linted/tested/etc... but they will be available to workflows and for interactive use. |\n| --install_galaxy | flag | — | — | Download and configure a disposable copy of Galaxy from github. |\n| --galaxy_branch | text | — | — | Branch of Galaxy to target (defaults to master) if a Galaxy root isn't specified. |\n| --galaxy_source | text | — | — | Git source of Galaxy to target (defaults to the official galaxyproject github source if a Galaxy root isn't specified. |\n| --skip_venv | flag | — | — | Do not create or source a virtualenv environment for Galaxy, this should be used to preserve an externally configured virtual environment or conda environment. |\n| --no_cache_galaxy | flag | — | — | Skip caching of Galaxy source and dependencies obtained with --install_galaxy. Not caching this results in faster downloads (no git) - so is better on throw away instances such with TravisCI. |\n| --no_cleanup | flag | — | — | Do not cleanup temp files created for and by Galaxy. |\n| --galaxy_email | text | planemo@galaxyproject.org | — | E-mail address to use when launching single-user Galaxy server. |\n| --docker, --no_docker | flag | false | — | Run Galaxy tools in Docker if enabled. |\n| --docker_cmd | text | docker | — | Command used to launch docker (defaults to docker). |\n| --docker_sudo, --no_docker_sudo | flag | — | — | Flag to use sudo when running docker. |\n| --docker_host | text | — | — | Docker host to target when executing docker commands (defaults to localhost). |\n| --docker_sudo_cmd | text | sudo | — | sudo command to use when --docker_sudo is enabled (defaults to sudo). |\n| --docker_run_extra_arguments | text | — | — | Extra arguments to pass to docker run. |\n| --mulled_containers, --biocontainers | flag | — | — | Test tools against mulled containers (forces --docker). Disables conda resolution unless any conda option has been set explicitly. |\n| --galaxy_startup_timeout | integer range | 900 | — | Wait for galaxy to start before assuming Galaxy did not start. |\n| --job_config_file | file | — | — | Job configuration file for Galaxy to target. |\n| --job_workers | integer | 1 | — | Number of workers for the local job runner (default 1). |\n| --tool_dependency_dir | directory | — | — | Tool dependency dir for Galaxy to target. |\n| --tool_data_path | directory | — | — | Directory where data used by tools is located. Required if tests are run in docker and should make use of external reference data. |\n| --test_data | directory | — | — | test-data directory to for specified tool(s). |\n| --tool_data_table | path | — | — | tool_data_table_conf.xml file to for specified tool(s). |\n| --dependency_resolvers_config_file | file | — | — | Dependency resolver configuration for Galaxy to target. |\n| --brew_dependency_resolution | flag | — | — | Configure Galaxy to use plain brew dependency resolution. |\n| --shed_dependency_resolution | flag | — | — | Configure Galaxy to use brewed Tool Shed dependency resolution. |\n| --no_dependency_resolution | flag | — | — | Configure Galaxy with no dependency resolvers. |\n| --conda_prefix | directory | — | — | Conda prefix to use for conda dependency commands. |\n| --conda_exec | file | — | — | Location of conda executable. |\n| --conda_channels, --conda_ensure_channels | text | conda-forge,bioconda | — | Ensure conda is configured with specified comma separated list of channels. |\n| --conda_use_local | flag | — | — | Use locally built packages while building Conda environments. |\n| --conda_dependency_resolution | flag | — | — | Configure Galaxy to use only conda for dependency resolution. |\n| --conda_auto_install, --no_conda_auto_install | flag | true | — | Conda dependency resolution for Galaxy will attempt to install requested but missing packages. |\n| --conda_auto_init, --no_conda_auto_init | flag | true | — | Conda dependency resolution for Galaxy will auto install conda itself using miniforge if not availabe on conda_prefix. |\n| --simultaneous_uploads, --no_simultaneous_uploads | flag | false | — | When uploading files to Galaxy for tool or workflow tests or runs, upload multiple files simultaneously without waiting for the previous file upload to complete. |\n| --check_uploads_ok, --no_check_uploads_ok | flag | true | — | When uploading files to Galaxy for tool or workflow tests or runs, check that the history is in an 'ok' state before beginning tool or workflow execution. |\n| --profile | text | — | — | Name of profile (created with the profile_create command) to use with this command. |\n| --database_type | choice | auto | — | Type of database to use for profile - 'auto', 'sqlite', 'postgres', 'postgres_docker' , and postgres_singularity are available options. Use postgres to use an existing postgres server you user can access without a password via the psql command. Use postgres_docker to have Planemo manage a docker container running postgres. . Use postgres_singularity to have Planemo run postgres using singularity/apptainer. Data with postgres_docker is not yet persisted past when you restart the docker container launched by Planemo so be careful with this option. |\n| --postgres_psql_path | text | psql | — | Name or or path to postgres client binary (psql). |\n| --postgres_database_user | text | postgres | — | Postgres username for managed development databases. |\n| --postgres_database_host | text | — | — | Postgres host name for managed development databases. |\n| --postgres_database_port | text | — | — | Postgres port for managed development databases. |\n| --file_path | directory | — | — | Location for files created by Galaxy (e.g. database/files). |\n| --database_connection | text | — | — | Database connection string to use for Galaxy. |\n| --postgres-storage-location | text | — | — | storage path for postgres database, used for local singularity postgres. |\n| --shed_tool_conf | text | — | — | Location of shed tools conf file for Galaxy. |\n| --shed_tool_path | text | — | — | Location of shed tools directory for Galaxy. |\n| --shed_tool_data_table_config | text | — | — | Location of the shed tool data table config file for Galaxy (records data tables registered by shed-installed repositories). |\n| --shed_data_manager_config | text | — | — | Location of the shed data manager config file for Galaxy. |\n| --shed_data_dir | directory | — | — | Persistent base directory for shed-install state (local Galaxy engine). Seeds defaults for --shed_tool_conf, --shed_tool_path, --shed_tool_data_table_config and --shed_data_manager_config so shed installs (tools and their data tables) survive Galaxy restarts. Individual options still override. |\n| --galaxy_single_user, --no_galaxy_single_user | flag | true | — | By default Planemo will configure Galaxy to run in single-user mode where there is just one user and this user is automatically logged it. Use --no_galaxy_single_user to prevent Galaxy from running this way. |\n| --tool_evaluation_strategy | choice | — | — | Determines which process will evaluate the tool command line. If set to 'local' the tool command line will be templated in the job handler process. If set to 'remote' the tool command line will be built as part of the submitted job (beta). Setting this to 'remote' will also implicitly set metadata_strategy to 'extended', which is required for remote tool evaluation. |\n| --paste_test_data_paths, --no_paste_test_data_paths | flag | — | — | By default Planemo will use or not use Galaxy's path paste option to load test data into a history based on the engine type it is targeting. This can override the logic to explicitly enable or disable path pasting. |\n| --update_test_data | flag | — | — | Update test-data directory with job outputs (normally written to directory --job_output_files if specified.) |\n| --test_output | path | tool_test_output.html | — | Output test report (HTML - for humans) defaults to tool_test_output.html. |\n| --test_output_text | path | — | — | Output test report (Basic text - for display in CI) |\n| --test_output_markdown | path | — | — | Output test report (Markdown style - for humans & computers) |\n| --test_output_markdown_minimal | path | — | — | Output test report (Minimal markdown style - jost the table) |\n| --test_output_xunit | path | — | — | Output test report (xunit style - for CI systems |\n| --test_output_junit | path | — | — | Output test report (jUnit style - for CI systems |\n| --test_output_allure | directory | — | — | Output test allure2 framework resutls |\n| --test_output_json | path | tool_test_output.json | — | Output test report (planemo json) defaults to tool_test_output.json. |\n| --job_output_files | directory | — | — | Write job outputs to specified directory. |\n| --summary | choice | minimal | — | Summary style printed to planemo's standard output (see output reports for more complete summary). Set to 'none' to disable completely. |\n| --test_timeout | integer | 86400 | — | Maximum runtime of a single test in seconds. |\n| --fail_fast | flag | — | — | Stop on first job failure. |\n| --engine | choice | — | — | Select an engine to run or test artifacts such as tools and workflows. Defaults to a local Galaxy, but running Galaxy within a Docker container or the CWL reference implementation 'cwltool' and 'toil' be selected. |\n| --non_strict_cwl | flag | false | — | Disable strict validation of CWL. |\n| --no-container, --no_container | flag | false | — | If cwltool engine is used, disable Docker container usage. |\n| --docker_galaxy_image | text | quay.io/bgruening/galaxy | — | Docker image identifier for docker-galaxy-flavor used if engine type is specified as ``docker-galaxy``. Defaults to quay.io/bgruening/galaxy. |\n| --docker_extra_volume | path | — | — | Extra path to mount if --engine docker or `--biocontainers` or `--docker`. |\n| --ignore_dependency_problems | flag | false | — | When installing shed repositories for workflows, ignore dependency issues. These likely indicate a problem but in some cases may not prevent a workflow from successfully executing. |\n| --shed_install, --no_shed_install | flag | true | — | By default Planemo will attempt to install repositories needed for workflow testing. This may not be appropriate for production servers and so this can disabled by calling planemo with --no_shed_install. |\n| --install_tool_dependencies, --no_install_tool_dependencies | flag | false | — | Turn on installation of tool dependencies using classic toolshed packages. |\n| --install_resolver_dependencies, --no_install_resolver_dependencies | flag | true | — | Skip installing tool dependencies through resolver (e.g. conda). |\n| --install_repository_dependencies, --no_install_repository_dependencies | flag | true | — | Skip installing the repository dependencies. |\n| --galaxy_url | text | — | — | Remote Galaxy URL to use with external Galaxy engine. |\n| --galaxy_admin_key | text | — | — | Admin key to use with external Galaxy engine. |\n| --galaxy_user_key | text | — | — | User key to use with external Galaxy engine. |\n| --history_name | text | — | — | Name to give a Galaxy history, if one is created. |\n| --history_id | text | — | — | Send the results of the run to the history with the provided ID. A history with this ID must exist. |\n| --no_wait | flag | false | — | After invoking a job or workflow, do not wait for completion. |\n\n\n## Output\n\n\n\nUse `--test_output_json ` for machine-readable results; stdout and stderr are human-oriented.\n\n## Examples\n\n\n\n```sh\nplanemo test --test_output_json /_planemo_test_report.json\n```\n\n## Gotchas\n\n\n\nNo Foundry-specific gotchas recorded yet." + "body": "\n\n# `planemo test`\n\nRun specified tool or workflow tests within Galaxy.\n\n## Synopsis\n\n```text\nplanemo test [OPTIONS] TOOL_PATH\n```\n\n\n## Arguments\n\n| Argument | Type | Required | Help |\n|---|---|---|---|\n| TOOL_PATH | path | — | — |\n\n## Options\n\n| Option | Type | Default | Required | Help |\n|---|---|---|---|---|\n| --failed, --lf | flag | false | — | Re-run only failed tests from the previous run. Reads from --failed_json (or --test_output_json if not set) to determine which tests failed. |\n| --failed_json | path | — | — | JSON file from a previous planemo test run to read failed test IDs from when using --failed/--lf. Defaults to --test_output_json. |\n| --test_index | integer | [] | — | Index(es) of specific test(s) to run (1-based). Can be specified multiple times (e.g., --test_index 1 --test_index 3) to run specific tests. If not specified, all tests are run. |\n| --polling_backoff | integer | 0 | — | Poll resources with an increasing interval between requests. Useful when testing against remote and/or production instances to limit generated traffic. |\n| --use_cache, --no_use_cache | flag | false | — | Reuse cached job results if available. Off by default - Galaxy replays the outputs of an equivalent job, so a tool edited without a version bump is never actually re-run and the test passes against stale results. Only honored for tests defined in a test file; tests embedded in a tool's block run through the Galaxy test interactor, which ignores this. |\n| --cwltool_cache_directory | directory | — | — | Directory the cwltool engine caches computed steps in when --use_cache is enabled (defaults to a directory in the planemo workspace). |\n| --galaxy_root | directory | — | — | Root of development galaxy directory to execute command with. |\n| --galaxy_python_version | choice | — | — | Python version to start Galaxy under |\n| --extra_tools | path | — | — | Extra tool sources to include in Galaxy's tool panel (file or directory). These will not be linted/tested/etc... but they will be available to workflows and for interactive use. |\n| --install_galaxy | flag | — | — | Download and configure a disposable copy of Galaxy from github. |\n| --galaxy_branch | text | — | — | Branch of Galaxy to target (defaults to master) if a Galaxy root isn't specified. |\n| --galaxy_source | text | — | — | Git source of Galaxy to target (defaults to the official galaxyproject github source if a Galaxy root isn't specified. |\n| --skip_venv | flag | — | — | Do not create or source a virtualenv environment for Galaxy, this should be used to preserve an externally configured virtual environment or conda environment. |\n| --no_cache_galaxy | flag | — | — | Skip caching of Galaxy source and dependencies obtained with --install_galaxy. Not caching this results in faster downloads (no git) - so is better on throw away instances such with TravisCI. |\n| --no_cleanup | flag | — | — | Do not cleanup temp files created for and by Galaxy. |\n| --galaxy_email | text | planemo@galaxyproject.org | — | E-mail address to use when launching single-user Galaxy server. |\n| --docker, --no_docker | flag | false | — | Run Galaxy tools in Docker if enabled. |\n| --docker_cmd | text | docker | — | Command used to launch docker (defaults to docker). |\n| --docker_sudo, --no_docker_sudo | flag | — | — | Flag to use sudo when running docker. |\n| --docker_host | text | — | — | Docker host to target when executing docker commands (defaults to localhost). |\n| --docker_sudo_cmd | text | sudo | — | sudo command to use when --docker_sudo is enabled (defaults to sudo). |\n| --docker_run_extra_arguments | text | — | — | Extra arguments to pass to docker run. |\n| --mulled_containers, --biocontainers | flag | — | — | Test tools against mulled containers. Requires --docker or --singularity; --docker is enabled automatically if neither is given. Disables conda resolution unless any conda option has been set explicitly. |\n| --container_resolvers_config_file | file | — | — | Path to a Galaxy container resolvers configuration file to use instead of Galaxy's default resolvers. |\n| --galaxy_startup_timeout | integer range | 900 | — | Wait for galaxy to start before assuming Galaxy did not start. |\n| --job_config_file | file | — | — | Job configuration file for Galaxy to target. |\n| --job_workers | integer | 1 | — | Number of workers for the local job runner (default 1). |\n| --tool_dependency_dir | directory | — | — | Tool dependency dir for Galaxy to target. |\n| --tool_data_path | directory | — | — | Directory where data used by tools is located. Required if tests are run in docker and should make use of external reference data. |\n| --test_data | directory | — | — | test-data directory to for specified tool(s). |\n| --tool_data_table | path | — | — | tool_data_table_conf.xml file to for specified tool(s). |\n| --dependency_resolvers_config_file | file | — | — | Dependency resolver configuration for Galaxy to target. |\n| --brew_dependency_resolution | flag | — | — | Configure Galaxy to use plain brew dependency resolution. |\n| --shed_dependency_resolution | flag | — | — | Configure Galaxy to use brewed Tool Shed dependency resolution. |\n| --no_dependency_resolution | flag | — | — | Configure Galaxy with no dependency resolvers. |\n| --conda_prefix | directory | — | — | Conda prefix to use for conda dependency commands. |\n| --conda_exec | file | — | — | Location of conda executable. |\n| --conda_channels, --conda_ensure_channels | text | conda-forge,bioconda | — | Ensure conda is configured with specified comma separated list of channels. |\n| --conda_use_local | flag | — | — | Use locally built packages while building Conda environments. |\n| --conda_dependency_resolution | flag | — | — | Configure Galaxy to use only conda for dependency resolution. |\n| --conda_auto_install, --no_conda_auto_install | flag | true | — | Conda dependency resolution for Galaxy will attempt to install requested but missing packages. |\n| --conda_auto_init, --no_conda_auto_init | flag | true | — | Conda dependency resolution for Galaxy will auto install conda itself using miniforge if not availabe on conda_prefix. |\n| --simultaneous_uploads, --no_simultaneous_uploads | flag | false | — | When uploading files to Galaxy for tool or workflow tests or runs, upload multiple files simultaneously without waiting for the previous file upload to complete. |\n| --check_uploads_ok, --no_check_uploads_ok | flag | true | — | When uploading files to Galaxy for tool or workflow tests or runs, check that the history is in an 'ok' state before beginning tool or workflow execution. |\n| --profile | text | — | — | Name of profile (created with the profile_create command) to use with this command. |\n| --database_type | choice | auto | — | Type of database to use for profile - 'auto', 'sqlite', 'postgres', 'postgres_docker' , and postgres_singularity are available options. The default 'auto' means sqlite - a postgres server is only stood up when named. Use postgres to use an existing postgres server you user can access without a password via the psql command. Use postgres_docker to have Planemo manage a docker container running postgres. . Use postgres_singularity to have Planemo run postgres using singularity/apptainer. Data with postgres_docker is not yet persisted past when you restart the docker container launched by Planemo so be careful with this option. |\n| --postgres_psql_path | text | psql | — | Name or or path to postgres client binary (psql). |\n| --postgres_database_user | text | postgres | — | Postgres username for managed development databases. |\n| --postgres_database_host | text | — | — | Postgres host name for managed development databases. |\n| --postgres_database_port | text | — | — | Postgres port for managed development databases. |\n| --postgres-storage-location, --postgres_storage_location | directory | — | — | Storage path for PostgreSQL data managed through Singularity. |\n| --singularity, --no_singularity | flag | false | — | Run Galaxy tools in Singularity if enabled. |\n| --singularity_cmd | text | singularity | — | Command used to execute singularity (defaults to 'singularity'). |\n| --singularity_sudo, --no_singularity_sudo | flag | — | — | Flag to use sudo when running Singularity. |\n| --singularity_sudo_cmd | text | sudo | — | sudo command to use when --singularity_sudo is enabled (defaults to sudo). |\n| --file_path | directory | — | — | Location for files created by Galaxy (e.g. database/files). |\n| --database_connection | text | — | — | Database connection string to use for Galaxy. |\n| --shed_tool_conf | text | — | — | Location of shed tools conf file for Galaxy. |\n| --shed_tool_path | text | — | — | Location of shed tools directory for Galaxy. |\n| --shed_tool_data_table_config | text | — | — | Location of the shed tool data table config file for Galaxy (records data tables registered by shed-installed repositories). |\n| --shed_data_manager_config | text | — | — | Location of the shed data manager config file for Galaxy. |\n| --shed_data_dir | directory | — | — | Persistent base directory for shed-install state (local Galaxy engine). Seeds defaults for --shed_tool_conf, --shed_tool_path, --shed_tool_data_table_config and --shed_data_manager_config so shed installs (tools and their data tables) survive Galaxy restarts. Individual options still override. |\n| --galaxy_single_user, --no_galaxy_single_user | flag | true | — | By default Planemo will configure Galaxy to run in single-user mode where there is just one user and this user is automatically logged it. Use --no_galaxy_single_user to prevent Galaxy from running this way. |\n| --tool_evaluation_strategy | choice | — | — | Determines which process will evaluate the tool command line. If set to 'local' the tool command line will be templated in the job handler process. If set to 'remote' the tool command line will be built as part of the submitted job (beta). Setting this to 'remote' will also implicitly set metadata_strategy to 'extended', which is required for remote tool evaluation. |\n| --paste_test_data_paths, --no_paste_test_data_paths | flag | — | — | By default Planemo will use or not use Galaxy's path paste option to load test data into a history based on the engine type it is targeting. This can override the logic to explicitly enable or disable path pasting. |\n| --update_test_data | flag | — | — | Update test-data directory with job outputs (normally written to directory --job_output_files if specified.) |\n| --test_output | path | tool_test_output.html | — | Output test report (HTML - for humans) defaults to tool_test_output.html. |\n| --test_output_text | path | — | — | Output test report (Basic text - for display in CI) |\n| --test_output_markdown | path | — | — | Output test report (Markdown style - for humans & computers) |\n| --test_output_markdown_minimal | path | — | — | Output test report (Minimal markdown style - jost the table) |\n| --test_output_xunit | path | — | — | Output test report (xunit style - for CI systems |\n| --test_output_junit | path | — | — | Output test report (jUnit style - for CI systems |\n| --test_output_allure | directory | — | — | Output test allure2 framework resutls |\n| --test_output_json | path | tool_test_output.json | — | Output test report (planemo json) defaults to tool_test_output.json. |\n| --job_output_files | directory | — | — | Write job outputs to specified directory. |\n| --summary | choice | minimal | — | Summary style printed to planemo's standard output (see output reports for more complete summary). Set to 'none' to disable completely. |\n| --test_timeout | integer | 86400 | — | Maximum runtime of a single test in seconds. |\n| --fail_fast | flag | — | — | Stop on first failure. |\n| --engine | choice | — | — | Select an engine to run or test artifacts such as tools and workflows. Defaults to a local Galaxy, but running Galaxy within a Docker container or the CWL reference implementation 'cwltool' and 'toil' be selected. |\n| --non_strict_cwl | flag | false | — | Disable strict validation of CWL. |\n| --no-container, --no_container | flag | false | — | If cwltool engine is used, disable Docker container usage. |\n| --docker_galaxy_image | text | quay.io/bgruening/galaxy | — | Docker image identifier for docker-galaxy-flavor used if engine type is specified as ``docker-galaxy``. Defaults to quay.io/bgruening/galaxy. |\n| --docker_extra_volume | path | — | — | Extra path to mount if --engine docker or `--biocontainers` or `--docker`. |\n| --singularity_extra_volume | path | — | — | Extra path to mount if `--biocontainers` or `--singularity`. |\n| --ignore_dependency_problems | flag | false | — | When installing shed repositories for workflows, ignore dependency issues. These likely indicate a problem but in some cases may not prevent a workflow from successfully executing. |\n| --shed_install, --no_shed_install | flag | true | — | By default Planemo will attempt to install repositories needed for workflow testing. This may not be appropriate for production servers and so this can disabled by calling planemo with --no_shed_install. |\n| --install_tool_dependencies, --no_install_tool_dependencies | flag | false | — | Turn on installation of tool dependencies using classic toolshed packages. |\n| --install_resolver_dependencies, --no_install_resolver_dependencies | flag | true | — | Skip installing tool dependencies through resolver (e.g. conda). |\n| --install_repository_dependencies, --no_install_repository_dependencies | flag | true | — | Skip installing the repository dependencies. |\n| --galaxy_url | text | — | — | Remote Galaxy URL to use with external Galaxy engine. |\n| --galaxy_admin_key | text | — | — | Admin key to use with external Galaxy engine. |\n| --galaxy_user_key | text | — | — | User key to use with external Galaxy engine. |\n| --history_name | text | — | — | Name to give a Galaxy history, if one is created. |\n| --history_id | text | — | — | Send the results of the run to the history with the provided ID. A history with this ID must exist. |\n| --no_wait | flag | false | — | After invoking a job or workflow, do not wait for completion. |\n\n\n## Output\n\n\n\nUse `--test_output_json ` for machine-readable results; stdout and stderr are human-oriented.\n\n## Examples\n\n\n\n```sh\nplanemo test --test_output_json /_planemo_test_report.json\n```\n\n## Gotchas\n\n\n\nNo Foundry-specific gotchas recorded yet." } diff --git a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo.md b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo.md index 10eeb14..d5fa285 100644 --- a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo.md +++ b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/cli/planemo.md @@ -3,17 +3,17 @@ type: cli-tool tool: planemo origin: pypi package: planemo -package_version: "0.75.47" +package_version: "0.75.48" invoke: planemo -invoke_fallback: "uvx --from planemo==0.75.47 planemo" +invoke_fallback: "uvx --from planemo==0.75.48 planemo" availability_check: "planemo --version" docs_url: "https://planemo.readthedocs.io/" tags: - cli/planemo status: draft created: 2026-05-10 -revised: 2026-09-13 -revision: 6 +revised: 2026-11-09 +revision: 7 summary: "Galaxy tool/workflow runtime testing CLI; used by run-workflow-test and friends." --- @@ -23,7 +23,7 @@ Galaxy's runtime testing and authoring CLI. Foundry Molds invoke `planemo test`, ## Pin -`package_version` pins to the released `planemo==0.75.47` from PyPI — base upstream planemo, no fork. It is the single source of truth: the sync scripts carry no pin of their own and stamp whichever planemo they invoked, and `make check-planemo-pin` fails if any recorded copy of the version disagrees with this field. +`package_version` pins to the released `planemo==0.75.48` from PyPI — base upstream planemo, no fork. It is the single source of truth: the sync scripts carry no pin of their own and stamp whichever planemo they invoked, and `make check-planemo-pin` fails if any recorded copy of the version disagrees with this field. [galaxyproject/planemo#1636](https://github.com/galaxyproject/planemo/pull/1636) merged 2026-05-14 and first shipped in released **0.75.42**, so `planemo cli_metadata` and `planemo output_schema` are available in every release at or above this pin. Every Foundry consumer runs off this base pin: the workflow-test phases ([[run-workflow-test]], [[implement-galaxy-workflow-test]]) which need only `planemo test --test_output_json`, the convergence loop in [[convert-nfcore-module-to-galaxy-tool]], and the vendored-artifact regeneration story (`packages/planemo-cli-meta/`, `packages/planemo-test-report-schema/`). No fork pin is required. @@ -32,13 +32,13 @@ The floor is now 0.75.46 for a packaging reason, not a feature one. `galaxy-tool ## Install ```sh -uvx --from planemo==0.75.47 planemo --version +uvx --from planemo==0.75.48 planemo --version ``` For a persistent install: ```sh -uv tool install planemo==0.75.47 +uv tool install planemo==0.75.48 ``` Contributor laptops only need planemo when **regenerating** vendored artifacts (the schema JSON in `packages/planemo-test-report-schema/`, the planemo CLI manual pages under `content/cli/planemo/`). Normal Foundry development — `npm run validate`, `npm run test`, `npm run packages-test` — builds against the vendored JSON and does not invoke planemo. diff --git a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/notes/component-nf-core-tools.md b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/notes/component-nf-core-tools.md index 4612e2c..93a6433 100644 --- a/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/notes/component-nf-core-tools.md +++ b/plugins/foundry-skills/skills/convert-nfcore-module-to-galaxy-tool/references/notes/component-nf-core-tools.md @@ -5,9 +5,9 @@ tags: component: "nf-core/tools (Python package + ecosystem)" status: draft created: 2026-05-01 -revised: 2026-05-01 -revision: 1 -summary: "White paper on nf-core/tools — conventions, CLI surface, schema universe, container resolution. Survey, not decision." +revised: 2026-09-26 +revision: 2 +summary: "nf-core/tools conventions, CLI and Python surfaces, schema validation, module provenance, linting, and container-download limits." related_molds: - "[[summarize-nextflow]]" - "[[convert-nfcore-module-to-galaxy-tool]]" @@ -15,7 +15,13 @@ related_notes: - "[[convert-nfcore-module-to-galaxy-tool]]" - "[[component-nextflow-containers-and-envs]]" sources: - - "https://github.com/nf-core/tools" + - "https://github.com/nf-core/tools/tree/eb2f709090f4054f45437c34049ea2068567c339" + - "https://github.com/nf-core/tools/blob/eb2f709090f4054f45437c34049ea2068567c339/nf_core/__main__.py" + - "https://github.com/nf-core/tools/blob/eb2f709090f4054f45437c34049ea2068567c339/nf_core/pydantic_models.py" + - "https://github.com/nf-core/tools/blob/eb2f709090f4054f45437c34049ea2068567c339/nf_core/utils.py" + - "https://github.com/nf-core/tools/blob/eb2f709090f4054f45437c34049ea2068567c339/nf_core/modules/modules_json.py" + - "https://github.com/nf-core/tools/blob/eb2f709090f4054f45437c34049ea2068567c339/nf_core/pipelines/schema.py" + - "https://github.com/nf-core/tools/blob/eb2f709090f4054f45437c34049ea2068567c339/nf_core/pipelines/download/download.py" - "https://nf-co.re/docs/nf-core-tools" - "https://nf-co.re/pipelines.json" - "https://github.com/nf-core/modules" @@ -23,198 +29,191 @@ sources: - "https://github.com/nf-core/configs" --- -# `nf-core/tools` and the nf-core Pipeline Toolchain: A Technical Survey +# nf-core/tools -**Source clone:** `~/projects/repositories/nf-core-tools` (commit `b6c5737`, version `4.0.2`). +`nf-core/tools` provides the Python package published as `nf-core`, its `nf-core` CLI, and importable modules under `nf_core`. It creates and synchronizes pipeline templates, manages vendored modules and subworkflows, validates schemas, reports convention violations, and downloads pipeline code with selected container images. -## Overview +These contracts were checked against **nf-core/tools 4.1.0**, commit `eb2f709090f4054f45437c34049ea2068567c339`. Source inspection establishes the behavior described here. No pipeline, module test, remote installation, or container download was run for this review. -`nf-core/tools` is the official Python package that the nf-core community publishes to PyPI as `nf-core` (current release **4.0.2**, May 2026). It is a Click-based CLI plus an importable Python library (`nf_core.*`) that handles essentially every lifecycle task a pipeline author or operator performs against an nf-core Nextflow pipeline: scaffolding new pipelines from a Jinja-rendered cookiecutter template, installing and updating shared modules and subworkflows from `nf-core/modules`, linting, schema management, listing remote pipelines, downloading pipelines together with their container images for offline use, and synchronising pipelines with the upstream template as it evolves. +For [[convert-nfcore-module-to-galaxy-tool]], the useful evidence is a module's command, documented IO, package environment, tests, and source pin. For [[summarize-nextflow]], the package helps interpret nf-core conventions. It does not produce a workflow graph or determine which processes a particular input will execute. -Conceptually the package solves three problems for the community: **enforcing convention** (every nf-core pipeline shares a directory layout, file inventory, and metadata schema, and `nf-core pipelines lint` is the reference enforcer), **enabling code reuse across pipelines** (the `modules`/`subworkflows` subcommands implement a Git-tracked package manager whose state lives in `modules.json`), and **bridging the pipeline to its surrounding registries** (pipelines.json, nf-core/configs, nf-core/test-datasets). +## Pipeline and component conventions -Historically the package began as a small scaffolding helper around 2018 and has tracked the nf-core pipeline standard ever since. The 2.x line introduced subworkflows; 3.x rewrote the create UI in Textual and added the schema validator plugin model; 4.x consolidates around nf-test as the canonical pipeline test harness, replaces `pytest_workflow` and `pytest_modules` style harnesses, and tightens the `.nf-core.yml` schema with a Pydantic v2 model. +The template in `nf_core/pipeline-template/` describes a conventional pipeline. Feature selection in `.nf-core.yml` can remove template sections and adjust lint expectations. An existing pipeline can also use an older template. The current template is therefore evidence of expected structure, not a mandatory inventory for every pipeline. -It sits in the centre of an ecosystem of GitHub repositories: `nf-core/pipelines.json` (pipeline registry), `nf-core/modules` (the canonical module + subworkflow registry), `nf-core/configs` (institutional Nextflow configs), `nf-core/test-datasets` (branch-per-pipeline test data), and the website `nf-co.re` which serves the schema-builder web UI and API. The CLI talks to all of them over HTTPS and the GitHub API. +| Path | Meaning and qualifications | +|---|---| +| `main.nf`, `workflows/.nf` | Entrypoint and pipeline workflow definitions. Template utility subworkflows provide initialization, schema handling, and completion behavior. | +| `nextflow.config`, `conf/` | Manifest, parameter defaults, execution profiles, and included configuration. Conventional test configurations are `conf/test.config` and `conf/test_full.config`. Their filenames do not establish the complete profile inventory. | +| `nextflow_schema.json` | Parameter schema. Draft and grouping notation depend on the validation plugin. | +| `modules/nf-core/[/]/` | Vendored module directories. Expected source and metadata include `main.nf`, `meta.yml`, `environment.yml`, and nf-test files under `tests/`. Some tools have no subtool directory. | +| `modules/local/` | Pipeline-owned modules. They are not necessarily represented in `modules.json`. | +| `subworkflows/nf-core/`, `subworkflows/local/` | Reusable and pipeline-owned compositions. Subworkflow `meta.yml` documents component dependencies. | +| `assets/` | Input schemas, reporting configuration, and other pipeline assets, according to enabled features. | +| `bin/` | Pipeline helper scripts available to processes. | +| `docs/`, `README.md`, `CITATIONS.md`, `CHANGELOG.md` | Usage, outputs, citation, and release documentation. | +| `nf-test.config`, `tests/` | Pipeline test configuration and test suites. Test definitions and snapshots are distinct evidence from configuration-only test profiles. | +| `.nf-core.yml`, `modules.json` | Tools configuration and installed-component provenance. | -## The nf-core conventions the tools encode +Pipeline `files_exist` lint distinguishes required, recommended, forbidden, and discouraged files. For example, the pinned default inventory treats `nf-test.config` and `tests/default.nf.test` as required, while `conf/base.config` is recommended. Feature skips and explicit lint exclusions affect the result. `modules_structure` checks directory placement, not the full module file inventory. Module lint supplies separate source, metadata, environment, and test checks. -A pipeline that the tools recognise as nf-core compliant follows the layout reproduced verbatim in `nf_core/pipeline-template/`. The canonical structure is: +[[component-nf-core-module-conventions]] gives the detailed module rules. Their enforcement level matters: a warning is weaker evidence than a passing required check, and a skipped check establishes nothing about its subject. -- `main.nf` — entrypoint; imports `workflows/.nf` and the boilerplate utility subworkflows (`utils_nfcore_pipeline`, `utils_nfschema_plugin`). -- `nextflow.config` — sets `manifest`, `params`, `profiles` (at minimum `test`, `test_full`, `docker`, `singularity`, `conda`, `apptainer`, `arm`), and includes the conf/ files. -- `nextflow_schema.json` — JSON Schema (Draft-07) for `params`, with nf-core extensions (see *Schema universe*). -- `workflows/.nf` — the pipeline DSL2 workflow definition. -- `modules/nf-core///` — vendored modules pulled from `nf-core/modules`. Each module has `main.nf`, `meta.yml`, `environment.yml`, and `tests/main.nf.test` plus snapshots. -- `modules/local/` — pipeline-specific modules. -- `subworkflows/nf-core//` and `subworkflows/local//` — same split for subworkflows. `meta.yml` for subworkflows declares `components:` (the modules/subworkflows it depends on). -- `conf/base.config`, `conf/modules.config`, `conf/test.config`, `conf/test_full.config` — required by the `included_configs` and `base_config` lint checks. -- `assets/` — schema_input.json (sample sheet schema), MultiQC config, email templates, logos. -- `bin/` — pipeline-shipped helper scripts placed on `PATH` for processes. -- `docs/` — `usage.md`, `output.md`, `images/`. -- `tests/` — top-level nf-test files; the lint check `nf_test_content` enforces presence and naming. -- `.nf-core.yml` — see below. -- `modules.json` — see below. -- `nf-test.config`, `tower.yml`, `CHANGELOG.md`, `CITATIONS.md`, `CODE_OF_CONDUCT.md`, `LICENSE`, `README.md` — all enforced by `files_exist`. +### `.nf-core.yml` configuration -### `.nf-core.yml` +`load_tools_config` loads this file into the Pydantic models in `nf_core/pydantic_models.py`. -Validated by `NFCoreYamlConfig` (Pydantic v2) in `nf_core/utils.py`. The recognised fields are: +| Field | Contract | +|---|---| +| `repository_type` | Optional `pipeline` or `modules`. Component commands use repository type to choose their behavior. | +| `nf_core_version` | Optional string recording the tools version used for template creation or synchronization. | +| `org_path` | Organization path for a modules repository. | +| `lint` | Known check settings. Accepted values vary by check, including booleans and lists of excluded paths or configuration keys. | +| `template` | Recorded creation answers, including name, description, author, version, organization, and `skip_features`. | +| `bump_version` | Per-component boolean settings for modules repositories. | +| `update` | Module/subworkflow update configuration, including exclusions and SHA settings. | +| `container-registry` | Additional container registry prefixes allowed during container linting. The Python attribute is `container_registry`. | -- `repository_type: "pipeline" | "modules"` — discriminates pipeline repos from module repos. `nf_core/components/components_command.py` switches behaviour on this. -- `nf_core_version` — version of the tools used to render or last sync the template. -- `org_path` — used in modules-type repos (e.g. `nf-core` or an institutional fork). -- `lint:` — `NFCoreYamlLintConfig`; per-check disable map. A check listed here is skipped or downgraded; the `files_exist` and `files_unchanged` checks accept lists of paths to ignore. -- `template:` — `NFCoreTemplateConfig`; captures the cookiecutter answers (`name`, `description`, `author`, `version`, skipped features). -- `bump_version:` — modules-repo only; per-component opt-out of `bump-versions`. -- `update:` — pipeline-repo only; per-module pinning that suppresses `nf-core modules update`. +The models validate declared field types but use Pydantic's default handling of extra fields: unknown keys are ignored. This is not a closed schema that rejects misspellings. The `nfcore_yml` pipeline lint check separately checks repository type and recorded tools version. It does not provide a general unknown-key check. -Unknown fields are tolerated only via `.get` because the model is defined with explicit attributes; at lint time mismatches are reported by `nfcore_yml`. +A check set to `false` is disabled. A list has check-specific meaning, illustrated by: -### `modules.json` +```yaml +lint: + files_exist: + - assets/multiqc_config.yml + files_unchanged: + - .github/CONTRIBUTING.md + nextflow_config: + - manifest.name +``` -Lives at the pipeline root. JSON object of the form: +These settings change lint coverage. They do not change Nextflow execution behavior. -``` +### `modules.json` provenance + +The root manifest records installed components by repository URL, component type, installation directory, and component name: + +```json { - "name": "", - "homePage": "...", + "name": "example", + "homePage": "https://example.org/example", "repos": { "https://github.com/nf-core/modules.git": { - "modules": {"nf-core": {"/": {"branch":"master","git_sha":"...","installed_by":["modules"]}}}, - "subworkflows": {"nf-core": {"": {"branch":"master","git_sha":"...","installed_by":["subworkflows"]}}} + "modules": { + "nf-core": { + "samtools/sort": { + "branch": "master", + "git_sha": "", + "installed_by": ["modules"] + } + } + } } } } ``` -`nf_core/modules/modules_json.py` reads, validates, and rewrites it. Every `install/update/remove/patch` command mutates this file. The `installed_by` field tracks whether a module was added directly or pulled in transitively as a subworkflow dependency — removing the parent removes the child only if no other parent remains. `git_sha` pins the commit in `nf-core/modules` from which the module's directory was copied; `nf-core modules update` diffs the working tree against that SHA and against the requested SHA. +`git_sha` identifies the upstream component revision copied into the pipeline. `branch` records its source branch. An optional `patch` records a local patch file. Installation, update, removal, and patch operations can rewrite the manifest or component files. `ModulesJson` performs consistency checks and can reconstruct missing provenance through repository comparisons. Reconstruction may require network access or user choices. -## `nf-core` CLI surface +`installed_by` is a list of strings. A directly installed module has the marker `modules`, and a directly installed subworkflow has `subworkflows`. Dependencies can instead carry parent subworkflow names. Removal drops the relevant parent entry and removes a dependency when no installation parents remain. -The CLI is defined in `nf_core/__main__.py` using `click` + `rich-click`. Top-level groups: `pipelines`, `modules`, `subworkflows`, `test-datasets`, `interface` (TUI via `trogon`). +The manifest is an inventory of managed, installed components. It is not a list of every executed process. It can include unused modules and excludes local modules that are not managed there. Includes, aliases, workflow calls, and conditions in `.nf` source establish how installed code participates in a pipeline. -### `nf-core pipelines` +For conversion provenance, preserve repository URL, component path, recorded SHA, and any patch or local changes. The containing pipeline's HEAD is not automatically the source module's upstream SHA. A standalone modules checkout has its own repository revision instead. -- `create` — Textual TUI (or `--template-yaml` for headless) that renders `nf_core/pipeline-template/` through Jinja using a `template_features.yml` answer set. Skipped features remove sections of the template. -- `lint` — runs the lint test battery (see *Linting*); supports `--release`, `--fix`, `--key`, `--show-passed`, `--fail-warned`, `--fail-ignored`, `--json `, `--markdown `, `--sort-by`. -- `download` — clones a pipeline at a revision, optionally fetches container images. Shells out to `nextflow inspect -format json` to enumerate processes and containers (see *Container resolution*). -- `list` — fetches `https://nf-co.re/pipelines.json`, joins it with local clone state, prints table or `--json`. -- `launch` — interactive parameter wizard against `nextflow_schema.json`; can post to the nf-co.re web GUI for collaborative editing then poll back. -- `create-params-file` — non-interactive; emits a YAML params file populated with schema defaults. -- `sync` — fetches the current template, re-renders against the recorded answers in `.nf-core.yml`, commits to a `TEMPLATE` branch, opens a PR back to `dev`. -- `bump-version` — rewrites `manifest.version` in `nextflow.config`, `nextflow_schema.json`, `CITATIONS.md`, etc. -- `create-logo` — produces nf-core-styled logos. -- `rocrate` — emits Research Object Crate metadata via `repo2rocrate`. -- `schema validate ` — validates a params file against `nextflow_schema.json`. -- `schema build` — interactive; opens the web schema builder, polls for the result, writes back. -- `schema lint` — schema-itself validation (Draft-07 + nf-core conventions). -- `schema docs` — generates Markdown documentation from the schema. +## CLI surfaces -### `nf-core modules` +The Click application is defined in `nf_core/__main__.py`. The top-level groups are `pipelines`, `modules`, `subworkflows`, and `test-datasets`. `interface` provides a Trogon UI over the same commands. -- `list remote` / `list local` — enumerate modules in a remote modules repo or the current pipeline's vendored set. -- `install `, `update `, `remove ` — package-manager operations against `modules.json`. -- `create /` — scaffolds a new module from `nf_core/module-template/`. -- `info ` — pretty-prints `meta.yml` for a remote or local module. -- `lint` — runs the module lint suite (see below). -- `patch ` — captures local diffs against the upstream module as a `.diff` file that survives `update`. -- `bump-versions` — bumps tool versions in `environment.yml` and the container directive. -- `test ` — runs the module's nf-test. +### Pipeline commands -### `nf-core subworkflows` +| Command under `nf-core pipelines` | Scope | +|---|---| +| `create` | Render the Jinja pipeline template. Supports an interactive creation UI and template YAML input. | +| `lint` | Check pipeline conventions. Supports check selection, release checks, fixes, JSON/Markdown reports, and options that make warnings or ignored checks fail. `--fix` can modify files. | +| `download` | Obtain pipeline code at selected revisions and optionally download Singularity, Apptainer, or Docker images. | +| `list` | Combine the nf-core remote pipeline registry with local clone information. Supports JSON output. | +| `launch` | Build launch parameters through schema-driven interaction, including web-based editing. | +| `create-params-file` | Generate a YAML parameter file from schema defaults, with prompting controlled by its options. | +| `sync` | Re-render the pipeline template from recorded answers. Its configured workflow can create template commits and pull requests. | +| `bump-version` | Update version-bearing pipeline metadata. | +| `create-logo`, `rocrate` | Generate logos and Research Object Crate metadata. | +| `schema validate` | Validate a parameter file against the selected pipeline schema. | +| `schema build` | Create or edit a schema, including interaction with the web schema builder. | +| `schema lint` | Validate schema structure and nf-core schema conventions. | +| `schema docs` | Generate Markdown or HTML parameter documentation. | -Same surface as modules (`create`, `install`, `update`, `remove`, `list`, `info`, `lint`, `patch`, `test`) backed by a shared `nf_core/components/` layer. A subworkflow's `meta.yml` declares `components: [, ]` which the install command resolves transitively. +### Module, subworkflow, and fixture commands -### `nf-core test-datasets` +`modules` and `subworkflows` share component-management machinery for `list remote`, `list local`, `info`, `install`, `update`, `remove`, `create`, `patch`, `lint`, and `test`. List commands support JSON output. Component tests invoke nf-test. Installation and updates resolve components from the selected modules repository, and subworkflow installation can resolve dependencies transitively. -- `search` — keyword search across branches of `nf-core/test-datasets`. -- `list` — list all data files for the current pipeline branch. -- `list-branches` — list branches (one per pipeline). +`modules bump-versions` updates environment and container versions. `modules containers` includes `create`, `conda-lock`, and `list` for container/environment management. The two component groups are similar but do not have identical command inventories. -### `nf-core interface` +`test-datasets search`, `list`, and `list-branches` expose the nf-core fixture repository. Fixture branches organize data for pipelines and shared test cases. A branch name is not an immutable data pin. -Trogon-rendered TUI wrapping the click app — useful for discovery, not a separate command surface. +Machine-readable output is command-specific. Pipeline listing and linting, and module/subworkflow listing, have JSON options. Human-oriented tables and prompts remain common. A consumer should check the selected command's output contract before parsing its terminal text. -A flag worth flagging: **almost no commands offer JSON output by default**. `pipelines list` and `pipelines lint` do (`--json`); the rest are human-oriented Rich tables and prompts. Programmatic consumers usually drop to the Python API. +## Python interfaces -## Python API +The implementation can be imported, but this review establishes contracts for the pinned release only. It does not establish a stable external Python API across releases. -The package exposes modules under `nf_core/` whose `__init__.py` files are deliberately minimal — most public API is reached by importing the leaf modules: +| Import | Relevant behavior | +|---|---| +| `nf_core.pipelines.schema.PipelineSchema` | Load schemas, validate parameters and schema structure, obtain defaults, and generate documentation. Plugin selection affects schema handling. | +| `nf_core.pipelines.lint.PipelineLint` | Run pipeline checks and assemble their results. | +| `nf_core.pipelines.list.Workflows` | Represent remote registry entries and local pipelines. | +| `nf_core.pipelines.download.DownloadWorkflow` | Coordinate code and container downloads. | +| `nf_core.pipelines.create.create.PipelineCreate` | Render pipeline templates. | +| `nf_core.modules.modules_json.ModulesJson` | Load, reconcile, and update managed-component provenance. | +| `nf_core.modules.modules_repo.ModulesRepo` | Resolve refs and component contents in a modules repository. | +| `nf_core.components.components_command.ComponentCommand` | Shared component-command handling for repository layout and component discovery. | +| `nf_core.utils.is_pipeline_directory` | Require `main.nf` and `nextflow.config`. Missing `.nf-core.yml` or `modules.json` does not itself fail this helper. | +| `nf_core.utils.fetch_wf_config` | Retrieve configuration through `nextflow config -o json`, then add statically detected `params.*` assignments from `main.nf`. | -- `nf_core.pipelines.schema.PipelineSchema` — `.load_schema()`, `.validate_params()`, `.validate_schema()`, `.get_schema_defaults()`, `.schema_to_markdown()`. The most stable internal interface; reused by `lint`, `launch`, `create-params-file`. -- `nf_core.pipelines.lint.PipelineLint` — registry of lint tests as instance methods named after the entries in `lint_tests`. The `_get_results_md()`, `_get_lint_results()` outputs include a `"nf_core_tools_version"` field, per-category test arrays (`tests_pass`, `tests_warned`, `tests_failed`, `tests_ignored`, `tests_fixed`), and counts. -- `nf_core.pipelines.list.Workflows` — wraps `https://nf-co.re/pipelines.json`; exposes `.remote_workflows` and `.local_workflows` lists each containing a `Pipeline` model. -- `nf_core.pipelines.download.DownloadWorkflow` — full download orchestrator, including container fetch. -- `nf_core.pipelines.create.create.PipelineCreate` — programmatic scaffolding. -- `nf_core.modules.modules_json.ModulesJson` — read/write the manifest. -- `nf_core.modules.modules_repo.ModulesRepo` — clone and resolve refs in a modules repository (default `https://github.com/nf-core/modules.git`, branch `master`, all overridable via env vars `NF_CORE_MODULES_REMOTE`, `NF_CORE_MODULES_NAME`, `NF_CORE_MODULES_DEFAULT_BRANCH`). -- `nf_core.components.components_command.ComponentCommand` — base class shared by modules and subworkflows operations; `.get_local_components()`, `.has_modules_file()`, `.check_modules_structure()`. -- `nf_core.utils` — `is_pipeline_directory`, `fetch_wf_config` (runs `nextflow config`), `load_tools_config` (returns a Pydantic `NFCoreYamlConfig`), `setup_requests_cachedir` (a 1-day `requests_cache` for GitHub calls), `GitHubAPISession` (rate-limit-aware), `anaconda_package`, `get_biocontainer_tag`, `determine_base_dir`, `is_file_binary`. +`ModulesJson.get_all_components("modules")` returns a dictionary keyed by repository URL, with lists of `(installation_directory, component_name)` tuples. It does not return one flat list of triples or process calls. -There is **no documented stable API contract**. The README and online docs cover the CLI; `nf_core.*` modules are imported by other tools (`nf-core/configs` scripts, nf-validation, internal nf-core webapps) but breaking changes happen across major releases. Type hints are mostly in place since 4.x. +`fetch_wf_config` takes a workflow path and an optional `cache_config` boolean, which defaults to `true`. It has no profile-selection argument and does not retain the full profile definitions. Its scraped `main.nf` parameter entries have `None` values, can overwrite values returned by Nextflow, and do not evaluate assignment expressions. Cache identity uses the root `nextflow.config` and `main.nf` contents. Changes only to included configuration or environment can therefore leave a cached result stale. -## The schema universe +Profile discovery and effective profile selection are separate operations. [[component-nextflow-inspect]] describes `nextflow config -show-profiles` and the configuration layers. Conventional test filenames alone are insufficient evidence of profile names or effective settings. -- `nextflow_schema.json` — JSON Schema Draft-07 for `params`. nf-core layers conventions on top: top-level `definitions` groups parameters into UI panels; per-property keywords `fa_icon` (FontAwesome), `hidden` (boolean, hides from launch UI), `help_text`, `mimetype` (for file params; checked by `check_for_input_mimetype`), and a custom `default` resolution that tolerates nulls and Nextflow-style closures. The schema is consumed by `pipelines launch`, `create-params-file`, the nf-co.re schema builder, the in-pipeline `nf-validation` / `nf-schema` Nextflow plugins, and Seqera Platform. -- `assets/schema_input.json` — JSON Schema for the sample-sheet CSV/TSV; consumed at runtime by the `nf-schema` plugin's `samplesheetToList` operator. -- `pipeline_template.yml` (in tools, `nf_core/pipelines/create/template_features.yml`) — describes the cookiecutter feature flags (`fastqc`, `multiqc`, `nf_schema`, `igenomes`, `email`, `slackreport`, `adaptivecard`, …). Each entry has `skippable_paths`, `forbidden_paths`, `nfcore_yml_skip_value`, etc. -- `.nf-core.yml` — covered above. Pydantic-validated; `additionalProperties` is effectively closed because Pydantic v2 with explicit fields ignores extras by default. -- `modules.json` — covered above. Validated by `nf_core/modules/modules_json.py` against an internal jsonschema. -- `meta.yml` (modules and subworkflows) — YAML; declares `name`, `description`, `keywords`, `tools` (each with version, license, doi, homepage, biocontainer/container hints), `input` and `output` channel specifications including types, patterns, and ontology terms (EDAM where present), and `authors`/`maintainers`. The `meta_yml` lint check validates against a JSON Schema bundled in the tools repo. **For introspection use cases, `meta.yml` is the most valuable single file**: it's the only declarative source of channel IO shapes per module. -- `nf-test.config` and per-module `tests/main.nf.test`, `tests/main.nf.test.snap` — nf-test's own format (Groovy DSL + JSON snapshots). The `nf_test_content` lint check parses these for required tags and the `setup` block that pulls test data from `nf-core/test-datasets`. +## Schemas and module metadata -## The downstream ecosystem +| Artifact | Validation and interpretation | +|---|---| +| `nextflow_schema.json` | The pinned `PipelineSchema` selects Draft 2020-12 and `$defs` for `nf-schema`, or Draft-07 and `definitions` for `nf-validation`. If neither plugin is found, it defaults to nf-schema notation. Schema lint checks the expected `$schema` URI. | +| `assets/schema_input.json` | Common sample-sheet schema consumed by nf-schema runtime handling. Its meaning comes from the schema and plugin, not the filename alone. | +| `nf_core/pipelines/create/template_features.yml` | Tools-owned feature metadata controlling generated template sections and lint expectations. It is not a pipeline parameter schema. | +| `.nf-core.yml` | Pydantic tools configuration, with unknown keys ignored as described above. | +| `modules.json` | Typed provenance structure maintained through procedural consistency checks. It is not a parameter schema or runtime process inventory. | +| Module `meta.yml` | Descriptions, keywords, tool records including citation fields, container metadata, channel IO, topic metadata, and authors/maintainers. Module lint uses the metadata schema from the selected modules repository. | +| Subworkflow `meta.yml` | Component dependencies and documented channel interfaces, with its own schema and structure. | +| `nf-test.config`, `*.nf.test`, `*.nf.test.snap` | Test configuration, Groovy test definitions, and snapshots. They establish declared cases and expectations, not results of a new execution. | -How resolution works at runtime: +Parameter schemas can include UI and plugin extensions such as `fa_icon`, `hidden`, `help_text`, and file-related annotations. A passing JSON Schema check does not type-check all uses of that parameter in workflow code or prove its scientific suitability. -- **Pipeline registry**: `https://nf-co.re/pipelines.json` is fetched by `nf_core.pipelines.list` (cached via `requests_cache`). Anonymous; no auth. -- **GitHub API**: `nf_core.utils.GitHubAPISession` wraps `requests_cache.CachedSession` with token discovery from `GITHUB_TOKEN` / `GITHUB_AUTH_TOKEN` and rate-limit retry. Used for branch/release enumeration (`get_repo_releases_branches`) and SHA resolution (`get_repo_commit`). -- **Modules repo**: `nf_core.modules.modules_repo.ModulesRepo` performs a real `git clone --no-checkout` of `https://github.com/nf-core/modules.git` (override via `NF_CORE_MODULES_REMOTE`) into the user's nf-core cache directory, then `git checkout` of specific component subtrees. -- **Test-datasets**: `nf_core.test_datasets.test_datasets_utils` calls `https://api.github.com/repos/nf-core/test-datasets/branches` for branch listing and `https://raw.githubusercontent.com/nf-core/test-datasets//` for content. The website also publishes `https://raw.githubusercontent.com/nf-core/website/refs/heads/main/public/pipeline_names.json` as the canonical list of pipeline-named branches. -- **Configs**: not directly fetched by the tools CLI; pipelines `includeConfig` from `https://raw.githubusercontent.com/nf-core/configs/master/...` at Nextflow runtime, gated by `params.custom_config_base`. The `configs` lint check verifies the include statement is present. +The pinned module template represents `input` as a list of channels, each containing a list of named tuple members. Its `output` is a mapping from emitted channel names to channel/tuple descriptions, and `topics` documents topic emissions. Older module metadata can use different shapes. Subworkflow metadata has a different input/output layout. Read the actual metadata schema and source revision instead of assuming both `input` and `output` are always lists of lists. -## Linting, in detail +`meta.yml` is useful declarative documentation, but it is hand-maintained. Reconcile it with `main.nf` for actual channel declarations, command behavior, output patterns, and cardinality. `environment.yml` provides package/channel specifications. Containers and package environments are separate evidence: a container tag alone does not enumerate every dependency or prove equivalence to a generated Galaxy environment. -The pipeline lint registry is the list `lint_tests` in `nf_core/pipelines/lint/__init__.py`: +## Lint results and their limits -`files_exist`, `nextflow_config`, `nf_test_content`, `files_unchanged`, `actions_nf_test`, `actions_awstest`, `actions_awsfulltest`, `readme`, `pipeline_todos`, `pipeline_if_empty_null`, `plugin_includes`, `pipeline_name_conventions`, `template_strings`, `schema_lint`, `schema_params`, `system_exit`, `schema_description`, `actions_schema_validation`, `merge_markers`, `modules_json`, `multiqc_config`, `modules_structure`, `local_component_structure`, `base_config`, `modules_config`, `nfcore_yml`, `rocrate_readme_sync`, `container_configs`. In `--release` mode `version_consistency` and `included_configs` are added. +Pipeline checks cover file inventory, template-file changes, Nextflow configuration, parameter/schema agreement, nf-test and CI conventions, managed components, documentation, code hygiene, and container configuration. Release mode adds version consistency and included-configuration checks. Module checks separately cover `main.nf`, metadata, environments, source differences, upstream versions, tests, TODOs, deprecations, and patches. -Categories: +`schema_params` compares flat configuration parameters with the schema, subject to plugin-specific ignored parameters. Complex parameter objects are outside that comparison. `modules_json` checks managed-component consistency. These checks do not reconstruct data flow or prove that every installed module is called. A clean lint result also does not execute the scientific commands or establish that a snapshot is meaningful. -- **Files exist / files unchanged**: `files_exist` checks the canonical inventory (over 60 paths). `files_unchanged` diffs template-shipped files against what `pipelines create` would render today, flagging local edits. -- **Schema**: `schema_lint` validates the JSON Schema itself; `schema_params` cross-checks every param declared in `nextflow.config` (via `fetch_wf_config`) against the schema, both directions. `schema_description` requires a description on every parameter. -- **Modules**: `modules_json` integrity (every directory under `modules/nf-core/` has an entry, every entry has a directory, SHAs are resolvable). `modules_structure` checks the `//{main.nf,meta.yml,environment.yml,tests/}` layout. `local_component_structure` does the same for `modules/local/`. -- **CI**: `actions_nf_test`, `actions_awstest`, `actions_awsfulltest`, `actions_schema_validation` parse `.github/workflows/*.yml` and assert presence of expected jobs. -- **Code hygiene**: `merge_markers`, `pipeline_todos` (TODO grep), `template_strings` (no leftover `{{ jinja }}`), `system_exit` (Groovy `System.exit` is forbidden — use `error()`). -- **Container/config**: `nextflow_config` requires a long list of manifest fields and `params.*` defaults; `container_configs` validates containers settings; `base_config`, `modules_config` check `conf/`; `multiqc_config` validates `assets/multiqc_config.yml`. +Pipeline `--json` reports contain: -The module lint suite (`nf_core/modules/lint/`): `main_nf` (process structure, container directive form, output channels), `meta_yml` (schema-validate the meta), `environment_yml` (conda channels, package pinning, name == module name), `module_changes` (working-tree diff against pinned SHA), `module_version` (compare with upstream `master`), `module_tests` (nf-test presence and `tags`), `module_todos`, `module_deprecations`, `module_patch` (patch file integrity). +- `nf_core_tools_version` and `date_run`. +- `tests_pass`, `tests_warned`, `tests_failed`, `tests_ignored`, and `tests_fixed`, each containing `[check_id, message]` entries. +- Corresponding `num_tests_*` counts and `has_tests_*` booleans. +- `markdown_result`, containing the rendered report. -`--json ` writes a structured report keyed by: +These are pipeline-check result fields. Module/subworkflow reporting has its own component result structure. Preserve the tools version and configured exclusions when using lint output as evidence, and distinguish passed checks from ignored or fixed checks. -``` -nf_core_tools_version, date_run, -tests_pass[], tests_warned[], tests_failed[], tests_ignored[], tests_fixed[], -num_tests_pass/warned/failed/ignored/fixed, -has_tests_pass/warned/failed/ignored/fixed (booleans), -markdown_result -``` +## Container discovery and downloads -Every entry is `[check_id, message]`. Disabling a check is done in `.nf-core.yml`: - -```yaml -lint: - files_exist: false # disable entirely - files_unchanged: # or pass arguments - - .github/CONTRIBUTING.md - nextflow_config: - - manifest.name -``` - -## Container resolution - -Modules declare containers via the canonical Groovy ternary: +A conventional module can select an engine-specific container: ```groovy container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -222,56 +221,28 @@ container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity 'biocontainers/fastqc:0.12.1--hdfd78af_0' }" ``` -For tools with multiple dependencies the **mulled-v2** convention is used: `https://depot.galaxyproject.org/singularity/mulled-v2-:-0`, where the hash is reproducible from the sorted package list (`galaxy-tool-util` provides the hash function; `nf-core/tools` accepts both pre-computed hashes and resolves them via Biocontainers/Quay). - -`nf-core pipelines download --container-system singularity --container-cache-utilisation amend` materialises images by: - -1. Cloning the pipeline at the requested revision. -2. Running `nextflow inspect -format json -profile ` per requested profile and collecting the `container` field of each process. The download module retries inspect with a synthetic `outdir` params file when the pipeline aborts on missing `outdir` only. -3. For each unique container URI, pulling via `singularity pull` / `apptainer pull` / `docker pull` into the `--singularity-cache-dir` (or NXF_SINGULARITY_CACHEDIR) layout that Nextflow itself expects. - -Wave / Seqera Containers fits as an alternative ternary branch and as the alternative registries `community.wave.seqera.io/library/...` (Docker) and `community-cr-prod.seqera.io/docker/registry/v2/...` (Singularity), encoded in `nf_core/pipelines/download/utils.py`. The download flow recognises both and pulls them by URL. - -## Practical patterns - -### Listing every module a pipeline uses - -The authoritative source is `modules.json`. Read `repos["https://github.com/nf-core/modules.git"]["modules"]["nf-core"]` keys and walk `modules/nf-core//` directories. The Python helper is: - -```python -from nf_core.modules.modules_json import ModulesJson -mj = ModulesJson(pipeline_dir) -mj.load() -modules = mj.get_all_components("modules") # -> list[(repo_url, install_dir, name)] -``` - -For the upstream version + SHA per module, the same JSON has `git_sha` and `branch` per entry. - -### Enumerating test profiles - -Profiles live in `nextflow.config` in a `profiles { ... }` block. Static parsing is brittle because Groovy allows arbitrary code; the canonical answer is `nf_core.utils.fetch_wf_config(path)` which shells out to `nextflow config -flat` and returns the resolved keys. Profile names are recoverable from `conf/test.config` and `conf/test_full.config` filenames (the convention enforced by `files_exist`). +Multi-package modules can use `mulled-v2` images. Wave/Seqera container references are another supported convention. [[component-nextflow-containers-and-envs]] covers these URI forms and the evidence needed to resolve package requirements. Process-selector overrides in pipeline configuration can replace a module's container directive. -### Every container image used by every process +The pinned download implementation requires Nextflow 25.04.4 or newer for inspection. It invokes `nextflow inspect -format json` on the selected entrypoint. The profile string starts with the selected container system and, when test-container inspection is enabled, appends `test,test_full` in the same invocation. It does not independently inspect every possible profile combination. -The only sound static answer is also the one `nf-core download` uses: `nextflow inspect -format json -profile main.nf`. This emits a JSON document with a `processes` array, each with a `container` field already resolved by Nextflow's interpolation, eliminating the ternary. `nf_core/pipelines/download/download.py::run_nextflow_inspect` wraps this. For static-only use, parsing `modules/*/*/main.nf` for `container` directives and then resolving the ternary against an assumed engine yields a reasonable approximation, but module overrides in `conf/modules.config` (`withName: { container = '...' }`) can still bypass it. +It gathers `container` values from the returned `processes` array and deduplicates them. Those values are task-preview references for loaded process definitions. They are not a complete inventory of runtime images for every input or branch. Dynamic expressions can fail, loaded unused processes can appear, and files that are never loaded are outside inspection's scope. [[component-nextflow-inspect]] gives the precise preview limits. -### IO shape per module +A failure identifying missing `outdir` triggers one retry with an ephemeral parameter file providing `outdir: "nf-core-tools-inspect"`. Other missing launch parameters can still fail. The implementation also recognizes the strict-syntax “Invalid process directive” error and reports a compatibility problem with older pipeline syntax. -`modules/nf-core///meta.yml`'s `input:` and `output:` sections are the declarative source. They are list-of-list structured, where each top-level entry corresponds to a positional channel and each nested entry is a tuple element with `type:`, `description:`, `pattern:`, and optionally `ontologies:`. +| Download system | Materialization behavior | +|---|---| +| `none` | Download pipeline code without container images. | +| `singularity`, `apptainer` | Fetch images through engine-aware handling of image URLs and Docker references. Cache options `amend`, `copy`, and `remote` affect storage/reuse. Singularity and Apptainer have separate Nextflow cache/library environment variables. | +| `docker` | Require Docker and an accessible daemon, pull images, and save archives under `docker-images`. Only `copy` is accepted when a cache-utilization mode is supplied for Docker. This does not place images in a Singularity cache. | -## Limitations and gaps for static introspection +Container fetching and code resolution can use remote services. Tags and branches remain mutable unless an immutable revision or image digest is recorded. Remote inputs, configuration includes, and other runtime dependencies can still require network access. Download success does not establish a fully offline run, compatibility with actual workflow inputs, or scientific correctness. -`nf-core/tools` is a **convention enforcer and package manager**, not a workflow analyser. It does not: +## Ecosystem and evidence boundaries -- Build a process or channel graph. There is no `nf_core` API that returns a DAG. `nextflow inspect` returns a flat process list, not edges. For graph-level insight the realistic paths are the Nextflow language server (`nextflow-io/language-server`), parsing the DSL2 AST yourself, or running `-with-dag` and parsing the generated DOT/HTML. -- Resolve channel topology between modules. `meta.yml` documents channel shapes per module but says nothing about how `workflows/.nf` wires them. -- Type-check parameters end-to-end (only the `params.*` declared in `nextflow_schema.json` are validated; runtime usage is unchecked). -- Provide a stable Python API contract — re-imports across major versions break. -- Cover non-nf-core pipelines. A pipeline that lacks `.nf-core.yml`, `modules.json`, or the canonical layout is rejected by `is_pipeline_directory` and most subcommands. -- Resolve container images without running Nextflow. The static ternary is engine-conditional and can be overridden in `conf/modules.config`. +The pipeline registry is published at `https://nf-co.re/pipelines.json`. Module management defaults to `https://github.com/nf-core/modules.git`, organization `nf-core`, and branch `master`, with overrides from `NF_CORE_MODULES_REMOTE`, `NF_CORE_MODULES_NAME`, and `NF_CORE_MODULES_DEFAULT_BRANCH`. GitHub access supports token discovery through `GITHUB_TOKEN` and `GITHUB_AUTH_TOKEN` and cached requests. Reproducible provenance requires the resolved commit, not just these defaults. -For workflow topology, the right tools are `nextflow inspect` (process list + containers + resolved configs), the Nextflow language server (AST), `nf-test` (reified IO at test time), and Seqera Platform / nf-tower (runtime DAG). +`nf-core/test-datasets` supplies shared fixtures. Pipelines commonly include institutional settings from `nf-core/configs` during Nextflow configuration evaluation. Neither repository's branch name makes the referenced content immutable. Network access, credentials, local cache state, environment, and selected revisions can affect tooling results. -## Open gaps +Non-nf-core pipelines are not uniformly rejected. The basic pipeline-directory helper only checks two files, and some tools operate on general Nextflow projects. Commands enforcing nf-core conventions need the relevant metadata and structure. A missing nf-core layout is a limitation for those commands, not proof that the workflow is invalid Nextflow. -_Updated when contact with real pipelines reveals an nf-core convention or tooling behaviour we hadn't accounted for._ +For a workflow summary, retain source-derived IO, includes and aliases, channel connections, conditional branches, commands, and test cases. For one-module conversion, retain the source pin, local patches, documented and declared IO, every relevant package specification, and fixture provenance. Lint, configuration dumps, metadata, and container previews each contribute evidence within their own scope. None substitutes for a runtime test or supplies the full workflow graph. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/SKILL.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/SKILL.md index 73e7aa2..b869a48 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/SKILL.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/SKILL.md @@ -15,7 +15,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `summary-cwl`. Schema: summary-cwl. Produced by `summarize-cwl`. Structured CWL summary emitted by summarize-cwl; consumed alongside the Galaxy interface brief. - Read artifact `cwl-galaxy-interface`. Produced by `cwl-summary-to-galaxy-interface`. Preceding Galaxy interface brief from cwl-summary-to-galaxy-interface that pins inputs, outputs, and labels. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_provenance.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_provenance.json index 4baebc1..5b5ece7 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_provenance.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/cwl-summary-to-galaxy-data-flow/index.md", "revision": 3, "content_hash": "ed3f1d14cab3958ec321bcc108dd6274bec965cd502763ce0d5abd060a90f28b", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "94ce4ea20107680b467d03e8889b63689774e325" }, - "cast_at": "2026-09-13T20:58:26.653Z", + "cast_at": "2026-11-10T10:05:12.328Z", "refs": [ { "kind": "pattern", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Ground collection reshape, relabel, cleanup, and map-over choices in corpus-observed Galaxy recipes.", "trigger": "When CWL scatter, arrays, nested arrays, records, or secondary-file contracts require explicit Galaxy collection operations.", - "src_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", - "dst_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", "source": "deterministic" }, { @@ -36,8 +36,8 @@ "evidence": "corpus-observed", "purpose": "Ground conditional-branch and optional-step choices in curated, corpus-observed Galaxy when/pick_value patterns.", "trigger": "When data-flow translation needs optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", - "src_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", - "dst_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", "source": "deterministic" }, { @@ -51,8 +51,8 @@ "evidence": "corpus-observed", "purpose": "Ground genomic-interval operation choices in curated, corpus-observed Galaxy interval recipes.", "trigger": "When the workflow operates on genomic intervals (BED/GFF/VCF coordinate features) and data-flow translation needs overlap, merge, coverage, windowing, masking, or set-algebra steps.", - "src_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", - "dst_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", "source": "deterministic" }, { @@ -147,8 +147,8 @@ "evidence": "hypothesis", "purpose": "Keep the data-flow brief separate from gxformat2 templating and concrete step implementation.", "verification": "Promote after two worked CWL-to-Galaxy translations preserve this Mold boundary without moving fields.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -177,8 +177,8 @@ "evidence": "hypothesis", "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's wiring and collection decisions settle, and append data-flow obligations the CWL summary can't settle — a scatter shape Galaxy can't express directly, an expression-derived value with no Galaxy equivalent, a `linkMerge`/`pickValue` fan-in with no settled Galaxy form.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -248,7 +248,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_verify.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_verify.json index e4c3c30..b33b8db 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_verify.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-cwl", + "--schema-sha256", + "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md index bd4ddc7..b00b1d5 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md @@ -13,9 +13,9 @@ tags: - topic/collection-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy collection transformation patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." related_notes: - "[[iwc-transformations-survey]]" - "[[iwc-conditionals-survey]]" @@ -25,10 +25,13 @@ related_patterns: - "[[reshape-relabel-remap-by-collection-axis]]" - "[[fan-in-bundle-consume-and-flatten]]" - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" - "[[sync-collections-by-identifier]]" - "[[harmonize-by-sortlist-from-identifiers]]" - "[[regex-relabel-via-tabular]]" - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" - "[[collection-swap-nesting-with-apply-rules]]" - "[[collection-split-identifier-via-rules]]" - "[[collection-build-list-paired-with-apply-rules]]" @@ -42,45 +45,46 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: collection patterns -This is the runtime-facing map for Galaxy collection transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. -## Cleanup +## Enter or leave a collection -- [[collection-cleanup-after-mapover-failure]] — use `__FILTER_EMPTY_DATASETS__` or `__FILTER_FAILED_DATASETS__` after map-over when empty or errored elements would break downstream steps. -- [[collection-unbox-singleton]] — use `__EXTRACT_DATASET__` with `which: first` when a known one-element collection must become a dataset. +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. -## Map-over Lifecycle Recipes +## Keep mapped siblings aligned -- [[manifest-to-mapped-collection-lifecycle]] — build collection elements from manifest/table rows, map a tool over them, then relabel or reshape outputs. -- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped outputs, sync siblings from surviving identifiers, then gate final reports. -- [[reshape-relabel-remap-by-collection-axis]] — correct the mapped axis after domain fan-out using Apply Rules and deterministic relabeling. -- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection consumer, then flatten or aggregate results. +First identify what changed. Membership, order, and labels are separate decisions. -## Identifiers +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. -- [[sync-collections-by-identifier]] — membership sync: extract identifiers from one collection and filter or relabel a sibling collection. -- [[harmonize-by-sortlist-from-identifiers]] — order sync: sort one sibling collection by another collection's identifier order. -- [[regex-relabel-via-tabular]] — label rewrite: derive new element identifiers in tabular form and apply them with `__RELABEL_FROM_FILE__`. -- [[relabel-via-rules-and-find-replace]] — relabel inside a structural reshape, currently grounded in the influenza fan-out pattern. +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. -## Structural Reshape +## Change the collection axes -- [[collection-flatten-after-fanout]] — collapse nested collection output to a flat list when the outer axis no longer matters. -- [[collection-build-named-bundle]] — assemble individual outputs into a named collection bundle for publishing or downstream fan-in. -- [[collection-swap-nesting-with-apply-rules]] — use Apply Rules to swap `list:list` axes. -- [[collection-split-identifier-via-rules]] — use Apply Rules regex columns to derive nested list identifiers from one identifier string. -- [[collection-build-list-paired-with-apply-rules]] — use Apply Rules to promote identifier columns into `list:paired` structure. +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. -## Bridges +## Follow a map-over lifecycle -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. ## See also diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md index 23fae29..2dfe890 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md @@ -12,9 +12,9 @@ tags: - topic/galaxy-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy when and pick_value conditional patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." related_notes: - "[[iwc-conditionals-survey]]" related_patterns: @@ -30,29 +30,33 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: conditionals patterns -This is the runtime-facing map for Galaxy conditional workflow choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. -## Direct Gates +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. -- [[conditional-run-optional-step]] — expose or derive a boolean, connect it as `inputs.when`, and use `when: $(inputs.when)` to skip optional steps. -- [[conditional-gate-on-nonempty-result]] — compute a boolean from empty/non-empty dataset or collection state before gating downstream reporting/export. The MGnify recipe is corpus-backed but clunky pending verified-pattern workflow work. +## Skip work without replacing its output -## Routes and Fallbacks +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. -- [[conditional-route-between-alternative-outputs]] — run one of several `when`-gated alternatives, then merge possible outputs with `pick_value`. -- [[conditional-transform-or-pass-through]] — optionally transform a value, then use `pick_value` to choose transformed output or original fallback. +## Continue with one value -## Not Conditionals +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. -- [[collection-cleanup-after-mapover-failure]] — use collection filters when empty or failed mapped elements should be dropped or replaced. This is collection-state cleanup, not a `when` gate. -- `__FILTER_NULL__` — catalog capability for null outputs after conditional steps, but no IWC-backed pattern candidate in the conditionals survey. Zero uptake is not itself an anti-pattern call. +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. ## See Also - [[iwc-conditionals-survey]] — conditionals survey and evidence trail. -- [[galaxy-collection-patterns]] — companion MOC for collection transforms and cleanup. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md index 8123493..de9ec07 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md @@ -13,8 +13,8 @@ tags: - topic/interval-transform status: draft created: 2026-06-10 -revised: 2026-08-03 -revision: 2 +revised: 2026-09-23 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." related_notes: - "[[iwc-interval-operations-survey]]" @@ -34,6 +34,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/SKILL.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/SKILL.md index 7c4a383..23df28e 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/SKILL.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/SKILL.md @@ -14,7 +14,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs - Read artifact `summary-cwl`. Schema: summary-cwl. Produced by `summarize-cwl`. Structured CWL summary emitted by summarize-cwl; the source-of-truth JSON for Galaxy interface choices. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_provenance.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_provenance.json index d22dff5..40f6fc8 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_provenance.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/cwl-summary-to-galaxy-interface/index.md", "revision": 3, "content_hash": "9e6b0aaea5d1662a0515cd932352e418d1539da95ed81b4e8193ce0c98d48640", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "94ce4ea20107680b467d03e8889b63689774e325" }, - "cast_at": "2026-09-13T20:58:26.857Z", + "cast_at": "2026-11-10T10:05:12.343Z", "refs": [ { "kind": "research", @@ -103,8 +103,8 @@ "evidence": "corpus-observed", "purpose": "Choose stable Galaxy workflow input/output labels and promoted checkpoint outputs.", "trigger": "When deciding labels, public outputs, checkpoint outputs, or fixture-compatible collection inputs.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -118,8 +118,8 @@ "evidence": "hypothesis", "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's input, output, and label decisions settle, and append interface obligations the CWL summary leaves open — an unmapped secondaryFiles bundle, an unpinned parameter default, a collection shape the source types don't determine.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -182,7 +182,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_verify.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_verify.json index e4c3c30..b33b8db 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_verify.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-cwl", + "--schema-sha256", + "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/SKILL.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/SKILL.md index 588c42d..b31fab8 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/SKILL.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/SKILL.md @@ -18,7 +18,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `cwl-galaxy-data-flow`. Produced by `cwl-summary-to-galaxy-data-flow`. Galaxy data-flow brief from cwl-summary-to-galaxy-data-flow that pins abstract operations and collection choices. - Read artifact `iwc-comparison-notes`. Produced by `compare-against-iwc-exemplar`. Structural diff guidance from compare-against-iwc-exemplar (run on the design briefs); steers the skeleton toward IWC-aligned structure before per-step authoring. Carries an inline gxformat2 excerpt of the nearest exemplar. - Read artifact `iwc-exemplar-gxformat2`. Produced by `compare-against-iwc-exemplar`. Cleaned gxformat2 view of the nearest IWC exemplar's relevant subgraph from compare-against-iwc-exemplar; pattern-match the draft's input/collection shapes, map-over wiring, output promotion, and post-job actions against this concrete idiom. Absent when no nearest exemplar was found. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -27,8 +27,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli @@ -45,6 +45,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load On Demand +- `references/cli/draft-next-step.json`: CLI command reference packaged as a sidecar. Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop. Use when: deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step. - `references/cli/draft-validate.json`: CLI command reference packaged as a sidecar. Validate the emitted draft against draft-contract rules (sentinel form, topology, _plan_* placement) before handing off. Use when: after writing or modifying the draft workflow file. - `references/patterns/galaxy-collection-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded collection pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges. - `references/patterns/galaxy-conditionals-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded conditional pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_provenance.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_provenance.json index f33cef3..2e55cc3 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_provenance.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_provenance.json @@ -4,12 +4,28 @@ "mold": { "name": "cwl-summary-to-galaxy-template", "path": "content/molds/cwl-summary-to-galaxy-template/index.md", - "revision": 5, - "content_hash": "a3dabae5502a3ebe29c6de4f0a07d1359c9c3166b96e7ea2d02cc8c042bb7892", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 6, + "content_hash": "b14eef623c16859a525a138e7fd4beddbb5cf8568ab9f4c1cdb9b12fc7803a44", + "commit": "db8801f2f0a31746e35cf399dd96cdbb8aa36dcf" }, - "cast_at": "2026-09-13T20:58:26.922Z", + "cast_at": "2026-11-10T10:22:51.464Z", "refs": [ + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[draft-next-step]]", + "src": "content/cli/gxwf/draft-next-step.md", + "dst": "references/cli/draft-next-step.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop.", + "trigger": "When deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step.", + "verification": "Cast the skill, emit a draft with a drafty step inside a subworkflow, confirm draft-next-step returns that step's full path.", + "src_hash": "d4fe7945945a57846fe8b168e7bacba04f739de1d86bc613ae6f0e5879ac2f55", + "dst_hash": "8638645f5974d8072dc3c63e522b7ab25d562f8d6217ea93b61f0a29f371d6f7", + "source": "deterministic" + }, { "kind": "cli-command", "mode": "sidecar", @@ -37,8 +53,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded collection pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges.", - "src_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", - "dst_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", "source": "deterministic" }, { @@ -52,8 +68,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded conditional pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", - "src_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", - "dst_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", "source": "deterministic" }, { @@ -67,8 +83,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded genomic-interval pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for interval overlap, merge, coverage, windowing, masking, or set-algebra on coordinate features.", - "src_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", - "dst_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", "source": "deterministic" }, { @@ -82,8 +98,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded tabular pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for tabular filtering, projection, joins, aggregation, text-processing recipes, or tabular-collection bridges.", - "src_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", - "dst_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", "source": "deterministic" }, { @@ -150,8 +166,8 @@ "purpose": "Respect the handoff from abstract data-flow draft to gxformat2 skeleton.", "trigger": "When translating abstract nodes, unresolved tool needs, and placeholder transformations into template TODOs.", "verification": "Promote after two worked CWL-to-Galaxy templates preserve the data-flow/template split without schema changes.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -165,8 +181,8 @@ "evidence": "corpus-observed", "purpose": "Group the settled step set into titled stage frames (the gxformat2 `comments:` array) so the skeleton carries the analysis-stage narrative IWC authors annotate by hand. Schema-legal and optional.", "trigger": "After topology is settled and the skeleton can be partitioned into named analysis stages (inputs, per-stage step clusters, parameter-derivation knots, visualization/outputs).", - "src_hash": "a206d5e08a8caaeaa42d9eadcd78eb0e2724c5756be681ca4b543cf0f7269176", - "dst_hash": "a206d5e08a8caaeaa42d9eadcd78eb0e2724c5756be681ca4b543cf0f7269176", + "src_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", + "dst_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", "source": "deterministic" }, { @@ -180,8 +196,8 @@ "evidence": "hypothesis", "purpose": "Emit the gxformat2 draft superset: TODO tool_id, optional state / tool_shed_repository, and per-step _plan_state / _plan_context planning fields.", "verification": "Promote after a downstream per-step implementation Mold consumes _plan_state and _plan_context without round-tripping back through the source summary.", - "src_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", - "dst_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", + "src_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", + "dst_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", "source": "deterministic" }, { @@ -195,8 +211,8 @@ "evidence": "corpus-observed", "purpose": "Choose stable workflow input/output labels, testable checkpoint outputs, and fixture-compatible workflow interfaces while drafting the skeleton.", "trigger": "When the template decides workflow inputs, workflow outputs, promoted checkpoints, or collection output identifiers that future tests will need to address.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -211,8 +227,8 @@ "purpose": "Use the gxformat2 structural vocabulary for workflow inputs, outputs, steps, and placeholder wiring.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "4fe42382f99655115c53c1afde0da0d0556ad77984da7d0ffa8ab725ea260aa3", - "dst_hash": "4fe42382f99655115c53c1afde0da0d0556ad77984da7d0ffa8ab725ea260aa3", + "src_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", + "dst_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" }, @@ -245,8 +261,8 @@ "evidence": "hypothesis", "purpose": "Close the open entries the settled topology discharges, and append a blocking entry for any settled step whose declared output no wired input can supply — the computability gap gxwf validation cannot see, raised here rather than left for the per-step loop to hit.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -261,8 +277,8 @@ "purpose": "Output contract: the emitted gxformat2 draft conforms to [[galaxy-workflow-draft]]. Cast bundles the JSON Schema so the skill carries its output shape alongside the [[draft-validate]] CLI checks.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", - "dst_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" }, @@ -355,7 +371,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_required_tools.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_required_tools.json index 54fd1c9..1a8a44d 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_required_tools.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_required_tools.json @@ -3,13 +3,14 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": [ + "gxwf draft-next-step", "gxwf draft-validate" ] } diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_verify.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_verify.json index e4c3c30..b33b8db 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_verify.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-cwl", + "--schema-sha256", + "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/cli/draft-next-step.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/cli/draft-next-step.json new file mode 100644 index 0000000..296abe7 --- /dev/null +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/cli/draft-next-step.json @@ -0,0 +1,42 @@ +{ + "type": "cli-command", + "tool": "gxwf", + "command": "draft-next-step", + "summary": "Pick the next drafty step a harness should work on, or report no remaining work; deterministic topological + alphabetical tiebreak.", + "source_path": "content/cli/gxwf/draft-next-step.md", + "source_revision": 1, + "package": "@galaxy-tool-util/cli", + "description": "Pick the next step a downstream agent should work on (or report no remaining work)", + "synopsis": "gxwf draft-next-step [options] ", + "args": [ + { + "raw": "file", + "name": "file", + "required": true, + "variadic": false, + "description": "Draft workflow file (.gxwf.yml)" + } + ], + "options": [ + { + "flags": "--format ", + "name": "format", + "description": "Force input format: format2 (default; native is rejected)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--output-format ", + "name": "outputFormat", + "description": "Output format: json (default) or markdown", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false, + "defaultValue": "json" + } + ], + "body": "# `gxwf draft-next-step`\n\nDeterministic next-step picker for the per-step authoring loop. Wraps `nextDraftStep` from `@galaxy-tool-util/schema`. Native (.ga) input is rejected — drafts are format2-only.\n\nThe harness loop reads as: `while (next = gxwf draft-next-step ).draft: invoke per-step skill on next.step`.\n\n## Output\n\nDefault output is pretty JSON — the agent-loop wire format. Same input → byte-identical output. `--output-format markdown` renders a human-glance checklist instead.\n\nJSON fields:\n\n- `draft` (bool) — `true` while a drafty step remains, `false` at the terminal cases below.\n- `step` (string array) — the picked step's **path**: a single id at top level (`[\"merge assembled transcripts\"]`), or the full path through containing subworkflow steps when the pick is nested. The leaf (last element) is the step to work on.\n- `work` (string array) — the chosen step's remaining `TODO[...]` sentinels followed by its `_plan_*` fields, in checklist order.\n\nAt the terminal cases the object is just `{ \"draft\": false }` — no `step` or `work` keys.\n\nTwo terminal cases both surface as `draft: false`:\n\n- The workflow's class is already `GalaxyWorkflow` (concretization complete, class promoted).\n- The class is still `GalaxyWorkflowDraft` but every step is fully resolved (no `TODO_*`, no `_plan_*`). The harness's next move is [[draft-extract]] (which calls `promoteFullyConcreteDrafts` to flip the class) followed by terminal [[validate]].\n\n## Examples\n\n```bash\ngxwf draft-next-step workflow.gxwf.yml\ngxwf draft-next-step workflow.gxwf.yml --output-format markdown\n```\n\nHarness loop sketch:\n\n```bash\nwhile [ \"$(gxwf draft-next-step workflow.gxwf.yml | jq -r .draft)\" = \"true\" ]; do\n step=$(gxwf draft-next-step workflow.gxwf.yml | jq -r '.step | last')\n # invoke per-step skill on step, which mutates workflow.gxwf.yml in place\ndone\n```\n\n## Gotchas\n\n- Pick order is **topological** (a step waits until its `in:` sources are concretized) with **alphabetical tiebreak** on step id at the same level. Same input across runs returns the same pick — safe to call repeatedly without state.\n- Descends into draft subworkflows; the returned `step` array is the full path through containing subworkflow steps (leaf last).\n- A `draft: false` result does **not** mean the workflow has passed [[validate]] yet. It means the per-step authoring loop is done; the harness still owns the promote + terminal-validate pass.\n- This command does not mutate the workflow file. It's a pure observer.\n- Insertion order of step dicts does not affect output — semantically equal workflows produce identical picks." +} diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md index efd7c87..9256dbf 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md @@ -4,12 +4,13 @@ tags: - target/galaxy status: draft created: 2026-06-12 -revised: 2026-06-12 -revision: 1 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-comments-survey]]" - "[[gxformat2-schema]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" summary: "How to annotate a gxformat2 workflow with editor comments: one titled frame per analysis stage, populate contains_steps, color decorative." diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md index 2a66f1a..9fc6e51 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-06 -revised: 2026-05-10 -revision: 2 +revised: 2026-11-10 +revision: 6 related_notes: - "[[gxformat2-schema]]" - "[[galaxy-data-flow-draft-contract]]" @@ -17,6 +17,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[implement-galaxy-tool-step]]" - "[[advance-galaxy-draft-step]]" @@ -117,14 +118,14 @@ steps: ## Topology repair escalation -The template Mold settles topology, but it can miss a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. +The template Mold settles topology, but a settled draft can still be unable to give a correct run. The common case is a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. When one surfaces, the per-step loop does not author topology to fix it. It **escalates**: -- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. -- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and re-wires the affected region: one producer step, a small sub-path, or honest narrowing of the output. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. +- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs, or that the chosen wrapper can't produce it. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. Earlier Molds and the loop raise blocking entries too. +- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and makes the smallest structural edits that close them. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. -This is **repair**, distinct from **settling**: it is narrow (one named region), reactive (only a detected computability gap triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each escalation must strictly reduce that count, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. +This is **repair**, distinct from **settling**: it is narrow (the entries bound it), reactive (an open blocking entry triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each repair must leave fewer than it found, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. ## Open work diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/gxformat2-schema.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/gxformat2-schema.md index 7682a33..af5053a 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/gxformat2-schema.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/gxformat2-schema.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-05 -revised: 2026-05-06 -revision: 2 +revised: 2026-11-06 +revision: 5 license: MIT license_file: LICENSES/galaxy-tool-util-ts.LICENSE related_notes: @@ -17,69 +17,58 @@ related_notes: - "[[galaxy-workflow-draft]]" sources: - "https://github.com/jmchilton/galaxy-tool-util-ts/blob/7ae4ecd0ba8d492225f58a6d455c4cc5317298f0/packages/schema/src/galaxy-workflow.ts" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py" companions: - "gxformat2.schema.json" -summary: "Vendored structural JSON Schema for gxformat2 workflows: vocabulary for inputs, outputs, steps, and step subtypes." +summary: "Vendored gxformat2 structural schema, its input, output, and step vocabulary, and the limits of schema-only validation." --- -> **Vendored from upstream**, pinned at SHA `7ae4ecd`. One file lives next to this note, declared in `companions:` so casting carries it into every bundle this note reaches: -> -> - `gxformat2.schema.json` — Draft-07 JSON Schema generated from the `@galaxy-tool-util/schema` Effect-TS schema via `gxwf structural-schema --format format2`. **Agents and casting should consume this** when validating gxformat2 workflows or reasoning about the closed vocabulary of input/step types. -> -> **Re-sync:** `pnpm sync:vendored --update`. Sync builds galaxy-tool-util-ts and re-runs the generator before copy. +# gxformat2 structural schema -## Top-level shape +The companion `gxformat2.schema.json` is a vendored Draft-07 JSON Schema generated from the `@galaxy-tool-util/schema` Effect-TS model at `7ae4ecd`. Casting carries it with this note. Use it to inspect the structural vocabulary of a gxformat2 workflow, then check connections, tool references, and runtime behavior with gxwf and Galaxy validation. The schema does not establish that an `outputSource` exists or that a tool state is valid for its wrapper. -A gxformat2 document is an object with required `inputs`, `outputs`, `class` (`enum: ["GalaxyWorkflow"]`), and `steps`. Optional: `id`, `label`, `doc`, `uuid`, `report`, `tags`, `comments`, `creator`, `license`, `release`. +The schema is regenerated by `pnpm sync:vendored --update`, which builds the upstream package and runs `gxwf structural-schema --format format2` before copying it here. Do not edit the companion by hand. -## Workflow input vocabulary +## Root and input fields -Each entry under `inputs` carries: +The root requires `class: GalaxyWorkflow`, `inputs`, `outputs`, and `steps`. Those sections accept arrays or keyed maps. The root also permits `id`, `label`, `doc`, `uuid`, `report`, `tags`, `comments`, `creator`, `license`, and `release`. -- `type` — closed enum: `null | boolean | int | long | float | double | string | integer | text | File | data | collection`. Drives whether the input is a parameter, a single dataset, or a collection. -- `format` — Galaxy datatype extension when `type ∈ {File, data}`. See [[galaxy-datatypes-conf]] for valid values. -- `collection_type` — free-form string (e.g. `list`, `paired`, `list:paired`, `list:list`) when `type = collection`. Closed value set lives in [[galaxy-collection-semantics]], not in this schema. -- `optional` — boolean. -- `default` — opaque (`/schemas/unknown`); type-dependent. -- `label`, `doc`, `id`, `position` — metadata. +Input entries describe the public workflow interface. The array form exposes these fields: -## Workflow output vocabulary +| Field | Structural schema | Meaning and use | +|---|---|---| +| `id`, `label`, `doc` | Optional | Identify or describe the input. Check the effective imported name before using it as a test `job:` key. | +| `type` | Optional string, string array, or `null` | Enum members are `null`, `boolean`, `int`, `long`, `float`, `double`, `string`, `integer`, `text`, `File`, `data`, and `collection`. This is accepted vocabulary, not a list of preferred authoring spellings. | +| `format` | Optional string array or `null` | Galaxy datatype extension filter. See [[galaxy-datatypes-conf]]. | +| `collection_type` | Optional string or `null` | Collection shape such as `list:paired`. The schema does not enumerate valid shapes. See [[galaxy-collection-semantics]]. | +| `optional`, `default` | Optional boolean or `null`, and opaque value, respectively | Control whether an input can be omitted and what value is supplied when absent. See [[gxformat2-workflow-inputs]] for runtime behavior. | +| `position` | Optional `{top, left}` numeric coordinates or `null` | Workflow editor placement. | -Top-level `outputs` can be an array or object map. Each output entry carries: +The generated schema does not include all fields accepted by current gxformat2 input handling, including `restrictions`, `suggestions`, and `restrictOnConnections`. [[gxformat2-workflow-inputs]] distinguishes that implementation behavior from what this vendored schema can check. -- `label` — public workflow-output name; tests and users should address this stable label. -- `outputSource` — producing step output reference. -- `doc` — optional prose context, string or string array. -- `id` — optional identifier. -- `type` — same permissive type vocabulary as inputs, when needed. +## Output names and connections -Producer-side output actions are separate from top-level `outputs`. Step `out:` entries can carry `change_datatype`, `rename`, `add_tags`, `remove_tags`, `hide`, `delete_intermediate_datasets`, and `set_columns`, or a plain string output name. See [[galaxy-workflow-testability-design]] for authoring posture: `label` is the public API; producer-side actions make the dataset useful. +An output entry can carry `id`, `label`, `doc`, `outputSource`, and `type`. `outputSource` names the producing step output, written `step/output`. The output's **public name** is not always its `label`: normalized gxformat2 export places a native Galaxy workflow-output label in `id`, and the pinned IWC [SRA manifest workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs.ga) exports `id: paired_output`. Current gxformat2 import chooses `label` when present and otherwise `id`. Check the effective name after import or conversion before writing the sibling test's `outputs:` key. See [[galaxy-workflow-testability-design]]. -## Workflow step vocabulary +The schema makes `outputSource` optional, even though a usable produced output needs a valid source. Its string type cannot prove that the referenced step and port exist. gxformat2 lint checks the step reference. Workflow validation must also check that the producing port exists. -Each entry under `steps` (`WorkflowStepSchema`) carries `type` ∈ `{tool, subworkflow, pause, pick_value}`, plus `tool_id`, `tool_shed_repository`, `tool_version`, `in`, `out`, `state`, `tool_state`, `run`, `runtime_inputs`, `when`, `errors`, `uuid`, `label`, `doc`, `position`, `id`. +The step part of a `source` or `outputSource` depends on how the workflow lists its steps. In list form it is the step's `label` when one is present, otherwise its `id`. In map form, for a step with no `label`, it is the map key. -### `state` vs `tool_state` +A step's `in:` key addresses a parameter nested in a conditional or section with a `|`-joined path, and a repeat instance with `_N`. For example, `input_type|i` wires an input inside a conditional in `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences`, and `style_cond|type_cond|pick_from_0|value` wires the first instance of a repeat inside two conditionals in [[conditional-route-between-alternative-outputs]]. The key names the path, and the conditional's selector value stays in `state`. Each `_N` connection also needs its own entry in the repeat in `state`. The repeat creates the input terminal, so without the entry the connection does nothing and the job can still succeed. -Both hold the same thing — a map from tool parameter names to values — and a step should -carry only one. Which one marks how the workflow was produced: +Step `out:` entries serve a different purpose. They can be output names or records with `id` and actions such as `rename`, `change_datatype`, `add_tags`, `remove_tags`, `hide`, `delete_intermediate_datasets`, and `set_columns`. These actions affect produced datasets or history presentation. They do not supply a top-level output's public name. -- **`state` is the authoring key.** A typed map, with conditionals and repeats written as - nested YAML. Molds that *author* a workflow or draft write `state`. Upstream agrees: of - the twelve gxformat2 format2 examples that carry step state, ten use `state`, and the - schema's own field docs say `tool_state` "may be present in workflows exported from Galaxy - but shouldn't be written by humans." -- **`tool_state` is the export key.** `gxwf convert --to format2` emits it for every tool - step, so any format2 view of a `.ga` workflow — the IWC corpus included — arrives carrying - `tool_state`. Molds that *read*, summarize, or preserve an existing workflow keep the key - they were given rather than rewriting it. +A `rename` value is not a tool-XML label template. Galaxy's [`RenameDatasetAction`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py#L152-L256) fills in two forms. `#{input}` becomes the name of the dataset bound to that tool input. Use the tool's input name, with `.` for a nested one (e.g. `queries_0.input2`), and optional `| basename`, `| upper` or `| lower`. `${label}` becomes the value of the workflow parameter input with that label. Write `#{gtf_to_extend} BREW3R extended` to name an output after its input. A `${name}` with no matching parameter input is an informal, untyped parameter ([`get_informal_replacement_parameters`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py#L3094-L3103)). The run form asks for a free-text value, and an API or Planemo run that supplies no `replacement_params` leaves `${name}` literal in the dataset name. The tool-XML variables `${on_string}` and `${tool.name}` work only in a wrapper's ``; in a rename they fall into this case. This is what the workflow linters report as an "untyped parameter"; declare a parameter input instead. -Nothing enforces this. Galaxy and `gxwf` both accept a nested map under either key, so a -hand-authored `tool_state` validates, converts, and runs; no gate reports the mismatch. It is -a convention that keeps authored and exported workflows distinguishable on sight. +## Step fields and state -## Caveats +The structural step `type` enum is `tool`, `subworkflow`, `pause`, and `pick_value`, with `null` also admitted. gxformat2's source schema defaults an omitted type to `tool`. A step can also carry `id`, `label`, `doc`, `position`, `tool_id`, `tool_shed_repository`, `tool_version`, `in`, `out`, `state`, `tool_state`, `run`, `runtime_inputs`, `when`, `errors`, and `uuid`. The structural schema does not require subtype-specific fields such as `tool_id` on a tool step, so validation must continue beyond JSON Schema. -- **Doc strings dropped.** Effect-TS `JSONSchema.make` does not preserve description fields. Narrative grounding for what each input/step type means lives in this note and in upstream gxformat2 SALAD (`$GXFORMAT2/schema/v19_09/workflow.yml`). -- **Repeated `$id: "/schemas/unknown"` markers.** The `Schema.Unknown` placeholder for opaque values (such as `default`) emits a duplicate `$id` 15× in the gxformat2 schema. Strict Ajv refuses to compile; pass `{ strict: false }` or strip the duplicates before compile. -- **`collection_type` is not a closed enum.** The schema accepts any string; valid composite shapes are pinned in [[galaxy-collection-semantics]]. +For hand-authored tool steps, use `state` for typed nested YAML, including conditionals and repeats. A converted workflow may contain `tool_state`, as the pinned IWC SRA manifest example does. Treat that as the representation received, not an authoring recommendation. Current import accepts `tool_state` as a map or JSON string, but gives `state` precedence if both fields are present. The structural schema permits both, so use only one on a step. Preserve an exported representation when reading or carrying it forward, and use `state` when authoring a new step. + +## What schema validation leaves open + +The root and array-form entries constrain their listed properties, but generated map-form `inputs` and `outputs` each include a permissive object branch. Passing this schema therefore does not prove that every map entry has only documented fields. In map form, the `type` enum and `format` shape can also escape the documented constraints. Every form still needs semantic checks for connections and tool configuration. + +The generated JSON also omits field descriptions and repeats `$id: "/schemas/unknown"` on opaque values. Some strict Ajv configurations reject those duplicate identifiers. If that occurs, compile with the project's supported non-strict configuration or remove duplicate markers in a validation copy. Keep the vendored companion unchanged. The upstream [gxformat2 source schema](https://github.com/galaxyproject/gxformat2/blob/main/schema/v19_09/workflow.yml) supplies field documentation that this generated artifact lacks. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md index bd4ddc7..b00b1d5 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md @@ -13,9 +13,9 @@ tags: - topic/collection-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy collection transformation patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." related_notes: - "[[iwc-transformations-survey]]" - "[[iwc-conditionals-survey]]" @@ -25,10 +25,13 @@ related_patterns: - "[[reshape-relabel-remap-by-collection-axis]]" - "[[fan-in-bundle-consume-and-flatten]]" - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" - "[[sync-collections-by-identifier]]" - "[[harmonize-by-sortlist-from-identifiers]]" - "[[regex-relabel-via-tabular]]" - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" - "[[collection-swap-nesting-with-apply-rules]]" - "[[collection-split-identifier-via-rules]]" - "[[collection-build-list-paired-with-apply-rules]]" @@ -42,45 +45,46 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: collection patterns -This is the runtime-facing map for Galaxy collection transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. -## Cleanup +## Enter or leave a collection -- [[collection-cleanup-after-mapover-failure]] — use `__FILTER_EMPTY_DATASETS__` or `__FILTER_FAILED_DATASETS__` after map-over when empty or errored elements would break downstream steps. -- [[collection-unbox-singleton]] — use `__EXTRACT_DATASET__` with `which: first` when a known one-element collection must become a dataset. +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. -## Map-over Lifecycle Recipes +## Keep mapped siblings aligned -- [[manifest-to-mapped-collection-lifecycle]] — build collection elements from manifest/table rows, map a tool over them, then relabel or reshape outputs. -- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped outputs, sync siblings from surviving identifiers, then gate final reports. -- [[reshape-relabel-remap-by-collection-axis]] — correct the mapped axis after domain fan-out using Apply Rules and deterministic relabeling. -- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection consumer, then flatten or aggregate results. +First identify what changed. Membership, order, and labels are separate decisions. -## Identifiers +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. -- [[sync-collections-by-identifier]] — membership sync: extract identifiers from one collection and filter or relabel a sibling collection. -- [[harmonize-by-sortlist-from-identifiers]] — order sync: sort one sibling collection by another collection's identifier order. -- [[regex-relabel-via-tabular]] — label rewrite: derive new element identifiers in tabular form and apply them with `__RELABEL_FROM_FILE__`. -- [[relabel-via-rules-and-find-replace]] — relabel inside a structural reshape, currently grounded in the influenza fan-out pattern. +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. -## Structural Reshape +## Change the collection axes -- [[collection-flatten-after-fanout]] — collapse nested collection output to a flat list when the outer axis no longer matters. -- [[collection-build-named-bundle]] — assemble individual outputs into a named collection bundle for publishing or downstream fan-in. -- [[collection-swap-nesting-with-apply-rules]] — use Apply Rules to swap `list:list` axes. -- [[collection-split-identifier-via-rules]] — use Apply Rules regex columns to derive nested list identifiers from one identifier string. -- [[collection-build-list-paired-with-apply-rules]] — use Apply Rules to promote identifier columns into `list:paired` structure. +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. -## Bridges +## Follow a map-over lifecycle -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. ## See also diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md index 23fae29..2dfe890 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md @@ -12,9 +12,9 @@ tags: - topic/galaxy-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy when and pick_value conditional patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." related_notes: - "[[iwc-conditionals-survey]]" related_patterns: @@ -30,29 +30,33 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: conditionals patterns -This is the runtime-facing map for Galaxy conditional workflow choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. -## Direct Gates +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. -- [[conditional-run-optional-step]] — expose or derive a boolean, connect it as `inputs.when`, and use `when: $(inputs.when)` to skip optional steps. -- [[conditional-gate-on-nonempty-result]] — compute a boolean from empty/non-empty dataset or collection state before gating downstream reporting/export. The MGnify recipe is corpus-backed but clunky pending verified-pattern workflow work. +## Skip work without replacing its output -## Routes and Fallbacks +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. -- [[conditional-route-between-alternative-outputs]] — run one of several `when`-gated alternatives, then merge possible outputs with `pick_value`. -- [[conditional-transform-or-pass-through]] — optionally transform a value, then use `pick_value` to choose transformed output or original fallback. +## Continue with one value -## Not Conditionals +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. -- [[collection-cleanup-after-mapover-failure]] — use collection filters when empty or failed mapped elements should be dropped or replaced. This is collection-state cleanup, not a `when` gate. -- `__FILTER_NULL__` — catalog capability for null outputs after conditional steps, but no IWC-backed pattern candidate in the conditionals survey. Zero uptake is not itself an anti-pattern call. +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. ## See Also - [[iwc-conditionals-survey]] — conditionals survey and evidence trail. -- [[galaxy-collection-patterns]] — companion MOC for collection transforms and cleanup. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md index 8123493..de9ec07 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md @@ -13,8 +13,8 @@ tags: - topic/interval-transform status: draft created: 2026-06-10 -revised: 2026-08-03 -revision: 2 +revised: 2026-09-23 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." related_notes: - "[[iwc-interval-operations-survey]]" @@ -34,6 +34,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md index 445ed6c..591b3f5 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md @@ -13,8 +13,8 @@ tags: - topic/tabular-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-10-15 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." related_notes: - "[[iwc-tabular-operations-survey]]" @@ -40,38 +40,39 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: tabular patterns -This is the runtime-facing map for Galaxy tabular transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation pages are the actionable references. +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. -## Row And Column Operations +## Rows and columns in one table -- [[tabular-filter-by-column-value]] — keep/drop rows by string column value with `Filter1`. -- [[tabular-filter-by-regex]] — keep/drop rows by regex or pattern matching. -- [[tabular-cut-and-reorder-columns]] — project and reorder columns with `Cut1`. -- [[tabular-compute-new-column]] — compute row-wise values into a new column. +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. -## Joins And Aggregation +## Combine or summarize tables -- [[tabular-join-on-key]] — join two tabular datasets by key columns. -- [[tabular-group-and-aggregate-with-datamash]] — group and aggregate rows with `datamash_ops`. -- [[tabular-sql-query]] — use SQL when filtering, joining, and projection are clearer as one query. +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. -## Text-Processing Recipes +## Text-processing recipes -- [[tabular-prepend-header]] — add a header row with an awk/text-processing step. -- [[tabular-synthesize-bed-from-3col]] — build BED from three-column tabular inputs. -- [[tabular-split-taxonomy-string]] — split taxonomy-like strings into useful columns. -- [[tabular-relabel-by-row-counter]] — synthesize row labels from row order. +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. -## Bridges +## Between tables and collections -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. ## See also diff --git a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json index 3a39f43..dd3a94a 100644 --- a/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json +++ b/plugins/foundry-skills/skills/cwl-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json @@ -75,7 +75,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_version": { "anyOf": [ @@ -108,7 +108,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "out": { "anyOf": [ @@ -309,7 +309,7 @@ ] }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "release": { "anyOf": [ @@ -412,22 +412,28 @@ "items": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" } ] }, @@ -437,22 +443,28 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" }, { "type": "string" @@ -927,7 +939,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "label": { "anyOf": [ @@ -1040,7 +1052,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_state": { "anyOf": [ @@ -1856,7 +1868,9 @@ "text", "File", "data", - "collection" + "collection", + "color", + "directory_uri" ], "type": "string" } @@ -1952,10 +1966,10 @@ ] }, "restrictions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "suggestions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "type": { "enum": [ @@ -1963,6 +1977,9 @@ "string" ], "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" } }, "required": [ @@ -1971,6 +1988,96 @@ "type": "object" }, "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { "additionalProperties": false, "properties": { "default": { @@ -2016,7 +2123,7 @@ "required": [], "type": "object" }, - "Shared10": { + "Shared11": { "additionalProperties": false, "properties": { "default": { @@ -2124,7 +2231,7 @@ ], "type": "object" }, - "Shared11": { + "Shared12": { "additionalProperties": false, "properties": { "default": { @@ -2230,7 +2337,7 @@ "required": [], "type": "object" }, - "Shared12": { + "Shared13": { "additionalProperties": false, "properties": { "default": { @@ -2337,7 +2444,7 @@ ], "type": "object" }, - "Shared13": { + "Shared14": { "additionalProperties": false, "properties": { "default": { @@ -2424,7 +2531,143 @@ ], "type": "object" }, - "Shared14": { + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { "anyOf": [ { "items": { @@ -2457,7 +2700,7 @@ } ] }, - "Shared15": { + "Shared18": { "anyOf": [ { "type": "null" @@ -2488,11 +2731,11 @@ } ] }, - "Shared16": { + "Shared19": { "anyOf": [ { "items": { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, "type": "array" }, @@ -2500,7 +2743,7 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, { "type": "string" @@ -2522,7 +2765,7 @@ } ] }, - "Shared17": { + "Shared20": { "anyOf": [ { "type": "null" @@ -2610,7 +2853,7 @@ "$ref": "#/$defs/Shared1" }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "uuid": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/SKILL.md b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/SKILL.md index 72e3baa..95adb04 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/SKILL.md +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/SKILL.md @@ -13,7 +13,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs -- Read artifact `summary-cwl`. Schema: summary-cwl. Produced by `summarize-cwl`. Structured CWL summary from summarize-cwl; carries test fixtures, job inputs, expected outputs. +- Read artifact `summary-cwl`. Schema: summary-cwl. Produced by `summarize-cwl`. Structured CWL summary from summarize-cwl; lists discovered test cases, job-file paths, and expected-output references. ## Outputs @@ -26,7 +26,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load Upfront - `references/schemas/galaxy-workflow-test-plan.schema.json`: Schema file copied verbatim into the bundle. Output contract: the emitted plan conforms to galaxy-workflow-test-plan. Cast bundles the JSON Schema; validate with `foundry validate-galaxy-workflow-test-plan`. -- `references/schemas/summary-cwl.schema.json`: Schema file copied verbatim into the bundle. Read the CWL summary's test cases, job inputs, expected outputs, and assertion evidence. +- `references/schemas/summary-cwl.schema.json`: Schema file copied verbatim into the bundle. Read the CWL summary's test cases, job-file paths, expected-output references, and assertion evidence. ## Load On Demand @@ -39,11 +39,20 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. +- Before starting, check that the installed validator carries the galaxy-workflow-test-plan schema this bundle was cast with: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. ## Procedure -Translate CWL test fixtures, job inputs, expected outputs, and assertion evidence into a Galaxy workflow test plan. The output is a reviewable YAML handoff conforming to galaxy-workflow-test-plan, not a concrete `tests-format` file; implement-galaxy-workflow-test owns final YAML authoring and static validation. Because this plan is translated from real CWL test fixtures, set `source.derived_from: test-evidence` and prefer `evidence: test-evidence` on the assertions it carries. +Turn the CWL summary's discovered test cases into a reviewable Galaxy test plan. Preserve each case's job-file and expected-output provenance so implement-galaxy-workflow-test can author the final test against the concrete Galaxy workflow. The plan records assertion intent and open mappings, not a `tests-format` test file. + +For each `tests[]` case, open its `job_path` when available. Map the declared CWL input values onto the Galaxy inputs the summary and translation imply. Keep `File` locations, `Directory` contents, `secondaryFiles`, arrays, and collection element identifiers visible where they affect the fixture or its Galaxy shape. Record the source path or URL and any known checksum. If a job file is missing or an input cannot be mapped, leave the fixture or label unresolved with a reason. A `job_path` in the summary is a pointer to inputs, not the inputs themselves. + +Translate each recorded expected output into a Galaxy output to check. Use the CWL output id and any supplied path, URL, checksum, or assertion as evidence for an appropriate `tests-format` assertion family. Record what the assertion should establish, its expected value and tolerance when supported, and the source of that expectation. Do not infer a passing value from an output name or turn a checksum into a Galaxy assertion without confirming that the translated output can be compared that way. For collections and sidecars, identify the element or file that the check actually addresses. + +Bind input and output labels to a concrete Galaxy draft only if one is available to inspect. Otherwise mark inferred labels as assumed or unresolved, set `workflow.label_source` accordingly, and record the mapping that implement-galaxy-workflow-test must confirm. Set `source.derived_from: test-evidence` for a plan based on discovered CWL tests, and `evidence: test-evidence` only for assertions supported by those tests. If a useful Galaxy check must instead be inferred from workflow intent, mark that assertion `evidence: intent` and explain its lower confidence. + +Record unavailable fixtures, expression-dependent shapes, untranslatable assertions, and missing label or datatype mappings in `unresolved[]` or `warnings[]` as appropriate. Put outputs deliberately left unasserted in `omissions[]` with a reason. If the summary has no usable test case, report that coverage gap rather than presenting a fabricated case as translated evidence. Validate the emitted `galaxy-test-plan.yml` with `foundry validate-galaxy-workflow-test-plan` before handing it to implement-galaxy-workflow-test. ## Feedback Mode diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_provenance.json b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_provenance.json index a2d2a5e..011fa08 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_provenance.json +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "cwl-test-to-galaxy-test-plan", "path": "content/molds/cwl-test-to-galaxy-test-plan/index.md", - "revision": 2, - "content_hash": "b708f94d8cbc0698ed076f62ff6d6194da5ae34c3cdb6443597c433eeca625e3", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 3, + "content_hash": "b55214a0b2bf18e61a75a6c7aa32cf69992aa703480b754e067707dfc262813e", + "commit": "a59d6025d612ce7013b09fa1ccb58f60507ccc6a" }, - "cast_at": "2026-09-13T20:58:26.983Z", + "cast_at": "2026-10-26T16:18:39.955Z", "refs": [ { "kind": "research", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Choose which workflow outputs and promoted checkpoints make meaningful assertions.", "trigger": "When deciding which outputs to assert and which labels the plan should bind to.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -36,8 +36,8 @@ "evidence": "corpus-observed", "purpose": "Distinguish accepted IWC-style test shortcuts from assertion smells while translating tests.", "trigger": "When deciding whether to use existence-only, size-only, image-dimension, or tolerant output checks.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -51,8 +51,8 @@ "evidence": "corpus-observed", "purpose": "Emit Galaxy/IWC-style job input fixtures, remote locations, hashes, and collection input shapes.", "trigger": "When writing job inputs or deciding whether fixtures belong in test-data, Zenodo, ENA/SRA, or CVMFS.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -66,8 +66,8 @@ "evidence": "corpus-observed", "purpose": "Describe Galaxy workflow-test assertion intent and tolerances for translated expected outputs.", "trigger": "When turning CWL expected outputs into Galaxy test-plan assertions.", - "src_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", - "dst_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", "source": "deterministic" }, { @@ -81,8 +81,8 @@ "evidence": "corpus-observed", "purpose": "Keep the plan addressable by stable labels and artifacts Planemo can connect back to invocations, jobs, and outputs.", "trigger": "When recording the labels and checkpoints the downstream test must address.", - "src_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", - "dst_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", + "src_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "dst_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", "source": "deterministic" }, { @@ -97,8 +97,8 @@ "purpose": "Output contract: the emitted plan conforms to [[galaxy-workflow-test-plan]]. Cast bundles the JSON Schema; validate with `foundry validate-galaxy-workflow-test-plan`.", "verification": "Cast the skill on a CWL summary carrying tests and confirm the emitted YAML validates and downstream [[implement-galaxy-workflow-test]] consumes it.", "license": "MIT", - "src_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", - "dst_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", + "src_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "dst_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "source": "deterministic" }, { @@ -110,7 +110,7 @@ "used_at": "runtime", "load": "upfront", "evidence": "corpus-observed", - "purpose": "Read the CWL summary's test cases, job inputs, expected outputs, and assertion evidence.", + "purpose": "Read the CWL summary's test cases, job-file paths, expected-output references, and assertion evidence.", "license": "MIT", "src_hash": "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "dst_hash": "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", @@ -129,8 +129,8 @@ "trigger": "When mapping expected outputs, tolerances, or fixture assertions into Galaxy workflow-test assertion intent.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", - "dst_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", + "src_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "dst_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } @@ -148,7 +148,7 @@ "consumes": [ { "id": "summary-cwl", - "description": "Structured CWL summary from [[summarize-cwl]]; carries test fixtures, job inputs, expected outputs.", + "description": "Structured CWL summary from [[summarize-cwl]]; lists discovered test cases, job-file paths, and expected-output references.", "inherited_schema": "[[summary-cwl]]", "producers": [ "summarize-cwl" diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_verify.json b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_verify.json index c56d682..30acd9d 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_verify.json +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-cwl", + "--schema-sha256", + "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "{artifact_path}" ] }, @@ -22,6 +24,8 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-workflow-test-plan", + "--schema-sha256", + "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md index 6d0fef7..bd1160e 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-11 -revision: 6 +revised: 2026-10-21 +revision: 10 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,231 +16,109 @@ related_notes: - "[[validate-tests]]" - "[[iwc-tabular-operations-survey]]" - "[[galaxy-discover-datasets]]" -summary: "Decision and idiom guide for picking planemo workflow-test assertions: which family per output type, how to size tolerances, when to validate." +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." --- -# Planemo asserts: idiom and decision guide +# Choosing Planemo workflow-test assertions -Companion to [[iwc-test-data-conventions]] (input shapes), [[galaxy-workflow-testability-design]] (workflow structure before test YAML exists), and [[iwc-shortcuts-anti-patterns]] (what's accepted vs smell). This note is forward-looking: when authoring a new `-tests.yml`, which assertion family fits which output, and what the recommended tolerances and operators are. +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. -The **vocabulary itself is not restated here** — every assertion's parameter list, types, defaults, required fields, and Python docstring is rendered from the test-format JSON Schema at [[tests-format]]. Assertion names below deep-link into that page (e.g. [[tests-format#has_text_model|has_text]] jumps straight to that `$def`). +## 1. Choose by the property, then the output type -## 1. Choose by output type +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | -The single most useful decision table. Pick the row that matches the file format the workflow emits; default to the recommended assertion family. +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. -| Output type | Default assertion family | Why | Fallback | -|---|---|---|---| -| **Plain text reports / logs** (FastQC summary, MultiQC text section) | [[tests-format#has_text_model|has_text]] (substring on a known stable token) + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Substring is robust across versions; line count catches truncation | [[tests-format#has_text_matching_model|has_text_matching]] (regex) when token text changes per run | -| **HTML reports** (MultiQC HTML, custom dashboards) | [[tests-format#has_text_model|has_text]] against stable section names | HTML embeds timestamps and asset hashes; byte-diff is hopeless | [[tests-format#has_size_model|has_size]] + large `delta` as last resort | -| **Tabular** (TSV, CSV, BED-like) | [[tests-format#has_n_columns_model|has_n_columns]] + [[tests-format#has_text_model|has_text]] for headers + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Schema-shape + content probes; tolerant to row reordering | [[tests-format#has_line_matching_model|has_line_matching]] for a known canonical row | -| **VCF** | `compare: diff` with `lines_diff: 6` | The `lines_diff: 6` constant matches the typical VCF header preamble that embeds `##fileDate=` and `##source=` | [[tests-format#has_text_matching_model|has_text_matching]] for known variants | -| **BAM** | [[tests-format#has_size_model|has_size]] + [[tests-format#has_archive_member_model|has_archive_member]] (BAM is a gzipped block format) | Headers contain `@PG` lines with command lines and Galaxy job IDs; never byte-diff | Convert to SAM in a downstream step and assert on text | -| **FASTA** (deterministic — assemblies, consensus) | `file:` exact comparison or [[tests-format#has_text_model|has_text]] for known sequence | Output is byte-stable when the upstream tool is deterministic | [[tests-format#has_n_lines_model|has_n_lines]] if line wrapping varies | -| **FASTA** (non-deterministic — RepeatModeler libraries) | `compare: sim_size` with large `delta:` | Family content varies run-to-run | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | -| **FASTQ** (rare as workflow output) | [[tests-format#has_n_lines_model|has_n_lines]] (must be multiple of 4) | Quality scores are read-id-dependent | `compare: sim_size` | -| **JSON** (deterministic — config dumps, params) | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] / [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] | Surgical: assert on the property you care about, ignore the rest | `compare: diff` if the JSON is fully canonical | -| **JSON** (stochastic — HyPhy stats, MCMC results) | `has_text: text: "{"` (existence-only) | Embedded floats break any structural assertion; see [[iwc-shortcuts-anti-patterns]] §1 | [[tests-format#has_h5_keys_model|has_h5_keys]]-equivalent for top-level structural keys | -| **HDF5 / AnnData** | [[tests-format#has_h5_keys_model|has_h5_keys]] + [[tests-format#has_h5_attribute_model|has_h5_attribute]] for known structure | Asserts shape, not values — values are float-noisy | [[tests-format#has_size_model|has_size]] with `delta:` | -| **XML** | [[tests-format#is_valid_xml_model|is_valid_xml]] + [[tests-format#has_element_with_path_model|has_element_with_path]] + `element_text_is`/`element_text_matches` | Surgical structural probes; byte-diff fails on whitespace/attribute-order | [[tests-format#has_n_elements_with_path_model|has_n_elements_with_path]] for cardinality | -| **PNG / image plots** | [[tests-format#has_image_width_model|has_image_width]] + [[tests-format#has_image_height_model|has_image_height]] + [[tests-format#has_size_model|has_size]] with `delta:` (5–10%) | Catches "rendered something with the right dimensions"; corpus norm | [[tests-format#has_image_center_of_mass_model|has_image_center_of_mass]] if mask/segmentation outputs are deterministic | -| **TIFF / multipage images** | [[tests-format#has_image_frames_model|has_image_frames]] + [[tests-format#has_image_channels_model|has_image_channels]] + [[tests-format#has_size_model|has_size]] with `delta:` | Same logic, channel/frame structure matters | — | -| **Archives** (zip, tar.gz) | `has_archive_member: path: "regex"` with nested `asserts:` | Asserts on a specific member; archive timestamps never byte-stable | [[tests-format#has_size_model|has_size]] with `delta:` | -| **GFF / GTF** | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` + [[tests-format#has_text_model|has_text]] for known feature | Tools reorder freely; hard to byte-diff | `has_n_columns: 9` (GFF spec) | -| **Cool / HiC matrices** | `compare: sim_size` with multi-MB `delta:` | Binary, run-to-run variance | [[tests-format#has_archive_member_model|has_archive_member]] for HDF5 component | +## 2. Compare operators and expected artifacts -When in doubt: start with [[tests-format#has_size_model|has_size]] + `delta_frac: 0.1`. It catches the catastrophic failure mode (empty / 10x bigger output). Then add a content probe. +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. -### 1a. If the assertion is too weak, revisit the workflow output +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. -Assertion choice sometimes reveals a workflow-design problem. If the only available output can support only a size check or image-dimension smoke test, check whether the workflow should expose a stronger checkpoint before settling for the weak assertion. +## 3. Set tolerances from the failure boundary -- Scanpy's plot outputs use size and image-dimension assertions, but the same workflow also exposes AnnData HDF5 checkpoints and a cluster-count table (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end uses coarse size bands for coverage/mapped-read outputs, but expression/count tables get stronger regex and exact-line checks (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. -Rule: if a translated workflow exposes only weakly assertable final reports, consult [[galaxy-workflow-testability-design]] and consider promoting a table/text/HDF5 checkpoint before writing the final assertions. +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. -## 2. The `compare:` operators +## 4. Text assertions and repeated checks -Top-level on a `file:` output assertion. Vocabulary, in decreasing strictness: - -- **`diff`** (default). Byte-for-byte equality with optional `lines_diff:` tolerance for a fixed number of header lines. Use only when the upstream tool is deterministic on fixed inputs *and* the output has no embedded timestamps, command lines, version banners, or hash-ordered Python-dict-style keys. -- **`re_match` / `re_match_multiline`**. Each line of the expected fixture is a regex that must match the corresponding output line. Useful when a few fields per row are timestamped but the rest is canonical. Rare in the corpus. -- **`contains`**. The expected fixture is a substring of the output. Cheap; weak. Prefer `asserts: has_text` for new code unless you genuinely have a multi-line block to assert as a whole. -- **`sim_size`**. Output file size matches the fixture's size within `delta:` (bytes) or `delta_frac:` (fraction). Use when the output is necessarily non-deterministic but its rough size is reproducible (RepeatModeler libraries, HiC matrices, Bayesian sampler outputs). - -Picking `lines_diff:`: count the mutable header lines in the output format. VCF: ~6 (`##fileformat`, `##fileDate`, `##source`, `##reference`, contig/info lines vary). SAM/text headers: count `@HD`/`@PG`/`@CO` lines. Set `lines_diff:` to the count exactly — looser values mask real diffs. - -## 3. Tolerance picking - -**`delta:`** is bytes (for [[tests-format#has_size_model|has_size]] and `compare: sim_size`) or absolute count (for [[tests-format#has_n_lines_model|has_n_lines]], [[tests-format#has_n_columns_model|has_n_columns]], [[tests-format#has_image_width_model|has_image_width]], etc.). Suffix multipliers documented in the schema — `1K`, `1M`, `1G` work. - -**`delta_frac:`** is a fraction (0.1 = 10%). Use when expected size scales with input volume. Three IWC tests use it (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`); the rest use absolute `delta:`. - -**Picking magnitudes** (from corpus survey in [[iwc-shortcuts-anti-patterns]] §2): - -- Image dimensions: `delta: 25–30` pixels (5% of typical matplotlib defaults). -- Image file size: `delta: 5K–60K` (5–10% of file size). -- Small text reports: `delta: 1K–10K`. -- HTML reports: `delta: 25K–100K`. -- BAM files: `delta: 1M–10M`. -- RepeatModeler / Bayesian sampler outputs: `delta: 30K–90M` (extreme, but justified by the underlying nondeterminism). - -**Heuristic for new outputs:** `delta_frac: 0.1` is a defensible default. Tighten if the output proves more deterministic than expected. - -## 4. Text family — [[tests-format#has_text_model|has_text]] vs [[tests-format#has_text_matching_model|has_text_matching]] vs [[tests-format#has_line_model|has_line]] vs [[tests-format#has_line_matching_model|has_line_matching]] vs [[tests-format#has_n_lines_model|has_n_lines]] - -All five are common; choose by what you're verifying. - -- **[[tests-format#has_text_model|has_text]]** — output contains the substring `text:`. Anywhere in the output, any number of times. Add `n:` / `min:` / `max:` to constrain occurrence count. Add `delta:` to allow slack on the count. -- **[[tests-format#has_text_matching_model|has_text_matching]]** — output matches the regex `expression:`. Use sparingly; prefer literal [[tests-format#has_text_model|has_text]] when you can. -- **[[tests-format#has_line_model|has_line]]** — output has at least one *line* matching `line:` exactly. Use when line boundaries matter (e.g. asserting on a specific row in a table). -- **[[tests-format#has_line_matching_model|has_line_matching]]** — same but with regex. -- **[[tests-format#has_n_lines_model|has_n_lines]]** — assert the line count is `n:` ± `delta:`. - -A common combination in IWC: `has_n_lines: n: 100, delta: 5` + `has_text: text: "expected_token"` — line-count sanity-check plus a content marker. This catches both truncation and content drift in one assertion pair. - -`negate: true` is supported on every assertion. Used for the "this output should NOT contain X" case. - -## 5. Collection output assertions (`element_tests:`) - -For Galaxy collection outputs, the test format keys element assertions by element identifier: +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. ```yaml -my_collection_output: - element_tests: - sample_1: +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: asserts: - has_text: - text: "expected" - sample_2: - file: test-data/expected_sample_2.txt -``` - -Optional `attributes:` at the collection level can assert on the produced collection shape: - -```yaml -my_collection_output: - attributes: {collection_type: list:list} - element_tests: - ... + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 ``` -Nested collections: outer `element_tests:` keyed by outer identifier; inner uses `elements:` (note plural, no `_tests` suffix on the inner). See [[iwc-test-data-conventions]] §2f for the live example. - -For a list-of-files where every element should pass the same minimal check, the existence-probe pattern (`has_text: "{"` for JSON; `has_size: min: 100` for any non-empty binary) is widely used and accepted in IWC. - -## 6. The validate-against-workflow inner loop - -A `-tests.yml` file can be **structurally invalid** in two distinct ways: - -1. **Schema-invalid** — wrong field names, wrong nesting, wrong types. Caught by the test-format JSON Schema. -2. **Workflow-incoherent** — schema-valid YAML, but the input/output labels don't match the actual workflow. Renaming an output in the `.ga` and forgetting to update its sibling `-tests.yml` produces this case. Planemo will surface it as an "output not found" error at test-runtime, but only after a full workflow run. - -The `@galaxy-tool-util/schema` npm package ships **two** validators that catch both cases statically — no Galaxy or Planemo invocation needed: +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. -- **`validateTestsFile(yaml)`** — runs the file against `tests.schema.json` (AJV). Reports schema violations with paths. -- **`checkTestsAgainstWorkflow(workflow, tests)`** — cross-checks a `.ga` / format2 workflow against a tests file: missing input labels, missing output labels, type incompatibilities (e.g. test supplies a `File` for a parameter typed `int`). +## 5. Collection and structured-output examples -Both are pure-JS, take milliseconds, and have no Galaxy dependency. Wire them into the inner authoring loop: +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" ``` -edit -tests.yml - → validateTestsFile() # schema gate - → checkTestsAgainstWorkflow(.ga, tests) # coherence gate - → planemo workflow_test_on_invocation # assertion gate (no full re-run) - → planemo test # full integration (slow) -``` - -The first two gates short-circuit cheap mistakes before a slow planemo run. They are the static-validation equivalent of `gxwf` for tests, and the `implement-galaxy-workflow-test` mold should reference them as its primary inner-loop tooling. Source: galaxy-tool-util-ts package, `src/test-format/index.ts` exports. - -## 7. Authoring loop — generation, then refinement - -Reviewer convention is to **generate** the initial `-tests.yml` rather than hand-write it. Two planemo subcommands cover this: - -- **`planemo workflow_test_init --from_invocation `** ([[planemo-workflow_test_init]]) — given a successful Galaxy invocation ID, emit a `-tests.yml` with a `job:` block that captures all inputs (with SHA-1 hashes) and an `outputs:` block with `file:` references to the actual outputs (downloaded into `test-data/`). Hand-tighten the assertions afterward. -- **`planemo workflow_test_on_invocation `** ([[planemo-workflow_test_on_invocation]]) — re-evaluate an edited `-tests.yml` against a saved invocation without re-running the workflow. The fast inner loop for assertion iteration; complements the static gates in §6. - -Together these cut the assertion-iteration cost dramatically. An agent should: - -1. Run the workflow once on usegalaxy.* (or local) to get a known-good invocation. -2. `--from_invocation` to bootstrap the test file. -3. Replace the autogenerated `file:` exact-comparison assertions with assertion-family-appropriate alternatives per §1. -4. [[planemo-workflow_test_on_invocation]] after each edit; full [[planemo-test]] at the end. - -## 8. What the schema gives you for free - -When the test-format schema lands as a Foundry-rendered note, the agent can consult any assertion's `$def` directly for: parameter types, defaults, required fields, the `that` discriminator constant, and the original Python docstring (carried through as `description`). This note does **not** restate that vocabulary — it complements it with the corpus-grounded *which-and-when*. - -What's still missing from the schema and worth keeping in research notes: - -- This decision table (§1) — output-type → assertion family. -- Tolerance magnitudes (§3) — corpus-derived defaults. -- The `validateTestsFile` / `checkTestsAgainstWorkflow` integration story (§6). -- Anti-pattern flags — see [[iwc-shortcuts-anti-patterns]]. -## 9. Common combinations (recipes) +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. -Six recipes worth memorizing. +## 6. Validate labels and schema before execution -**Stable text report (FastQC summary, simple stats).** -```yaml -my_report: - asserts: - has_n_lines: { n: 12, delta: 2 } - has_text: { text: "Total Sequences" } -``` - -**MultiQC HTML report.** -```yaml -multiqc_report: - asserts: - has_text: { text: "Filtered Reads" } - has_text: { text: "FastQC" } -``` +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: -**VCF (pinned tool, fixed reference).** -```yaml -called_variants: - file: test-data/expected.vcf - compare: diff - lines_diff: 6 +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga ``` -**Stochastic JSON (HyPhy-style).** -```yaml -hyphy_meme: - element_tests: - geneA: { asserts: { has_text: { text: "{" } } } - geneB: { asserts: { has_text: { text: "{" } } } -``` +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. -**Matplotlib plot.** -```yaml -umap_plot: - asserts: - has_size: { size: 68416, delta: 6000 } - has_image_width: { width: 601, delta: 30 } - has_image_height: { height: 429, delta: 25 } -``` +## 7. Create and refine the test -**AnnData (HDF5).** -```yaml -clustered_anndata: - asserts: - has_h5_keys: { keys: "obs/louvain" } - has_h5_keys: { keys: "var/highly_variable" } - has_h5_keys: { keys: "uns/rank_genes_groups" } - has_size: { size: 12000000, delta: 1500000 } -``` +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. -## 10. Cross-references +## 8. Source of the exact syntax -- [[galaxy-workflow-testability-design]] — decide which workflow outputs and checkpoints to expose before choosing assertions. -- [[iwc-test-data-conventions]] — input-side conventions (job inputs, collection shapes, `hashes:`, CVMFS). -- [[iwc-shortcuts-anti-patterns]] — accepted-vs-smell catalog and corpus prevalence; this note's mirror image. -- Test-format schema (`@galaxy-tool-util/schema` npm package) — authoritative vocabulary; will be vendored into a Foundry-rendered schema note. See `content/meta/casting.md` for the casting story. -- Planemo test-format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- [[galaxy-xsd]] — Galaxy XSD assertion source of truth, vendored from upstream. -- Tightening of the schema and Pydantic source: [galaxyproject/galaxy#22566](https://github.com/galaxyproject/galaxy/pull/22566). -- TS schema sync into npm: [jmchilton/galaxy-tool-util-ts#75](https://github.com/jmchilton/galaxy-tool-util-ts/pull/75). +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md index 59b3c25..72583a5 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-11 -revision: 3 +revised: 2026-10-26 +revision: 4 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[galaxy-tool-job-failure-reference]]" @@ -23,6 +23,8 @@ sources: - "~/projects/repositories/planemo/planemo/galaxy/activity.py" - "~/projects/repositories/planemo/planemo/galaxy/invocations" - "~/projects/repositories/planemo/planemo/galaxy/config.py" + - "~/projects/repositories/planemo/planemo/galaxy/serve.py" + - "~/projects/repositories/planemo/planemo/engine/galaxy.py" summary: "Reference for Planemo workflow test/run architecture, Galaxy modes, API polling, and noisy failure boundaries." --- @@ -94,6 +96,20 @@ Important operations: For workflows, Planemo invokes Galaxy using input labels/names. Stable generated labels are therefore important for both test execution and debugging. +### Workflow Import Failures + +Planemo imports the workflow while it brings Galaxy up for the run (`config.install_workflows()`, called from `_serve()` in `planemo/galaxy/serve.py` for managed Galaxy and from `ExternalGalaxyEngine.ensure_runnables_served` in `planemo/engine/galaxy.py` for external Galaxy, as of planemo 0.75.47). For gxformat2 it converts the workflow to a native dict client-side first, so Galaxy's own gxformat2 conversion never runs. That is outside the per-test `execute()` wrapper that turns exceptions into an `ErrorRunResponse`. When Galaxy rejects the import request, BioBlend raises `ConnectionError`, the test command exits non-zero, and Planemo writes neither the `--test_output_json` report nor the HTML report. No workflow, history, or invocation id exists. + +The evidence then lives in three places: + +| Evidence | What it holds | +|---|---| +| Upload response | HTTP status and body. A 400 usually names the failing field or validation rule. A 500 body rarely explains the cause. | +| Galaxy server log | The traceback for the failing request, the only place a 500's cause appears. Managed Galaxy streams its log into the Planemo terminal output and keeps it on disk. External Galaxy keeps it on the server. | +| Planemo terminal log | The BioBlend exception and the point in the run where it was raised. | + +Re-importing the same converted dict as the same user on the same server reproduces the failure without the rest of the Planemo run, which makes it the cheap loop for isolating a trigger. + Workflow testability design guidance lives in [[galaxy-workflow-testability-design]]. This architecture note only records why Planemo makes labels and workflow-level outputs operationally important. ## Structured Artifacts @@ -121,6 +137,7 @@ Planemo transforms and summarizes Galaxy failure information. These are likely i - Polling may summarize “at least one job is in error” while detailed job evidence is printed elsewhere or stored separately. - Missing outputs become assertion/output problems, which can conflate label mismatch, workflow output omission, optional output absence, failed invocation, or output download issue. - Output download failures may be logged but not always propagated as the most specific failure. +- A rejected workflow import aborts the run before any structured report is written, so terminal and server logs are the only record. - External Galaxy without an admin key may defer missing-tool problems until runtime. - `allow_tool_state_corrections=True` can let Galaxy adjust tool state during invocation, which is useful but can mask definition drift. - `--no_wait` is not suitable for debug conclusions because invocation creation can succeed before jobs fail. @@ -135,7 +152,7 @@ For [[run-workflow-test]]: For [[debug-galaxy-workflow-output]]: -- Start from structured Planemo output. +- Start from structured Planemo output. When there is none because the workflow import failed, start from the upload response and the Galaxy server log. - For missing-output failures, check label drift and omitted workflow-level outputs before assuming tool failure. - If the failure is job-level, inspect Galaxy job APIs described in [[galaxy-tool-job-failure-reference]]. - If the failure is invocation-level, inspect Galaxy invocation APIs described in [[galaxy-workflow-invocation-failure-reference]]. diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json index 1539281..82e4529 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json @@ -358,7 +358,7 @@ "properties": { "family": { "type": "string", - "description": "tests-format assertion family or compare operator this intent maps to, by name, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys, file, compare:diff, compare:sim_size. Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." + "description": "tests-format assertion family, compare operator, or output-attribute field this intent maps to, by name. Assertion families go inside the output's `asserts:`, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys. Compare operators and output-attribute fields sit beside `asserts:`: file, compare:diff, compare:sim_size, element_count (collection outputs), count (nested sub-collections), ftype, checksum, metadata. Do not use collection_type (state it in ExpectedOutput.collection_shape) or element_tests (a non-null element_identifier implies it). Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." }, "intent": { "type": "string", @@ -390,15 +390,16 @@ "string", "null" ], - "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions." + "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions. Address an element of a nested collection by joining identifiers outermost first with `/`, e.g. WT_REP1/forward in a list:paired." }, "evidence": { "type": "string", "enum": [ "test-evidence", + "fixture-derived", "intent" ], - "description": "Whether this assertion was translated from upstream test evidence or synthesized from intent." + "description": "Basis of the expected value: test-evidence means translated from an upstream assertion, checksum, or expected file; fixture-derived means measured from upstream test fixture bytes where no upstream assertion exists (e.g. snapshot-only nf-tests); intent means synthesized from workflow intent, wrapper XML, or step state with no upstream anchor." }, "confidence": { "type": "string", diff --git a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json index 8de90e7..99d551f 100644 --- a/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json +++ b/plugins/foundry-skills/skills/cwl-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json @@ -1110,6 +1110,18 @@ "default": "Collection", "title": "Class" }, + "count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Count" + }, "element_tests": { "anyOf": [ { @@ -1171,6 +1183,30 @@ ], "default": null, "title": "Elements" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Maximum Count" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum Count" } }, "title": "TestCollectionCollectionElementAssertions", @@ -1366,6 +1402,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestCollectionDatasetElementAssertions", @@ -1679,6 +1728,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestDataOutputAssertions", diff --git a/plugins/foundry-skills/skills/cwl-to-test-data/SKILL.md b/plugins/foundry-skills/skills/cwl-to-test-data/SKILL.md index f9c5c60..ffd2fe8 100644 --- a/plugins/foundry-skills/skills/cwl-to-test-data/SKILL.md +++ b/plugins/foundry-skills/skills/cwl-to-test-data/SKILL.md @@ -18,7 +18,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Outputs -- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Test data resolved from the CWL source's declared test cases, expressed as URLs/paths plus expected shapes for downstream test authoring. Shared id with find-test-data — the branch's search fallback fills only the inputs this Mold leaves unresolved. +- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Schema: test-data-refs. Test data resolved from the CWL source's declared test cases, expressed as URLs/paths plus expected shapes for downstream test authoring. Shared id with find-test-data — the branch's search fallback fills only the inputs this Mold leaves unresolved. ## Required Tools @@ -27,6 +27,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load Upfront - `references/schemas/summary-cwl.schema.json`: Schema file copied verbatim into the bundle. Input contract: read `tests[]` — each case's `job_path` inputs and `expected_outputs` — as the declared source of test data. +- `references/schemas/test-data-refs.schema.json`: Schema file copied verbatim into the bundle. Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning. ## Load On Demand @@ -36,7 +37,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- None declared. +- Before starting, check that the installed validator carries the test-data-refs schema this bundle was cast with: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `test-data-refs.json` before returning it: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json`. This checks artifact `test-data-refs` against the test-data-refs schema. ## Procedure @@ -49,8 +51,8 @@ This skill is the source-specific first leg of the harness's `test-data-resoluti 1. **Enumerate Galaxy inputs and their required shape.** From the interface brief, list each workflow input: label, Galaxy collection shape (File / list / paired / list:paired / record), and datatype. This is the *target shape* every ref must satisfy. 2. **Read the declared test cases.** From the summary's `tests[]`, read each case's `job_path` inputs — File/Directory objects with their `location`/`path`, `secondaryFiles`, arrays, and inline literals — plus `expected_outputs`. Prefer the case whose inputs best cover the workflow's inputs. 3. **Map each declared input onto a Galaxy input.** Match by id and shape onto the interface's input labels. A CWL File array maps to a Galaxy list; a File carrying `secondaryFiles` may map to a record or composite datatype — record element identifiers and any prep needed to reach the Galaxy collection shape (galaxy-workflow-testability-design). -4. **Emit refs.** Write one `test-data-refs.json` entry per resolved input: prefer a remote `location` URL when the job references one (remote-URL-first, iwc-test-data-conventions); fall back to the in-tree `path` plus provenance only when no URL is published. Carry datatype, collection element identifiers, and any subset/split prep. Each entry maps to an addressable workflow input label. -5. **Report genuine gaps.** An input with no declared test-case input of the right shape stays `resolved: false` with a reason — this is what the harness hands to find-test-data. Do not search public sources here; searching is find-test-data's job, not this skill's. +4. **Emit refs.** Write one `test-data-refs.json` entry per resolved input: prefer a remote `location` URL when the job references one (remote-URL-first, iwc-test-data-conventions); fall back to the in-tree `path` plus provenance only when no URL is published. Carry datatype, collection element identifiers, and any subset/split prep. Each entry maps to an addressable workflow input label. Shape each entry per test-data-refs with `resolved_by: cwl-to-test-data`, and validate the file with `foundry validate-test-data-refs` before returning. +5. **Report genuine gaps.** An input with no declared test-case input of the right shape stays `resolved: false` with a reason and `storage: unresolved` — this is what the harness hands to find-test-data. Do not search public sources here; searching is find-test-data's job, not this skill's. ### No fabrication diff --git a/plugins/foundry-skills/skills/cwl-to-test-data/_provenance.json b/plugins/foundry-skills/skills/cwl-to-test-data/_provenance.json index 74f3dac..bb6c928 100644 --- a/plugins/foundry-skills/skills/cwl-to-test-data/_provenance.json +++ b/plugins/foundry-skills/skills/cwl-to-test-data/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "cwl-to-test-data", "path": "content/molds/cwl-to-test-data/index.md", - "revision": 1, - "content_hash": "a9ea8415ba92031680c1163a566b99272f50c56ffca632f87451d0f2a8a13bbd", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 2, + "content_hash": "65121a333d68a1d41fe7965710dfdfbeceb5682a3543ae89ced250a3c4fdd594", + "commit": "2f8792a62ca53f074605ddbcfb25e75871f52b24" }, - "cast_at": "2026-09-13T20:58:27.033Z", + "cast_at": "2026-10-22T03:17:16.966Z", "refs": [ { "kind": "research", @@ -36,8 +36,8 @@ "evidence": "corpus-observed", "purpose": "Map each declared job input to an addressable Galaxy input label and the collection shape it must populate.", "trigger": "When mapping a CWL File-array or secondaryFiles input onto a Galaxy collection shape.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -51,8 +51,8 @@ "evidence": "corpus-observed", "purpose": "Express each ref remote-URL-first with SHA-1 integrity and per-input collection layout when recording resolved fixtures.", "trigger": "When writing each `test-data-refs` entry.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -69,6 +69,22 @@ "src_hash": "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "dst_hash": "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[test-data-refs]]", + "src": "package://@galaxy-foundry/gxwf-foundry#testDataRefsSchema", + "dst": "references/schemas/test-data-refs.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning.", + "verification": "Run the test-data-resolution chain on a pinned scenario and confirm each leg's test-data-refs.json validates and implement-galaxy-workflow-test stages from it.", + "license": "MIT", + "src_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "dst_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "source": "deterministic" } ], "artifacts": { @@ -77,6 +93,7 @@ "id": "test-data-refs", "kind": "json", "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", "description": "Test data resolved from the CWL source's declared test cases, expressed as URLs/paths plus expected shapes for downstream test authoring. Shared id with [[find-test-data]] — the branch's search fallback fills only the inputs this Mold leaves unresolved." } ], diff --git a/plugins/foundry-skills/skills/cwl-to-test-data/_verify.json b/plugins/foundry-skills/skills/cwl-to-test-data/_verify.json index e4c3c30..c27e197 100644 --- a/plugins/foundry-skills/skills/cwl-to-test-data/_verify.json +++ b/plugins/foundry-skills/skills/cwl-to-test-data/_verify.json @@ -10,6 +10,22 @@ "validator_bin": "foundry", "args": [ "validate-summary-cwl", + "--schema-sha256", + "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", + "{artifact_path}" + ] + }, + { + "artifact_id": "test-data-refs", + "direction": "output", + "kind": "json", + "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", + "validator_bin": "foundry", + "args": [ + "validate-test-data-refs", + "--schema-sha256", + "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/cwl-to-test-data/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/cwl-to-test-data/references/schemas/test-data-refs.schema.json b/plugins/foundry-skills/skills/cwl-to-test-data/references/schemas/test-data-refs.schema.json new file mode 100644 index 0000000..45df249 --- /dev/null +++ b/plugins/foundry-skills/skills/cwl-to-test-data/references/schemas/test-data-refs.schema.json @@ -0,0 +1,291 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://galaxyproject.org/foundry/schemas/test-data-refs.schema.json", + "$comment": "Canonical source: packages/gxwf-foundry/src/schemas/test-data-refs/test-data-refs.schema.json in galaxyproject/foundry. Mold frontmatter cites this schema via [[test-data-refs]] wiki-links; the cast pipeline imports the `testDataRefsSchema` runtime export and serializes it into cast bundles. $defs.Fixture is a verbatim copy of the galaxy-workflow-test-plan Fixture; a test keeps them equal.", + "title": "Galaxy workflow test-data refs", + "description": "Test data for each Galaxy workflow input, written by the test-data-resolution chain (a source-specific Mold, then find-test-data) and read by implement-galaxy-workflow-test. One ref per workflow input; an input no leg could resolve stays in the list as an honest gap.", + "$ref": "#/$defs/TestDataRefs", + "$defs": { + "TestDataRefs": { + "type": "object", + "additionalProperties": false, + "required": [ + "refs" + ], + "properties": { + "refs": { + "type": "array", + "items": { + "$ref": "#/$defs/Ref" + }, + "description": "One entry per workflow input label. A later leg of the chain keeps every entry an earlier leg resolved, unchanged, and replaces only unresolved ones." + } + } + }, + "Ref": { + "type": "object", + "additionalProperties": false, + "required": [ + "workflow_label", + "collection_shape", + "datatype", + "resolved", + "resolved_by", + "reason", + "fixture" + ], + "properties": { + "workflow_label": { + "type": "string", + "minLength": 1, + "description": "Workflow input this ref feeds: its label in the Galaxy interface brief, or its input port in the CWL design." + }, + "collection_shape": { + "type": [ + "string", + "null" + ], + "description": "Galaxy collection shape when the input is a collection, e.g. list, paired, list:paired, record; null for a single dataset." + }, + "datatype": { + "type": [ + "string", + "null" + ], + "description": "Galaxy datatype/extension, e.g. fastqsanger.gz, fasta, tabular; null when undecided." + }, + "resolved": { + "type": "boolean", + "description": "true when the ref points at real data of the required shape; false for an honest gap." + }, + "resolved_by": { + "type": [ + "string", + "null" + ], + "description": "Name of the Mold that resolved this input, e.g. nextflow-to-test-data or find-test-data; null while unresolved." + }, + "reason": { + "type": [ + "string", + "null" + ], + "description": "Why the input is unresolved; null when resolved." + }, + "fixture": { + "$ref": "#/$defs/Fixture", + "description": "Where the data lives. For a collection whose elements carry their own fixtures, storage and location may be null." + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + }, + "description": "Collection elements, nested for nested shapes (list:paired → sample → forward/reverse). Identifiers must match the staged test file." + }, + "prep": { + "type": [ + "string", + "null" + ], + "description": "Data-import-boundary prep needed to reach the Galaxy shape from the source data: a subset, split, or concatenation. A note on the ref, not an analysis step." + } + }, + "allOf": [ + { + "if": { + "properties": { + "resolved": { + "const": false + } + } + }, + "then": { + "properties": { + "resolved_by": { + "type": "null" + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "properties": { + "storage": { + "const": "unresolved" + }, + "location": { + "type": "null" + } + } + } + } + }, + "else": { + "properties": { + "resolved_by": { + "type": "string", + "minLength": 1 + }, + "reason": { + "type": "null" + } + }, + "anyOf": [ + { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + } + }, + { + "if": { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + "then": { + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + } + } + } + } + } + } + ] + }, + "Element": { + "type": "object", + "additionalProperties": false, + "required": [ + "identifier" + ], + "properties": { + "identifier": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "$ref": "#/$defs/Fixture" + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + } + } + } + }, + "DataElement": { + "description": "An element of a resolved collection: it carries a located fixture, or nested elements that each do.", + "anyOf": [ + { + "required": [ + "fixture" + ], + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + }, + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + }, + "Fixture": { + "type": "object", + "additionalProperties": false, + "required": [ + "storage", + "location", + "checksum", + "provenance" + ], + "properties": { + "storage": { + "type": [ + "string", + "null" + ], + "enum": [ + "remote-url", + "in-repo", + "cvmfs-string", + "generated-toy", + "unresolved", + null + ], + "description": "Where the fixture data lives or should live: remote-url (Zenodo/EBI/SRA location), in-repo (committed test-data path), cvmfs-string (data-table/built-in-index bare string), generated-toy (to be synthesized), or unresolved (not yet sourced)." + }, + "location": { + "type": [ + "string", + "null" + ], + "description": "Concrete URL, repo-relative path, or data-table string when known; null when the fixture still needs resolution." + }, + "checksum": { + "type": [ + "string", + "null" + ], + "description": "Integrity hash when available, e.g. SHA-1:; null when unknown." + }, + "provenance": { + "type": "string", + "description": "Where this fixture comes from or how it was chosen, e.g. a translated source test-data reference, a Zenodo record, or a note that test-data-resolution must supply it." + } + } + } + } +} diff --git a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/SKILL.md b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/SKILL.md index f633930..a6a40b7 100644 --- a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/SKILL.md +++ b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/SKILL.md @@ -17,7 +17,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Outputs -- Write artifact `workflow-debug-report` as `workflow-debug-report.md`. Format: `markdown`. Failure-surface classification with captured job/invocation/collection/assertion evidence and a recommended next step or reference-gap follow-up. +- Write artifact `workflow-debug-report` as `workflow-debug-report.md`. Format: `markdown`. Failure-surface classification with captured import/job/invocation/collection/assertion evidence and a recommended next step or reference-gap follow-up. ## Required Tools @@ -36,7 +36,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - `references/notes/iwc-shortcuts-anti-patterns.md`: Research note copied verbatim into the bundle. Decide whether a proposed debug fix aligns with accepted IWC testing shortcuts or masks a real failure. Use when: debugging suggests weakening assertions, widening deltas, switching to existence checks, or changing output labels. - `references/notes/nextflow-operators-to-galaxy-collection-recipes.md`: Research note copied verbatim into the bundle. Trace collection output failures back to possibly lossy operator translations. Use when: debugging wrong nesting, missing elements, branch merges, bad joins, or gather/reduction mismatches. - `references/notes/planemo-asserts-idioms.md`: Research note copied verbatim into the bundle. Classify whether a failure is an assertion-choice problem, tolerance problem, or real workflow-output regression. Use when: planemo reports output assertion failures or generated tests are too strict/too weak. -- `references/notes/planemo-workflow-test-architecture.md`: Research note copied verbatim into the bundle. Locate which Planemo artifact or Galaxy API surface preserves the failure evidence. Use when: planemo output is ambiguous, structured test JSON is available, or rerunning can be avoided by inspecting an existing invocation. +- `references/notes/planemo-workflow-test-architecture.md`: Research note copied verbatim into the bundle. Locate which Planemo artifact or Galaxy API surface preserves the failure evidence. Use when: planemo output is ambiguous, structured test JSON is available or missing because Galaxy rejected the workflow import, or rerunning can be avoided by inspecting an existing invocation. ## Validation @@ -46,16 +46,20 @@ Follow the procedure below and use the artifact/reference sections as the runtim Triage a failing Galaxy workflow test. Take the structured handoff from run-workflow-test, classify the failure surface before proposing any repair, and capture the reference evidence the surface requires. When the failure cannot be classified from existing references, recommend a focused follow-up rather than converting uncertainty into a guessed fix. -Classify before repairing. The same red output can be a tool/job failure, a workflow invocation failure, a collection-output mismatch, a missing workflow output, or an assertion mismatch — and each routes to a different reference surface and a different fix. Locate where the evidence lives first (planemo-workflow-test-architecture). +Classify before repairing. A red run can be a workflow import rejection, a tool/job failure, a workflow invocation failure, a collection-output mismatch, a missing workflow output, or an assertion mismatch—and each routes to a different reference surface and a different fix. Locate where the evidence lives first (planemo-workflow-test-architecture). ### Sequence -1. **Classify the first failure surface.** From the run's structured result, decide whether the first failure is a tool/job failure, a workflow invocation failure, a collection-output mismatch, a missing workflow output, or an assertion mismatch. Classify before proposing repairs. -2. **Capture job-failure evidence.** When a job is in `error`/`failed`/`stopped`, record job id, tool id, exit code, job messages, the stdout/stderr distinction, and output dataset state per galaxy-tool-job-failure-reference; check whether the wrapper's failure semantics already explain it. -3. **Capture invocation-failure evidence.** When the invocation state or messages indicate scheduling, materialization, cancellation, conditional, or output-resolution failure, record invocation state, the structured message reason, the affected step, any subworkflow path, and the jobs summary per galaxy-workflow-invocation-failure-reference; note whether Planemo surfaced or hid the relevant Galaxy API detail. -4. **Trace collection mismatches.** When a failing output is a collection or mapped output, diagnose shape, mapping, reduction, and element-identifier mismatches with galaxy-collection-semantics; for workflows translated from Nextflow, trace wrong nesting / missing elements / bad joins back to possibly-lossy operator translations via nextflow-operators-to-galaxy-collection-recipes. -5. **Read assertion failures honestly.** When the failure is an assertion, use planemo-asserts-idioms to decide whether it is an assertion-choice/tolerance problem or a real output regression. Before weakening an assertion, widening a delta, or switching to an existence check, confirm against iwc-shortcuts-anti-patterns that the relaxation is an accepted IWC shortcut and not masking a real failure. -6. **Discover reference gaps.** When the failure cannot be classified confidently from the references above, recommend a focused follow-up — reference documentation, pattern capture, API verification, or eval coverage — rather than emitting a repair recipe built on a guess. +1. **Classify the first failure surface.** From the run's structured result, decide whether the first failure is a workflow import rejection, a tool/job failure, a workflow invocation failure, a collection-output mismatch, a missing workflow output, or an assertion mismatch. Check import first: when the handoff reports `failure_modality: workflow-import-rejected`, or carries no workflow, history, or invocation id because Galaxy refused the workflow, no later surface exists to inspect. Classify before proposing repairs. +2. **Capture import-rejection evidence.** When Galaxy rejected the workflow import, record the HTTP status and response body from the import request, the Galaxy server-log traceback for that request when one exists, and the Planemo terminal log. A 500 leaves a traceback. A handled 400 usually does not, and its response body carries the reason. A managed Galaxy's traceback also appears in the Planemo terminal log. An external Galaxy's appears only in its own server log. No structured Planemo output exists in this case—Planemo writes no test JSON or HTML report when the import fails (planemo-workflow-test-architecture)—and steps 4–7 have nothing to inspect. +3. **Isolate an import rejection's trigger, then route it.** Bisect with small, self-contained synthetic gxformat2 workflows rather than truncating the real one. Truncation leaves dangling step references that fail with unrelated errors. Import each variant as the same user on the same server, through the same conversion Planemo used, change one structural feature at a time, and keep a variant that imports as the passing control. Imports can leave state behind—an embedded user-defined tool is created on first import—so use distinct tool definitions per variant or a fresh server when that state could mask the trigger. Then route by what the bisection shows: + - A minimal variant that passes the pipeline's static validation (`gxwf validate`) reproduces a 500 with an internal server traceback: an upstream Galaxy or gxformat2 defect. Record the smallest failing variant, its passing control, and the traceback as the repro. Do not reshape the workflow to dodge the defect. + - A 400 naming a schema, validation, or lint problem, or a 500 that disappears once the variant passes static validation: a workflow-content error. Route it back to the phase that emitted the offending construct. When static validation accepted the construct Galaxy rejected, also record that validator gap as its own finding. +4. **Capture job-failure evidence.** When a job is in `error`/`failed`/`stopped`, record job id, tool id, exit code, job messages, the stdout/stderr distinction, and output dataset state per galaxy-tool-job-failure-reference; check whether the wrapper's failure semantics already explain it. +5. **Capture invocation-failure evidence.** When the invocation state or messages indicate scheduling, materialization, cancellation, conditional, or output-resolution failure, record invocation state, the structured message reason, the affected step, any subworkflow path, and the jobs summary per galaxy-workflow-invocation-failure-reference; note whether Planemo surfaced or hid the relevant Galaxy API detail. +6. **Trace collection mismatches.** When a failing output is a collection or mapped output, diagnose shape, mapping, reduction, and element-identifier mismatches with galaxy-collection-semantics; for workflows translated from Nextflow, trace wrong nesting / missing elements / bad joins back to possibly-lossy operator translations via nextflow-operators-to-galaxy-collection-recipes. +7. **Read assertion failures honestly.** When the failure is an assertion, use planemo-asserts-idioms to decide whether it is an assertion-choice/tolerance problem or a real output regression. Before weakening an assertion, widening a delta, or switching to an existence check, confirm against iwc-shortcuts-anti-patterns that the relaxation is an accepted IWC shortcut and not masking a real failure. +8. **Discover reference gaps.** When the failure cannot be classified confidently from the references above, recommend a focused follow-up — reference documentation, pattern capture, API verification, or eval coverage — rather than emitting a repair recipe built on a guess. ## Feedback Mode diff --git a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/_provenance.json b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/_provenance.json index 1f54d7d..0a73ae2 100644 --- a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/_provenance.json +++ b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "debug-galaxy-workflow-output", "path": "content/molds/debug-galaxy-workflow-output/index.md", - "revision": 5, - "content_hash": "100b3f985f8e309092bfd05cf544a8286e0a878ee2df3a155dfebe6fcdb286f0", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 6, + "content_hash": "0ea72a26de30b8086e4d11fea09bc269e94320ec97fb317f6c0232cd92841009", + "commit": "eec35c4bdb6f486a98372cbf022cced5b84e6556" }, - "cast_at": "2026-09-13T20:58:27.128Z", + "cast_at": "2026-10-27T03:54:45.132Z", "refs": [ { "kind": "research", @@ -58,8 +58,8 @@ "evidence": "corpus-observed", "purpose": "Interpret Galaxy job-level failure evidence including stdio rules, exit code, job messages, and output dataset state.", "trigger": "When a failed workflow test includes errored jobs, tool stderr/stdout, non-zero exit codes, or red output datasets.", - "src_hash": "bf6530fca24f40780836e92d8d263f789038a6e0a722351ab8e8c9aa42646e2c", - "dst_hash": "bf6530fca24f40780836e92d8d263f789038a6e0a722351ab8e8c9aa42646e2c", + "src_hash": "188511486605e05d6d6ffb555b9c31287a1b6bd94c1f67dd8e483ce258415bfa", + "dst_hash": "188511486605e05d6d6ffb555b9c31287a1b6bd94c1f67dd8e483ce258415bfa", "source": "deterministic" }, { @@ -73,8 +73,8 @@ "evidence": "corpus-observed", "purpose": "Interpret Galaxy invocation-level failure evidence including invocation state, structured messages, and step job summaries.", "trigger": "When a failed workflow test has invocation failure, missing workflow outputs, cancelled/paused steps, subworkflow failures, or collection population errors.", - "src_hash": "e8115df6ddca447d47d77cb33becda847cdd19b2cbe9924782bfc180b94bb451", - "dst_hash": "e8115df6ddca447d47d77cb33becda847cdd19b2cbe9924782bfc180b94bb451", + "src_hash": "e86b17c0b39b88076f1a1f177cce9ef1f868de0b36437a624e4b630e2c126559", + "dst_hash": "e86b17c0b39b88076f1a1f177cce9ef1f868de0b36437a624e4b630e2c126559", "source": "deterministic" }, { @@ -88,8 +88,8 @@ "evidence": "corpus-observed", "purpose": "Decide whether a proposed debug fix aligns with accepted IWC testing shortcuts or masks a real failure.", "trigger": "When debugging suggests weakening assertions, widening deltas, switching to existence checks, or changing output labels.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -118,8 +118,8 @@ "evidence": "corpus-observed", "purpose": "Classify whether a failure is an assertion-choice problem, tolerance problem, or real workflow-output regression.", "trigger": "When Planemo reports output assertion failures or generated tests are too strict/too weak.", - "src_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", - "dst_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", "source": "deterministic" }, { @@ -132,9 +132,9 @@ "load": "on-demand", "evidence": "corpus-observed", "purpose": "Locate which Planemo artifact or Galaxy API surface preserves the failure evidence.", - "trigger": "When Planemo output is ambiguous, structured test JSON is available, or rerunning can be avoided by inspecting an existing invocation.", - "src_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", - "dst_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", + "trigger": "When Planemo output is ambiguous, structured test JSON is available or missing because Galaxy rejected the workflow import, or rerunning can be avoided by inspecting an existing invocation.", + "src_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "dst_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", "source": "deterministic" } ], @@ -144,7 +144,7 @@ "id": "workflow-debug-report", "kind": "markdown", "default_filename": "workflow-debug-report.md", - "description": "Failure-surface classification with captured job/invocation/collection/assertion evidence and a recommended next step or reference-gap follow-up." + "description": "Failure-surface classification with captured import/job/invocation/collection/assertion evidence and a recommended next step or reference-gap follow-up." } ], "consumes": [ diff --git a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-tool-job-failure-reference.md b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-tool-job-failure-reference.md index 0630071..9934a99 100644 --- a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-tool-job-failure-reference.md +++ b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-tool-job-failure-reference.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-workflow-invocation-failure-reference]]" - "[[planemo-workflow-test-architecture]]" @@ -15,123 +15,130 @@ related_molds: - "[[implement-galaxy-tool-step]]" - "[[debug-galaxy-workflow-output]]" sources: - - "~/projects/repositories/galaxy/lib/galaxy/tools/__init__.py" - - "~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/xml.py" - - "~/projects/repositories/galaxy/lib/galaxy/tool_util/output_checker.py" - - "~/projects/repositories/galaxy/lib/galaxy/jobs" - - "~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/xml.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/stdio.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/output_checker.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/command_factory.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/runners/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/model/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/schema.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/api/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/services/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/jobs.py" summary: "Reference for Galaxy tool stdio rules, job failure detection, job states, and job API failure surfaces." --- -# Galaxy Tool And Job Failure Reference +# Galaxy tool and job failure reference -This is reference material, not a debug recipe. Use it to understand what Galaxy can know about a failed tool job and which API surfaces preserve that evidence. +A failed Galaxy job preserves execution evidence separately from workflow scheduling and output assertions. Start with its state, exit code, structured messages, and separate tool and runner streams. A red downstream dataset or stderr text alone does not identify the original failure. -## Model +[[galaxy-workflow-invocation-failure-reference]] covers scheduling and dependency messages. [[planemo-workflow-test-architecture]] locates the structured Planemo artifacts, and [[debug-galaxy-workflow-output]] uses this evidence to classify a failure before proposing repairs. -Galaxy tool failure handling is layered: +## Job evidence -- The tool wrapper defines expected failure semantics through `detect_errors`, ``, exit-code checks, regex checks, and command strictness. -- The job runner executes the command and captures exit code plus tool/job stdout and stderr streams. -- Galaxy evaluates configured failure rules and records structured `job_messages`. -- The job reaches a terminal state, output datasets may become `error`, and dependent jobs may pause or fail later. -- Workflow invocation APIs summarize those jobs, but job APIs preserve the most detailed tool-level evidence. +Full job detail includes execution identity, state, exit code, structured messages, dependencies, and separate tool and runner streams. The combined stream fields provide compatibility views of those streams. -## Tool Wrapper Failure Controls +| Field | Evidence | +|---|---| +| `id`, `tool_id`, `tool_version`, `state` | Identify the execution and the wrapper version whose failure rules apply. | +| `exit_code` | Process exit code recorded by the runner. A missing value is not evidence of exit code zero. | +| `tool_stdout`, `tool_stderr` | Captured streams from the tool command. | +| `job_stdout`, `job_stderr` | Captured streams from the job wrapper or runner context. | +| `stdout`, `stderr` | Combined compatibility views of tool and job streams. They lose the stream provenance. | +| `job_messages` | Structured records from stdio checks or output discovery. Empty messages do not prove success. | +| `dependencies` | Job dependencies included with full detail. | -Important wrapper controls: +Prefer separate streams for diagnosis. A runner failure can occur before the tool executes, and output processing can fail after the command exits successfully. -| Control | Meaning | -|---|---| -| `detect_errors="default"` | For non-legacy XML tools without explicit ``, Galaxy defaults to non-zero exit-code failure detection. | -| `detect_errors="exit_code"` | Adds fatal checks for non-zero exit codes, plus optional OOM exit-code handling. | -| `detect_errors="aggressive"` | Adds non-zero exit-code checks plus broad stdout/stderr regexes for OOM and generic error text. | -| `` | Adds explicit exit-code ranges or values with levels such as `fatal`, `warning`, or `fatal_oom`. | -| `` | Adds case-insensitive stdout/stderr regex checks. | -| `` | Controls shell strictness; newer tool profiles default to `set -e` behavior. | +## Job and output dataset states -Current Galaxy source parses these rules while loading the tool, then stores exit-code and regex rules on the tool object. XML parser behavior lives under `~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/xml.py`; concrete stdio presets live under `~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/stdio.py`. +| Job state | Interpretation | +|---|---| +| `new`, `waiting`, `upload`, `queued`, `running`, `resubmitted` | Execution is pending or in progress. A timeout while polling these states needs timing and runner evidence. | +| `ok` | Galaxy accepted the execution. Check the expected outputs and assertions separately. | +| `error`, `failed` | Failure states. Inspect full detail and the producing job's outputs. | +| `paused` | Execution is blocked or paused. Inspect upstream inputs and pause information before blaming the tool command. | +| `skipped` | Execution was skipped, for example by a workflow condition. It is not itself a tool failure. | +| `stop`, `stopped`, `deleting`, `deleted` | Stopping or deletion states. Determine whether cancellation or deletion explains the missing result. | -## Rule Ordering And Levels +The API value for a stop in progress is `stop`, not `stopping`. Consumers use different terminal-state sets. Do not infer tool success from a workflow completion record, or infer permanent failure merely because polling stopped. -Galaxy evaluates explicit and preset rules in a defined order: +Output datasets have their own states, including `ok`, `error`, `paused`, `failed_metadata`, `deferred`, and `discarded`. `failed_metadata` identifies a metadata problem, while `deferred` denotes data awaiting materialization. Dataset deletion and purging are separate flags. For a red downstream output, trace its producing job and upstream datasets instead of treating job and dataset states as interchangeable. For mapped jobs, retain the failed member's job ID and collection element identifier. See [[galaxy-collection-semantics]]. -- Exit-code rules are evaluated before regex rules. -- Regexes search stdout and/or stderr case-insensitively. -- A fatal rule stops later checks. -- Warnings, log messages, and QC messages can produce `job_messages` without failing the job. +## Wrapper failure rules -Useful levels: +Galaxy loads an XML wrapper's failure rules from `` and ``, then checks the captured tool exit code and streams. Inspect the installed wrapper and its `profile` before interpreting a non-zero exit or stderr text. -| Level | Effect | +| Control | Behavior | |---|---| -| `log` | Records informational message; does not fail the job. | -| `qc` | Records QC message; does not fail the job. | -| `warning` | Records warning; does not fail the job. | -| `fatal` | Fails the job as a generic tool error. | -| `fatal_oom` | Fails as out-of-memory; runner behavior can use this for OOM handling/resubmission. | +| Omitted `detect_errors` or `detect_errors="default"` | Adds non-zero exit-code checks for profiles other than legacy `16.01` only when there is no `` element. An explicit `` disables this implicit preset, even if it contains no rules. | +| `detect_errors="exit_code"` | Adds fatal checks for negative and positive non-zero exit codes. Optional `oom_exit_code` adds an OOM-specific check before those ranges. | +| `detect_errors="aggressive"` | Adds non-zero exit checks and case-insensitive checks on both streams for OOM text, `exception:`, and `error:`. | +| `` | Defines explicit values or inclusive ranges and their severity. Open-ended ranges such as `1:` are supported. | +| `` | Searches selected tool streams case-insensitively. With no source specified, both stdout and stderr are searched. | +| `` | Requests `set -e` in the generated tool script. The default is true for profiles `20.09` and newer, false for older profiles. | -Low-confidence caveat: OOM resubmission behavior is runner/destination configuration dependent. The wrapper and output checker can classify OOM, but retry policy is not solely a wrapper property. +Explicit stdio rules are prepended to rules from an explicitly selected `exit_code` or `aggressive` preset. Adding a warning for an exit code does not override a later fatal preset rule matching that code. -## Job And Dataset States +When **no exit-code or regex rules exist**, the output checker falls back to treating any nonempty tool stderr as an error. In this fallback, a non-zero exit code with empty stderr does not itself trigger failure. Conversely, with configured rules, ordinary stderr text is not automatically fatal. -Job states relevant to workflow tests include: +Shell strictness and stdio checks serve different purposes. `set -e` can stop a multi-command script before later commands hide an earlier error, but follows the shell's exceptions and does not add `pipefail`. It does not guarantee that every component of a pipeline succeeded. Galaxy's command factory also bypasses script creation when the configured shell is `none`. -- In progress or queued: `new`, `queued`, `running`, `waiting`, `upload`, `resubmitted`. -- Success: `ok`. -- Failure or terminal problem: `error`, `failed`, `paused`, `stopped`, `deleted`, `skipped` depending context. +## Rule ordering and structured messages -Dataset states relevant to downstream failures include `ok`, `error`, `paused`, `failed_metadata`, `deferred`, and `discarded`. +Exit-code checks run before regex checks. Within each group, explicit rules precede presets. Regex rules check stderr before stdout when both are selected. -For workflow debugging, do not collapse job state and dataset state. A job can fail, its outputs can become error datasets, and a downstream workflow step can later fail because it consumes those datasets. +| XML level | Numeric `error_level` | Effect | +|---|---:|---| +| `log` | 1 | Records an informational message. | +| `qc` | 1.1 | Records a quality-control message. | +| `warning` | 2 | Records a warning. | +| `fatal` | 3 | Classifies a generic tool error. | +| `fatal_oom` | 4 | Classifies an out-of-memory error and stops further checking. | -## Stream And Message Fields +Log, QC, and warning matches do not fail the job by themselves. A generic `fatal` match does **not** stop all later checks. A later OOM match can upgrade the classification. The checker tracks the maximum severity, and level 4 ends checking. OOM classification can inform runner resubmission, but retry policy depends on the runner and destination configuration. -Galaxy distinguishes tool streams from job/runner streams: +Structured `job_messages` retain the matching rule evidence: -| Field | Meaning | +| Message `type` | Additional fields | |---|---| -| `tool_stdout` | stdout from the executed tool command. | -| `tool_stderr` | stderr from the executed tool command. | -| `job_stdout` | stdout from job wrapper/runner context. | -| `job_stderr` | stderr from job wrapper/runner context. | -| `stdout` | Combined stdout compatibility view. | -| `stderr` | Combined stderr compatibility view. | -| `job_messages` | Structured failure/warning messages produced by stdio and output checking. | -| `exit_code` | Process exit code observed by the runner. | +| `exit_code` | `exit_code`, `code_desc`, `desc`, `error_level`. | +| `regex` | `stream`, `match`, `code_desc`, `desc`, `error_level`. | +| `max_discovered_files` | `desc`, `error_level` for output-discovery failures, including too many files or an output name that is too long. | -Prefer `job_messages` plus separate streams for reference-quality failure interpretation. Combined `stdout` and `stderr` are useful for humans but lose provenance. +Regex `match` text is truncated after 256 characters when longer. `code_desc` is the rule's description, while `desc` includes the detected severity and explanation. Preserve the raw message rather than reducing it to “stderr contained an error.” Fallback stderr failure and runner errors need not produce stdio messages. -## API Surfaces +## Job API reference -Useful job APIs from Galaxy source under `~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/jobs.py`: +Paths are relative to the server URL and use encoded API IDs. -| API | Use | +| Method and path | Result | |---|---| -| `GET /api/jobs` | Filter jobs by state, tool id, workflow id, invocation id, history id, etc. | -| `GET /api/jobs/{job_id}` | Basic job detail. | -| `GET /api/jobs/{job_id}?full=true` | Full job detail, including streams and `job_messages`. | +| `GET /api/jobs` | Lists accessible jobs with filters including state, tool ID, history ID, workflow ID, and invocation ID. | +| `GET /api/jobs/{job_id}` | Basic job detail, including state, tool identity, and exit code. | +| `GET /api/jobs/{job_id}?full=true` | Adds separate and combined streams, `job_messages`, and dependencies. | | `GET /api/jobs/{job_id}/stdout` | Combined stdout as plain text. | | `GET /api/jobs/{job_id}/stderr` | Combined stderr as plain text. | -| `GET /api/jobs/{job_id}/console_output` | Live or stored tool stdout/stderr, useful while a job is running. | -| `GET /api/jobs/{job_id}/inputs` | Input datasets for the job. | -| `GET /api/jobs/{job_id}/outputs` | Output datasets and output collections. | -| `GET /api/jobs/{job_id}/metrics` | Job metrics, if available. | -| `GET /api/jobs/{job_id}/common_problems` | Known simple problems such as empty or duplicate inputs. | +| `GET /api/jobs/{job_id}/console_output` | Tool console output, subject to the requirements below. | +| `GET /api/jobs/{job_id}/inputs` | Job input datasets. | +| `GET /api/jobs/{job_id}/outputs` | Job output datasets and collections. | +| `GET /api/jobs/{job_id}/metrics` | Job metrics subject to access policy and availability. | +| `GET /api/jobs/{job_id}/common_problems` | Simple checks such as empty or duplicate inputs. These are clues, not a complete diagnosis. | -Admin-only or admin-enriched fields can include command line, traceback, destination, runner, handler, external id, and destination parameters. Do not assume a normal workflow-testing API key can retrieve everything. +`console_output` requires four query parameters: `stdout_position`, `stdout_length`, `stderr_position`, and `stderr_length`. They define the starting offsets and lengths of the requested stream ranges. -## Durable Reference Use +The destination must enable `live_tool_output_reporting`, including when retrieving a finished job through this endpoint. While running, Galaxy reads the requested ranges from tool stream files. Otherwise it returns stored tool streams. Full job detail exposes stored failure evidence independently of live reporting. -This note should inform generated skills when they need to preserve or inspect tool-level failure evidence: +Full detail does not grant administrator access. System details such as traceback, runner, destination, handler, and external job ID can require an administrator. Command-line exposure also depends on server configuration, and full-detail `job_metrics` are added for administrators. A normal workflow-testing key may therefore lack some server or runner evidence. -- A concrete workflow step should preserve tool id, version, input labels, output labels, datatype and collection shape so a later job failure can be traced back to the authoring decision. -- A runtime failure should not be classified from stderr alone if `job_messages`, exit code, and wrapper stdio rules are available. -- A workflow invocation failure may be caused by an upstream job, but invocation APIs are not a substitute for full job detail. +## Diagnostic pitfalls and verification limits -## Low-Confidence Or Version-Sensitive Points +- **Stderr is evidence, not a verdict.** Warnings can be harmless under configured stdio rules. An aggressive regex can also classify text as fatal despite exit code zero. Compare streams with the installed wrapper's rules. +- **Exit code zero is not scientific success.** Output collection, metadata, and assertions can still fail. Preserve output states and Planemo's `output_problems` alongside job records. +- **A paused consumer may hide an upstream failure.** Follow dependencies and producing jobs before changing the consumer's parameters. +- **No messages is not a clean bill of health.** A runner, fallback stderr check, or output-processing failure can leave no matching stdio record. Keep all available streams and state evidence. +- **Resubmission changes the context.** Record runner and destination details when available and distinguish an OOM classification from a configured retry decision. -- Tool profile defaults changed over time. Prefer current Galaxy source and wrapper profile over old generic rules. -- Some docs describe failure messages as stream text; current code preserves structured `job_messages` separately. -- OOM handling depends on job runner and destination configuration. -- Planemo may print or summarize job detail differently from raw Galaxy APIs; see [[planemo-workflow-test-architecture]]. +The parser, checker, state definitions, and API contracts were inspected in the pinned Galaxy source. No live run verified every runner's stream capture, OOM resubmission, or access configuration. Check the deployed server and installed wrapper before applying version-sensitive conclusions. diff --git a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-workflow-invocation-failure-reference.md b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-workflow-invocation-failure-reference.md index 73d45d3..05128f3 100644 --- a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-workflow-invocation-failure-reference.md +++ b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/galaxy-workflow-invocation-failure-reference.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-tool-job-failure-reference]]" - "[[planemo-workflow-test-architecture]]" @@ -16,123 +16,113 @@ related_molds: - "[[debug-galaxy-workflow-output]]" - "[[validate-galaxy-workflow]]" sources: - - "~/projects/repositories/galaxy/lib/galaxy/schema/invocation.py" - - "~/projects/repositories/galaxy/lib/galaxy/workflow/run.py" - - "~/projects/repositories/galaxy/lib/galaxy/workflow/modules.py" - - "~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/workflows.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/invocation.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/workflow/run.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/workflow/modules.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/model/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/api/workflows.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/services/invocations.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/jobs.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/galaxy/activity.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/runnable.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/reports/report_markdown.tpl" summary: "Reference for Galaxy workflow invocation states, messages, failure reasons, and invocation API surfaces." --- -# Galaxy Workflow Invocation Failure Reference +# Galaxy workflow invocation failure reference -This note describes workflow-level failure surfaces in Galaxy. It is separate from [[galaxy-tool-job-failure-reference]] because invocation state answers whether Galaxy could schedule and drive the workflow, while job state answers whether individual tool jobs succeeded. +A workflow invocation records Galaxy's progress in scheduling a workflow. Its jobs record execution of the tools. A `scheduled` invocation has finished scheduling, and a `completed` invocation has terminal jobs, but neither state proves that the jobs succeeded or the outputs passed their tests. -## Invocation Versus Job Failure +For a failed workflow test, preserve the invocation state and messages alongside job states and output problems. Tool exit codes, streams, and structured job messages are covered in [[galaxy-tool-job-failure-reference]]. [[planemo-workflow-test-architecture]] describes how Planemo collects the run evidence, and [[debug-galaxy-workflow-output]] uses it to classify failures before recommending repairs. -Important distinction: +## Invocation evidence -- Invocation state says whether Galaxy scheduled, cancelled, failed, or completed the workflow invocation. -- Job state says whether jobs produced by invocation steps succeeded or failed. -- Invocation messages explain scheduler/evaluation/cancellation problems. -- Step states usually describe scheduling progress, not actual job success, unless a legacy serialization mode substitutes job state. +Invocation records expose scheduling state, structured messages, step jobs, and registered outputs. Job summaries aggregate execution states, while full job records contain exit codes, streams, and structured job messages. These records describe different parts of a workflow failure. -A robust workflow test reference should inspect both invocation APIs and job APIs. +A run can fail during staging or an API request before Galaxy creates an invocation. An absent invocation ID is not an invocation failure reason. -## Invocation States +## Invocation and step states -Galaxy invocation states are defined in `~/projects/repositories/galaxy/lib/galaxy/schema/invocation.py` and related model code. - -| State | Meaning | +| Invocation state | Meaning for diagnosis | |---|---| -| `new` | Newly created invocation. | -| `requires_materialization` | Deferred or undeferred inputs must materialize before scheduling can continue. | -| `ready` | Ready for another scheduler iteration. | -| `scheduled` | Workflow has been scheduled. Older clients may treat this as successful terminal state. | -| `cancelling` | Scheduler will cancel jobs and subworkflow invocations. | -| `cancelled` | Cancellation terminal state. | -| `failed` | Scheduler/evaluation/materialization failure terminal state. | -| `completed` | Newer completion state after all jobs reach terminal states. | +| `new` | Newly created invocation awaiting scheduling. | +| `requires_materialization` | Inputs need materialization before scheduling can continue. This is a waiting state, not itself a failure. | +| `ready` | Ready for another scheduling iteration. | +| `scheduled` | All steps have been scheduled. Tool jobs can still be running or have failed. | +| `cancelling` | Cancellation requested. The scheduler will cancel jobs and subworkflow invocations. | +| `cancelled` | Invocation cancelled. Inspect cancellation messages. | +| `failed` | Invocation failed during scheduling or evaluation. Inspect messages and upstream jobs or inputs. | +| `completed` | All jobs reached terminal states. Those states include `ok`, `error`, `deleted`, `skipped`, `paused`, and `stopped`. | -Version caveat: `completed` may not be uniformly available across all Galaxy/Planemo combinations. Planemo accounts for both `scheduled` and `completed` as success-like invocation states. +Completion is distinct from successful execution. Inspect the job-state counts even when the invocation is `completed`, then check the expected outputs and assertions. Older servers may expose `scheduled` without the newer completion tracking. -## Invocation Message Reasons +Ordinary invocation step states are `new`, `ready`, and `scheduled`. They describe scheduling, not tool execution. The deprecated `legacy_job_state=true` option substitutes job state and can return one step entry per mapped job. It can also show incomplete information during partial scheduling. The default serialization retains workflow step identity, while each step's `jobs` exposes execution states. -Structured invocation messages are the main durable surface for workflow-level failures. +## Structured invocation messages -| Reason | Typical meaning | -|---|---| -| `dataset_failed` | Dataset input or upstream dataset was failed/unusable. | -| `collection_failed` | Collection was failed, incompletely populated, or unusable. | -| `job_failed` | Upstream dependent job failed. | -| `output_not_found` | Expected step/subworkflow output could not be resolved. | -| `expression_evaluation_failed` | Workflow expression or condition evaluation failed. | -| `when_not_boolean` | Conditional `when` expression did not return a boolean. | -| `unexpected_failure` | Scheduler or evaluation failure without a more specific safe reason. | -| `workflow_parameter_invalid` | Workflow parameter validation failed. | -| `step_input_deleted` | Step input dataset/collection was deleted. | -| `history_deleted` | Invocation history was deleted, causing cancellation. | -| `user_request` | User/API cancellation. | -| `cancelled_on_review` | Pause/review step rejected continuation. | -| `workflow_output_not_found` | Warning: declared workflow output was not found. | - -Message records can include affected workflow step id, dependent step id, job id, HDA/HDCA ids, output name, and nested workflow step index path. These fields matter more than free-text messages for reference-quality diagnosis. - -## Common Workflow-Level Failure Surfaces - -| Surface | Reference signal | -|---|---| -| Request-time validation | API returns an error before useful invocation state exists. | -| Input materialization | Invocation enters `requires_materialization`, then may fail with `dataset_failed`. | -| Conditional evaluation | Invocation messages include `expression_evaluation_failed` or `when_not_boolean`. | -| Missing output graph edge | Invocation messages include `output_not_found`. | -| Deleted/purged inputs | Invocation messages include `step_input_deleted` or `dataset_failed`. | -| Upstream job failure | Invocation messages or summaries point to `job_failed`; job API has details. | -| Collection population failure | Invocation messages include `collection_failed`; inspect HDCA and mapped jobs. | -| Pause/review rejection | Cancellation reason `cancelled_on_review`. | -| User/history cancellation | Cancellation messages `user_request` or `history_deleted`. | -| Warning-only missing output | `workflow_output_not_found` warning; may later become Planemo assertion failure. | - -## API Surfaces - -Useful invocation APIs from `~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/workflows.py`: - -| API | Use | +The `messages` list contains failure, cancellation, and warning records. The `reason` and associated IDs identify the failure and affected resources. A missing free-text explanation does not mean there is no diagnosable failure. + +| Reason | Category | Evidence to inspect | +|---|---|---| +| `dataset_failed` | Failure | `hda_id`, affected step, and optional dependent step. Inspect dataset state and the producing job or materialization failure. | +| `collection_failed` | Failure | `hdca_id` and dependent step. Inspect collection population and member jobs. | +| `job_failed` | Failure | `job_id` and dependent step. Full job detail contains execution evidence. This reason identifies a dependency failure, not every failed job in a workflow. | +| `output_not_found` | Failure | `output_name` and dependent step. A required step output could not be resolved. Check the connection and producer. | +| `expression_evaluation_failed` | Failure | Affected step. Check expressions, conditions, or `pick_value` selection requirements. Detailed expression errors can be withheld. | +| `when_not_boolean` | Failure | Affected step and `details`. The `when` result was not a boolean. | +| `unexpected_failure` | Failure | Optional affected step and `details`. If the record lacks detail, an administrator may need scheduler logs. | +| `workflow_parameter_invalid` | Failure | Parameter input step and validator `details`. Compare the supplied value with the workflow input constraints. | +| `step_input_deleted` | Failure | `hda_id` or `hdca_id`, affected step, and `details`. A referenced dataset or collection was deleted. | +| `history_deleted` | Cancellation | `history_id`. The invocation history was deleted. | +| `user_request` | Cancellation | Cancellation was requested by the user or API. | +| `cancelled_on_review` | Cancellation | Affected pause step. Review rejected continuation. | +| `workflow_output_not_found` | Warning | Affected step and `output_name`. A declared workflow output was not registered. Check its producer and output declaration. | + +`output_not_found` is an evaluation failure. `workflow_output_not_found` is a warning about a declared workflow output. A warning alone does not fail the invocation, but a test requiring that output can still fail. Check optional or conditional output behavior before assuming a tool failed. + +Galaxy deliberately omits potentially sensitive exception or expression text on some `unexpected_failure` and `expression_evaluation_failed` paths. Other paths provide safe `details`. Preserve the structured record and request server logs when needed instead of inferring the missing explanation. + +### Match message steps to invocation steps + +Message `workflow_step_id` and `dependent_workflow_step_id` identify integer workflow step indices. Match them to the invocation step's `order_index` within the relevant workflow. They are different from the encoded workflow database ID also named `workflow_step_id` in a step response. + +An invocation step's encoded `id` is the value accepted by the step API. Its `job_id` identifies a single job when present, while `jobs` includes mapped jobs with `step_details=true`. For a subworkflow, `subworkflow_invocation_id` identifies the nested invocation. The message's `workflow_step_index_path` supplies enclosing subworkflow context and excludes the failing step itself. Preserve the path along with the message instead of treating its step index as a top-level step. + +## Invocation API reference + +Paths below are relative to the Galaxy server URL. IDs in paths are encoded API IDs. + +| Method and path | Result or action | |---|---| -| `GET /api/invocations/{invocation_id}` | Invocation state, messages, inputs, outputs, and steps. | -| `GET /api/invocations/{invocation_id}?step_details=true` | Richer step detail, jobs, outputs, and output collections. | +| `GET /api/invocations/{invocation_id}` | Invocation state, messages, inputs, registered outputs, output values, and steps. | +| `GET /api/invocations/{invocation_id}?step_details=true` | Adds step jobs, outputs, and output collections. | | `GET /api/invocations/steps/{step_id}` | Detail for one invocation step. | -| `GET /api/invocations/{invocation_id}/steps/{step_id}` | Step detail scoped to invocation. | -| `GET /api/invocations/{invocation_id}/jobs_summary` | Job state summary across the invocation. | -| `GET /api/invocations/{invocation_id}/step_jobs_summary` | Job state summary per workflow invocation step. | -| `GET /api/invocations/{invocation_id}/completion` | Completion record, if available. | -| `DELETE /api/invocations/{invocation_id}` | Cancel invocation. | -| `PUT /api/invocations/{invocation_id}/steps/{step_id}` | Continue or cancel pause/review step. | -| `GET /api/invocations/{invocation_id}/report` | Invocation report surface. | +| `GET /api/invocations/{invocation_id}/steps/{step_id}` | Alias of the preceding endpoint. It ignores `invocation_id`, so the path does not establish step membership. | +| `GET /api/invocations/{invocation_id}/jobs_summary` | Aggregate job counts in `states` and a `populated_state`. | +| `GET /api/invocations/{invocation_id}/step_jobs_summary` | Job-source summaries, including mapped job groups and recursive subworkflow jobs. | +| `GET /api/invocations/{invocation_id}/completion` | Completion time, final job-state summary, and executed completion hooks. Returns JSON `null` when there is no completion record. | +| `GET /api/invocations/{invocation_id}/report` | Generated invocation report, separate from raw invocation messages and job records. | +| `DELETE /api/invocations/{invocation_id}` | Requests cancellation. This changes the run. | +| `PUT /api/invocations/{invocation_id}/steps/{step_id}` | Sets pause/review action using `{"action": true}` to proceed or `{"action": false}` to reject continuation. This changes the run. | Workflow aliases under `/api/workflows/{workflow_id}/invocations/...` also exist. -## Planemo Caveats - -Planemo polls invocation and job state, but its user-facing output may not expose all structured invocation messages. Treat Planemo terminal output as a summary; use raw invocation and job APIs when the failure class matters. - -Likely noisy boundaries: +Despite its name, `step_jobs_summary` is not a dictionary keyed by workflow step index. Each entry's `model` identifies the kind of object summarized, and `id` belongs to that kind. Step details associate a `Job` or `ImplicitCollectionJobs` summary with the workflow step. A mapped step can have many jobs, and an aggregate state cannot identify which collection member failed. -- Planemo may show failed job details but omit structured invocation messages. -- `step_jobs_summary` is useful in Galaxy but not necessarily surfaced directly by Planemo output. -- Subworkflow paths can be represented in Galaxy message fields but may not be displayed by Planemo. -- A workflow can be `completed` or `scheduled` while one or more jobs failed; job summaries must still be inspected. +## Diagnostic pitfalls -## Durable Reference Use +- **A downstream message may describe an upstream defect.** `dataset_failed`, `collection_failed`, and `job_failed` can identify the consuming step. Follow the dependent step or resource ID back to the producer before changing that consumer. +- **A waiting invocation is not automatically broken.** For `ready` or `requires_materialization`, inspect inputs, jobs, and pause actions. Preserve timeout duration if polling ends without a terminal result. +- **Successful scheduling does not validate output content.** Inspect job states, collection population, registered output labels, and assertion failures independently. See [[galaxy-collection-semantics]] for collection shape and member diagnostics. +- **One step is not necessarily one job.** Preserve failed member jobs for mapped collections and nested invocation IDs for subworkflows. Do not collapse them into a single top-level step state. +- **Missing completion data does not prove failure.** A `null` completion response means no record exists. An older server may not implement the endpoint at all. Use invocation and job states to diagnose the run. -Use this note when a workflow run fails before or around job scheduling, when outputs are missing without an obvious tool stderr cause, or when Planemo only reports a generic invocation failure. +## Planemo evidence and version limits -The goal is not to prescribe a single repair loop. The goal is to preserve which Galaxy API surface proves the failure mode so later Molds can decide whether the defect belongs to data-flow, template, concrete step implementation, workflow test assertions, or runtime environment. +Planemo 0.75.47 collects invocation messages in `invocation_details`. In its JSON test report, the path is `tests[].data.invocation_details.details.messages`. Its Markdown report template also renders those messages. -## Verification Gaps +Preserve `tool_test_output.json`, other reports when generated, and the Galaxy and Planemo versions. Start with the test's `invocation_details`, `execution_problem`, and `output_problems` rather than a terminal summary alone. Planemo's success check combines history state with invocation state and accepts both `scheduled` and `completed` as successful scheduling states. It does not make either invocation state proof of passing output assertions. -Actual runs should verify: +A staging or execution exception can still leave incomplete invocation details. Missing-output errors can come from label drift, a missing workflow output declaration, conditional absence, failed jobs, or a download problem. Job, collection, and nested invocation records can distinguish these causes when the collected artifacts are incomplete. -- Whether Planemo surfaces invocation `messages` for each reason. -- Whether `completed` and `/completion` are populated in the target Galaxy used by tests. -- How mapped collections and subworkflow failures appear in `step_jobs_summary`. -- How warning-only `workflow_output_not_found` interacts with Planemo assertions. +The state and endpoint definitions above describe the pinned Galaxy source. No live run was used to verify every message reason, collection mapping, or subworkflow representation. Check the deployed server's responses before relying on completion tracking or a particular report layout. diff --git a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-asserts-idioms.md index 6d0fef7..bd1160e 100644 --- a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-asserts-idioms.md +++ b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-asserts-idioms.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-11 -revision: 6 +revised: 2026-10-21 +revision: 10 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,231 +16,109 @@ related_notes: - "[[validate-tests]]" - "[[iwc-tabular-operations-survey]]" - "[[galaxy-discover-datasets]]" -summary: "Decision and idiom guide for picking planemo workflow-test assertions: which family per output type, how to size tolerances, when to validate." +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." --- -# Planemo asserts: idiom and decision guide +# Choosing Planemo workflow-test assertions -Companion to [[iwc-test-data-conventions]] (input shapes), [[galaxy-workflow-testability-design]] (workflow structure before test YAML exists), and [[iwc-shortcuts-anti-patterns]] (what's accepted vs smell). This note is forward-looking: when authoring a new `-tests.yml`, which assertion family fits which output, and what the recommended tolerances and operators are. +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. -The **vocabulary itself is not restated here** — every assertion's parameter list, types, defaults, required fields, and Python docstring is rendered from the test-format JSON Schema at [[tests-format]]. Assertion names below deep-link into that page (e.g. [[tests-format#has_text_model|has_text]] jumps straight to that `$def`). +## 1. Choose by the property, then the output type -## 1. Choose by output type +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | -The single most useful decision table. Pick the row that matches the file format the workflow emits; default to the recommended assertion family. +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. -| Output type | Default assertion family | Why | Fallback | -|---|---|---|---| -| **Plain text reports / logs** (FastQC summary, MultiQC text section) | [[tests-format#has_text_model|has_text]] (substring on a known stable token) + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Substring is robust across versions; line count catches truncation | [[tests-format#has_text_matching_model|has_text_matching]] (regex) when token text changes per run | -| **HTML reports** (MultiQC HTML, custom dashboards) | [[tests-format#has_text_model|has_text]] against stable section names | HTML embeds timestamps and asset hashes; byte-diff is hopeless | [[tests-format#has_size_model|has_size]] + large `delta` as last resort | -| **Tabular** (TSV, CSV, BED-like) | [[tests-format#has_n_columns_model|has_n_columns]] + [[tests-format#has_text_model|has_text]] for headers + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Schema-shape + content probes; tolerant to row reordering | [[tests-format#has_line_matching_model|has_line_matching]] for a known canonical row | -| **VCF** | `compare: diff` with `lines_diff: 6` | The `lines_diff: 6` constant matches the typical VCF header preamble that embeds `##fileDate=` and `##source=` | [[tests-format#has_text_matching_model|has_text_matching]] for known variants | -| **BAM** | [[tests-format#has_size_model|has_size]] + [[tests-format#has_archive_member_model|has_archive_member]] (BAM is a gzipped block format) | Headers contain `@PG` lines with command lines and Galaxy job IDs; never byte-diff | Convert to SAM in a downstream step and assert on text | -| **FASTA** (deterministic — assemblies, consensus) | `file:` exact comparison or [[tests-format#has_text_model|has_text]] for known sequence | Output is byte-stable when the upstream tool is deterministic | [[tests-format#has_n_lines_model|has_n_lines]] if line wrapping varies | -| **FASTA** (non-deterministic — RepeatModeler libraries) | `compare: sim_size` with large `delta:` | Family content varies run-to-run | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | -| **FASTQ** (rare as workflow output) | [[tests-format#has_n_lines_model|has_n_lines]] (must be multiple of 4) | Quality scores are read-id-dependent | `compare: sim_size` | -| **JSON** (deterministic — config dumps, params) | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] / [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] | Surgical: assert on the property you care about, ignore the rest | `compare: diff` if the JSON is fully canonical | -| **JSON** (stochastic — HyPhy stats, MCMC results) | `has_text: text: "{"` (existence-only) | Embedded floats break any structural assertion; see [[iwc-shortcuts-anti-patterns]] §1 | [[tests-format#has_h5_keys_model|has_h5_keys]]-equivalent for top-level structural keys | -| **HDF5 / AnnData** | [[tests-format#has_h5_keys_model|has_h5_keys]] + [[tests-format#has_h5_attribute_model|has_h5_attribute]] for known structure | Asserts shape, not values — values are float-noisy | [[tests-format#has_size_model|has_size]] with `delta:` | -| **XML** | [[tests-format#is_valid_xml_model|is_valid_xml]] + [[tests-format#has_element_with_path_model|has_element_with_path]] + `element_text_is`/`element_text_matches` | Surgical structural probes; byte-diff fails on whitespace/attribute-order | [[tests-format#has_n_elements_with_path_model|has_n_elements_with_path]] for cardinality | -| **PNG / image plots** | [[tests-format#has_image_width_model|has_image_width]] + [[tests-format#has_image_height_model|has_image_height]] + [[tests-format#has_size_model|has_size]] with `delta:` (5–10%) | Catches "rendered something with the right dimensions"; corpus norm | [[tests-format#has_image_center_of_mass_model|has_image_center_of_mass]] if mask/segmentation outputs are deterministic | -| **TIFF / multipage images** | [[tests-format#has_image_frames_model|has_image_frames]] + [[tests-format#has_image_channels_model|has_image_channels]] + [[tests-format#has_size_model|has_size]] with `delta:` | Same logic, channel/frame structure matters | — | -| **Archives** (zip, tar.gz) | `has_archive_member: path: "regex"` with nested `asserts:` | Asserts on a specific member; archive timestamps never byte-stable | [[tests-format#has_size_model|has_size]] with `delta:` | -| **GFF / GTF** | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` + [[tests-format#has_text_model|has_text]] for known feature | Tools reorder freely; hard to byte-diff | `has_n_columns: 9` (GFF spec) | -| **Cool / HiC matrices** | `compare: sim_size` with multi-MB `delta:` | Binary, run-to-run variance | [[tests-format#has_archive_member_model|has_archive_member]] for HDF5 component | +## 2. Compare operators and expected artifacts -When in doubt: start with [[tests-format#has_size_model|has_size]] + `delta_frac: 0.1`. It catches the catastrophic failure mode (empty / 10x bigger output). Then add a content probe. +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. -### 1a. If the assertion is too weak, revisit the workflow output +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. -Assertion choice sometimes reveals a workflow-design problem. If the only available output can support only a size check or image-dimension smoke test, check whether the workflow should expose a stronger checkpoint before settling for the weak assertion. +## 3. Set tolerances from the failure boundary -- Scanpy's plot outputs use size and image-dimension assertions, but the same workflow also exposes AnnData HDF5 checkpoints and a cluster-count table (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end uses coarse size bands for coverage/mapped-read outputs, but expression/count tables get stronger regex and exact-line checks (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. -Rule: if a translated workflow exposes only weakly assertable final reports, consult [[galaxy-workflow-testability-design]] and consider promoting a table/text/HDF5 checkpoint before writing the final assertions. +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. -## 2. The `compare:` operators +## 4. Text assertions and repeated checks -Top-level on a `file:` output assertion. Vocabulary, in decreasing strictness: - -- **`diff`** (default). Byte-for-byte equality with optional `lines_diff:` tolerance for a fixed number of header lines. Use only when the upstream tool is deterministic on fixed inputs *and* the output has no embedded timestamps, command lines, version banners, or hash-ordered Python-dict-style keys. -- **`re_match` / `re_match_multiline`**. Each line of the expected fixture is a regex that must match the corresponding output line. Useful when a few fields per row are timestamped but the rest is canonical. Rare in the corpus. -- **`contains`**. The expected fixture is a substring of the output. Cheap; weak. Prefer `asserts: has_text` for new code unless you genuinely have a multi-line block to assert as a whole. -- **`sim_size`**. Output file size matches the fixture's size within `delta:` (bytes) or `delta_frac:` (fraction). Use when the output is necessarily non-deterministic but its rough size is reproducible (RepeatModeler libraries, HiC matrices, Bayesian sampler outputs). - -Picking `lines_diff:`: count the mutable header lines in the output format. VCF: ~6 (`##fileformat`, `##fileDate`, `##source`, `##reference`, contig/info lines vary). SAM/text headers: count `@HD`/`@PG`/`@CO` lines. Set `lines_diff:` to the count exactly — looser values mask real diffs. - -## 3. Tolerance picking - -**`delta:`** is bytes (for [[tests-format#has_size_model|has_size]] and `compare: sim_size`) or absolute count (for [[tests-format#has_n_lines_model|has_n_lines]], [[tests-format#has_n_columns_model|has_n_columns]], [[tests-format#has_image_width_model|has_image_width]], etc.). Suffix multipliers documented in the schema — `1K`, `1M`, `1G` work. - -**`delta_frac:`** is a fraction (0.1 = 10%). Use when expected size scales with input volume. Three IWC tests use it (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`); the rest use absolute `delta:`. - -**Picking magnitudes** (from corpus survey in [[iwc-shortcuts-anti-patterns]] §2): - -- Image dimensions: `delta: 25–30` pixels (5% of typical matplotlib defaults). -- Image file size: `delta: 5K–60K` (5–10% of file size). -- Small text reports: `delta: 1K–10K`. -- HTML reports: `delta: 25K–100K`. -- BAM files: `delta: 1M–10M`. -- RepeatModeler / Bayesian sampler outputs: `delta: 30K–90M` (extreme, but justified by the underlying nondeterminism). - -**Heuristic for new outputs:** `delta_frac: 0.1` is a defensible default. Tighten if the output proves more deterministic than expected. - -## 4. Text family — [[tests-format#has_text_model|has_text]] vs [[tests-format#has_text_matching_model|has_text_matching]] vs [[tests-format#has_line_model|has_line]] vs [[tests-format#has_line_matching_model|has_line_matching]] vs [[tests-format#has_n_lines_model|has_n_lines]] - -All five are common; choose by what you're verifying. - -- **[[tests-format#has_text_model|has_text]]** — output contains the substring `text:`. Anywhere in the output, any number of times. Add `n:` / `min:` / `max:` to constrain occurrence count. Add `delta:` to allow slack on the count. -- **[[tests-format#has_text_matching_model|has_text_matching]]** — output matches the regex `expression:`. Use sparingly; prefer literal [[tests-format#has_text_model|has_text]] when you can. -- **[[tests-format#has_line_model|has_line]]** — output has at least one *line* matching `line:` exactly. Use when line boundaries matter (e.g. asserting on a specific row in a table). -- **[[tests-format#has_line_matching_model|has_line_matching]]** — same but with regex. -- **[[tests-format#has_n_lines_model|has_n_lines]]** — assert the line count is `n:` ± `delta:`. - -A common combination in IWC: `has_n_lines: n: 100, delta: 5` + `has_text: text: "expected_token"` — line-count sanity-check plus a content marker. This catches both truncation and content drift in one assertion pair. - -`negate: true` is supported on every assertion. Used for the "this output should NOT contain X" case. - -## 5. Collection output assertions (`element_tests:`) - -For Galaxy collection outputs, the test format keys element assertions by element identifier: +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. ```yaml -my_collection_output: - element_tests: - sample_1: +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: asserts: - has_text: - text: "expected" - sample_2: - file: test-data/expected_sample_2.txt -``` - -Optional `attributes:` at the collection level can assert on the produced collection shape: - -```yaml -my_collection_output: - attributes: {collection_type: list:list} - element_tests: - ... + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 ``` -Nested collections: outer `element_tests:` keyed by outer identifier; inner uses `elements:` (note plural, no `_tests` suffix on the inner). See [[iwc-test-data-conventions]] §2f for the live example. - -For a list-of-files where every element should pass the same minimal check, the existence-probe pattern (`has_text: "{"` for JSON; `has_size: min: 100` for any non-empty binary) is widely used and accepted in IWC. - -## 6. The validate-against-workflow inner loop - -A `-tests.yml` file can be **structurally invalid** in two distinct ways: - -1. **Schema-invalid** — wrong field names, wrong nesting, wrong types. Caught by the test-format JSON Schema. -2. **Workflow-incoherent** — schema-valid YAML, but the input/output labels don't match the actual workflow. Renaming an output in the `.ga` and forgetting to update its sibling `-tests.yml` produces this case. Planemo will surface it as an "output not found" error at test-runtime, but only after a full workflow run. - -The `@galaxy-tool-util/schema` npm package ships **two** validators that catch both cases statically — no Galaxy or Planemo invocation needed: +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. -- **`validateTestsFile(yaml)`** — runs the file against `tests.schema.json` (AJV). Reports schema violations with paths. -- **`checkTestsAgainstWorkflow(workflow, tests)`** — cross-checks a `.ga` / format2 workflow against a tests file: missing input labels, missing output labels, type incompatibilities (e.g. test supplies a `File` for a parameter typed `int`). +## 5. Collection and structured-output examples -Both are pure-JS, take milliseconds, and have no Galaxy dependency. Wire them into the inner authoring loop: +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" ``` -edit -tests.yml - → validateTestsFile() # schema gate - → checkTestsAgainstWorkflow(.ga, tests) # coherence gate - → planemo workflow_test_on_invocation # assertion gate (no full re-run) - → planemo test # full integration (slow) -``` - -The first two gates short-circuit cheap mistakes before a slow planemo run. They are the static-validation equivalent of `gxwf` for tests, and the `implement-galaxy-workflow-test` mold should reference them as its primary inner-loop tooling. Source: galaxy-tool-util-ts package, `src/test-format/index.ts` exports. - -## 7. Authoring loop — generation, then refinement - -Reviewer convention is to **generate** the initial `-tests.yml` rather than hand-write it. Two planemo subcommands cover this: - -- **`planemo workflow_test_init --from_invocation `** ([[planemo-workflow_test_init]]) — given a successful Galaxy invocation ID, emit a `-tests.yml` with a `job:` block that captures all inputs (with SHA-1 hashes) and an `outputs:` block with `file:` references to the actual outputs (downloaded into `test-data/`). Hand-tighten the assertions afterward. -- **`planemo workflow_test_on_invocation `** ([[planemo-workflow_test_on_invocation]]) — re-evaluate an edited `-tests.yml` against a saved invocation without re-running the workflow. The fast inner loop for assertion iteration; complements the static gates in §6. - -Together these cut the assertion-iteration cost dramatically. An agent should: - -1. Run the workflow once on usegalaxy.* (or local) to get a known-good invocation. -2. `--from_invocation` to bootstrap the test file. -3. Replace the autogenerated `file:` exact-comparison assertions with assertion-family-appropriate alternatives per §1. -4. [[planemo-workflow_test_on_invocation]] after each edit; full [[planemo-test]] at the end. - -## 8. What the schema gives you for free - -When the test-format schema lands as a Foundry-rendered note, the agent can consult any assertion's `$def` directly for: parameter types, defaults, required fields, the `that` discriminator constant, and the original Python docstring (carried through as `description`). This note does **not** restate that vocabulary — it complements it with the corpus-grounded *which-and-when*. - -What's still missing from the schema and worth keeping in research notes: - -- This decision table (§1) — output-type → assertion family. -- Tolerance magnitudes (§3) — corpus-derived defaults. -- The `validateTestsFile` / `checkTestsAgainstWorkflow` integration story (§6). -- Anti-pattern flags — see [[iwc-shortcuts-anti-patterns]]. -## 9. Common combinations (recipes) +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. -Six recipes worth memorizing. +## 6. Validate labels and schema before execution -**Stable text report (FastQC summary, simple stats).** -```yaml -my_report: - asserts: - has_n_lines: { n: 12, delta: 2 } - has_text: { text: "Total Sequences" } -``` - -**MultiQC HTML report.** -```yaml -multiqc_report: - asserts: - has_text: { text: "Filtered Reads" } - has_text: { text: "FastQC" } -``` +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: -**VCF (pinned tool, fixed reference).** -```yaml -called_variants: - file: test-data/expected.vcf - compare: diff - lines_diff: 6 +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga ``` -**Stochastic JSON (HyPhy-style).** -```yaml -hyphy_meme: - element_tests: - geneA: { asserts: { has_text: { text: "{" } } } - geneB: { asserts: { has_text: { text: "{" } } } -``` +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. -**Matplotlib plot.** -```yaml -umap_plot: - asserts: - has_size: { size: 68416, delta: 6000 } - has_image_width: { width: 601, delta: 30 } - has_image_height: { height: 429, delta: 25 } -``` +## 7. Create and refine the test -**AnnData (HDF5).** -```yaml -clustered_anndata: - asserts: - has_h5_keys: { keys: "obs/louvain" } - has_h5_keys: { keys: "var/highly_variable" } - has_h5_keys: { keys: "uns/rank_genes_groups" } - has_size: { size: 12000000, delta: 1500000 } -``` +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. -## 10. Cross-references +## 8. Source of the exact syntax -- [[galaxy-workflow-testability-design]] — decide which workflow outputs and checkpoints to expose before choosing assertions. -- [[iwc-test-data-conventions]] — input-side conventions (job inputs, collection shapes, `hashes:`, CVMFS). -- [[iwc-shortcuts-anti-patterns]] — accepted-vs-smell catalog and corpus prevalence; this note's mirror image. -- Test-format schema (`@galaxy-tool-util/schema` npm package) — authoritative vocabulary; will be vendored into a Foundry-rendered schema note. See `content/meta/casting.md` for the casting story. -- Planemo test-format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- [[galaxy-xsd]] — Galaxy XSD assertion source of truth, vendored from upstream. -- Tightening of the schema and Pydantic source: [galaxyproject/galaxy#22566](https://github.com/galaxyproject/galaxy/pull/22566). -- TS schema sync into npm: [jmchilton/galaxy-tool-util-ts#75](https://github.com/jmchilton/galaxy-tool-util-ts/pull/75). +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-workflow-test-architecture.md b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-workflow-test-architecture.md index 59b3c25..72583a5 100644 --- a/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-workflow-test-architecture.md +++ b/plugins/foundry-skills/skills/debug-galaxy-workflow-output/references/notes/planemo-workflow-test-architecture.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-11 -revision: 3 +revised: 2026-10-26 +revision: 4 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[galaxy-tool-job-failure-reference]]" @@ -23,6 +23,8 @@ sources: - "~/projects/repositories/planemo/planemo/galaxy/activity.py" - "~/projects/repositories/planemo/planemo/galaxy/invocations" - "~/projects/repositories/planemo/planemo/galaxy/config.py" + - "~/projects/repositories/planemo/planemo/galaxy/serve.py" + - "~/projects/repositories/planemo/planemo/engine/galaxy.py" summary: "Reference for Planemo workflow test/run architecture, Galaxy modes, API polling, and noisy failure boundaries." --- @@ -94,6 +96,20 @@ Important operations: For workflows, Planemo invokes Galaxy using input labels/names. Stable generated labels are therefore important for both test execution and debugging. +### Workflow Import Failures + +Planemo imports the workflow while it brings Galaxy up for the run (`config.install_workflows()`, called from `_serve()` in `planemo/galaxy/serve.py` for managed Galaxy and from `ExternalGalaxyEngine.ensure_runnables_served` in `planemo/engine/galaxy.py` for external Galaxy, as of planemo 0.75.47). For gxformat2 it converts the workflow to a native dict client-side first, so Galaxy's own gxformat2 conversion never runs. That is outside the per-test `execute()` wrapper that turns exceptions into an `ErrorRunResponse`. When Galaxy rejects the import request, BioBlend raises `ConnectionError`, the test command exits non-zero, and Planemo writes neither the `--test_output_json` report nor the HTML report. No workflow, history, or invocation id exists. + +The evidence then lives in three places: + +| Evidence | What it holds | +|---|---| +| Upload response | HTTP status and body. A 400 usually names the failing field or validation rule. A 500 body rarely explains the cause. | +| Galaxy server log | The traceback for the failing request, the only place a 500's cause appears. Managed Galaxy streams its log into the Planemo terminal output and keeps it on disk. External Galaxy keeps it on the server. | +| Planemo terminal log | The BioBlend exception and the point in the run where it was raised. | + +Re-importing the same converted dict as the same user on the same server reproduces the failure without the rest of the Planemo run, which makes it the cheap loop for isolating a trigger. + Workflow testability design guidance lives in [[galaxy-workflow-testability-design]]. This architecture note only records why Planemo makes labels and workflow-level outputs operationally important. ## Structured Artifacts @@ -121,6 +137,7 @@ Planemo transforms and summarizes Galaxy failure information. These are likely i - Polling may summarize “at least one job is in error” while detailed job evidence is printed elsewhere or stored separately. - Missing outputs become assertion/output problems, which can conflate label mismatch, workflow output omission, optional output absence, failed invocation, or output download issue. - Output download failures may be logged but not always propagated as the most specific failure. +- A rejected workflow import aborts the run before any structured report is written, so terminal and server logs are the only record. - External Galaxy without an admin key may defer missing-tool problems until runtime. - `allow_tool_state_corrections=True` can let Galaxy adjust tool state during invocation, which is useful but can mask definition drift. - `--no_wait` is not suitable for debug conclusions because invocation creation can succeed before jobs fail. @@ -135,7 +152,7 @@ For [[run-workflow-test]]: For [[debug-galaxy-workflow-output]]: -- Start from structured Planemo output. +- Start from structured Planemo output. When there is none because the workflow import failed, start from the upload response and the Galaxy server log. - For missing-output failures, check label drift and omitted workflow-level outputs before assuming tool failure. - If the failure is job-level, inspect Galaxy job APIs described in [[galaxy-tool-job-failure-reference]]. - If the failure is invocation-level, inspect Galaxy invocation APIs described in [[galaxy-workflow-invocation-failure-reference]]. diff --git a/plugins/foundry-skills/skills/discover-shed-tool/SKILL.md b/plugins/foundry-skills/skills/discover-shed-tool/SKILL.md index adb3cce..683d19a 100644 --- a/plugins/foundry-skills/skills/discover-shed-tool/SKILL.md +++ b/plugins/foundry-skills/skills/discover-shed-tool/SKILL.md @@ -21,8 +21,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli @@ -39,19 +39,20 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `galaxy-tool-pin.json` before returning it: run `foundry validate-galaxy-tool-discovery galaxy-tool-pin.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-tool-discovery galaxy-tool-pin.json`. This checks artifact `galaxy-tool-pin` against the galaxy-tool-discovery schema. +- Before starting, check that the installed validator carries the galaxy-tool-discovery schema this bundle was cast with: run `foundry validate-galaxy-tool-discovery --schema-sha256 0933a77437e523405e72d2b7ceacede6abee6cc51cc4c6da376d939a1439a2ee` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `galaxy-tool-pin.json` before returning it: run `foundry validate-galaxy-tool-discovery --schema-sha256 0933a77437e523405e72d2b7ceacede6abee6cc51cc4c6da376d939a1439a2ee galaxy-tool-pin.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-tool-discovery --schema-sha256 0933a77437e523405e72d2b7ceacede6abee6cc51cc4c6da376d939a1439a2ee galaxy-tool-pin.json`. This checks artifact `galaxy-tool-pin` against the galaxy-tool-discovery schema. ## Procedure Discover whether the Galaxy Tool Shed already publishes a wrapper for the tool a workflow step needs, and resolve the discovery to a `(owner, repo, tool_id, version, changeset_revision)` quintuple that downstream steps can pin and cache. -This skill is the **Tool Shed leg** of the `discover-or-author` branch in Galaxy-targeting per-step pipelines. On a hit, the skill recommends a pin and exits successfully. On a miss (or a low-quality hit), it falls through to author-galaxy-tool-wrapper. The branch itself is harness logic; this skill owns only the discovery half. +This skill is the **Tool Shed leg** of the `discover-or-author` branch in Galaxy-targeting per-step pipelines. On a hit, the skill recommends a pin and exits successfully. On a miss, it falls through to author-galaxy-tool-wrapper. The branch itself is harness logic; this skill owns only the discovery half. ### Inputs The skill expects, per step: -- A free-text **need** describing what the step should do (typically a one-line description of the tool, plus any constraints — file format in/out, container language, license preferences). +- A free-text **need** describing what the step should do (typically a one-line description of the tool, plus any constraints — file format in/out, container language, license preferences). It names the source executable when there is one. - Optional **owner hint** (e.g. `devteam`, `iuc`) when the caller has a strong prior. - Optional **exact-name hint** when the caller knows the canonical XML id. @@ -83,8 +84,10 @@ A structured recommendation object, JSON-shaped: `status` semantics: - `hit` — recommend pinning. Caller should cache and proceed. -- `weak` — candidate exists but the skill is not confident (e.g. only help-text matched, multiple owners with similar tools, deprecated repo, stale-index suspicion). Caller should confirm or fall through. -- `miss` — no usable hit. Caller falls through to author-galaxy-tool-wrapper. +- `weak` — candidate exists but the skill is not confident (e.g. only help-text matched, multiple owners with similar tools, deprecated repo, stale-index suspicion). Caller confirms the candidate before pinning; it never falls through to authoring. +- `miss` — no wrapper runs the executable or operation the need names. Caller falls through to author-galaxy-tool-wrapper. + +Classify on what a wrapper runs, not on fit. Fit depends on the step's ports, which discovery doesn't see, and implement-galaxy-tool-step checks it after summarize. A wrapper that runs the named executable is never a `miss`. ### Procedure @@ -92,7 +95,7 @@ The skill follows the gxwf-shaped discover-and-pin chain. **It does not call the #### 1. Search -Issue tool-search with the need's keywords. Start narrow: +Issue tool-search with the need's keywords, and with the executable when the need names one. Start narrow: ``` gxwf tool-search "" --json --max-results 10 @@ -110,7 +113,7 @@ For each hit, score on: - **Recency.** Recent `last_updated` strengthens a hit; very old wrappers can still be valid but warrant the `weak` classification. - **Duplicates across repos.** Two owners can publish wrappers with the same XML id. Either pick the maintained one or downgrade to `weak` and surface the choice. -Drop hits from deprecated repos when detectable. Note: deprecated repos can still appear in shed search results until the next index rebuild — see component-tool-shed-search §6. +Drop data-manager hits (usually `data_manager_*` repos): they're admin-only and write a tool data table, not a dataset a step can consume. Drop hits from deprecated repos when detectable. Note: deprecated repos can still appear in shed search results until the next index rebuild — see component-tool-shed-search §6. #### 3. Resolve to a pinnable version @@ -137,7 +140,7 @@ Prefer `--latest` so the newest changeset publishing that version wins (tool ver Combine the scored hit and the resolved pin into the recommendation object above: - One dominant hit + clean version+changeset resolution → `hit`. - Multiple plausible hits, ambiguous owner, deprecated suspicion, or only-help-text match → `weak` with the leading candidate plus alternates. -- No usable hit → `miss`. +- No wrapper runs what the need names → `miss`. Validate the recommendation with `validate-galaxy-tool-discovery` before returning it. Do not rely on prose-only shape checks; downstream phases branch on this contract. diff --git a/plugins/foundry-skills/skills/discover-shed-tool/_provenance.json b/plugins/foundry-skills/skills/discover-shed-tool/_provenance.json index cf395c2..f26cb85 100644 --- a/plugins/foundry-skills/skills/discover-shed-tool/_provenance.json +++ b/plugins/foundry-skills/skills/discover-shed-tool/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "discover-shed-tool", "path": "content/molds/discover-shed-tool/index.md", - "revision": 5, - "content_hash": "05c99cbac2ec6a4d130b06204c15d0e76cf4fd0e00cdc8164dd606dac72809e2", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 6, + "content_hash": "f861bb1cb096f6e293ed3aaa25fe2f5aed11b5a22020434579513ccf7ed63869", + "commit": "db8801f2f0a31746e35cf399dd96cdbb8aa36dcf" }, - "cast_at": "2026-09-13T20:58:27.321Z", + "cast_at": "2026-11-10T10:22:51.549Z", "cast_date": "2026-05-07", "cast_revision": 5, "cast_history": [ @@ -110,8 +110,8 @@ "evidence": "cast-validated", "purpose": "Validate the hit, weak, or miss recommendation emitted by Tool Shed discovery.", "verification": "Run discover-shed-tool against known FastQC, ambiguous BWA-style, and no-hit queries and validate each emitted recommendation.", - "src_hash": "c0940fc0265e8fd25955f4380f03e088b62d25155fe98e7f051a5a3957d2840c", - "dst_hash": "c0940fc0265e8fd25955f4380f03e088b62d25155fe98e7f051a5a3957d2840c", + "src_hash": "0933a77437e523405e72d2b7ceacede6abee6cc51cc4c6da376d939a1439a2ee", + "dst_hash": "0933a77437e523405e72d2b7ceacede6abee6cc51cc4c6da376d939a1439a2ee", "source": "deterministic" } ], diff --git a/plugins/foundry-skills/skills/discover-shed-tool/_required_tools.json b/plugins/foundry-skills/skills/discover-shed-tool/_required_tools.json index 7b5e5c0..504a01c 100644 --- a/plugins/foundry-skills/skills/discover-shed-tool/_required_tools.json +++ b/plugins/foundry-skills/skills/discover-shed-tool/_required_tools.json @@ -3,9 +3,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", diff --git a/plugins/foundry-skills/skills/discover-shed-tool/_verify.json b/plugins/foundry-skills/skills/discover-shed-tool/_verify.json index 6d85130..fa8013e 100644 --- a/plugins/foundry-skills/skills/discover-shed-tool/_verify.json +++ b/plugins/foundry-skills/skills/discover-shed-tool/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-tool-discovery", + "--schema-sha256", + "0933a77437e523405e72d2b7ceacede6abee6cc51cc4c6da376d939a1439a2ee", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/discover-shed-tool/references/schemas/galaxy-tool-discovery.schema.json b/plugins/foundry-skills/skills/discover-shed-tool/references/schemas/galaxy-tool-discovery.schema.json index 2158d15..6f10409 100644 --- a/plugins/foundry-skills/skills/discover-shed-tool/references/schemas/galaxy-tool-discovery.schema.json +++ b/plugins/foundry-skills/skills/discover-shed-tool/references/schemas/galaxy-tool-discovery.schema.json @@ -26,7 +26,7 @@ "weak", "miss" ], - "description": "Recommendation class. hit means proceed with the selected Tool Shed pin; weak means candidate exists but needs confirmation or may fall through; miss means no usable wrapper was found." + "description": "Recommendation class. hit means proceed with the selected Tool Shed pin; weak means a candidate exists and needs confirmation before it is pinned; miss means no wrapper runs the executable or operation the need names." }, "candidate": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/find-test-data/SKILL.md b/plugins/foundry-skills/skills/find-test-data/SKILL.md index b39993f..88cd870 100644 --- a/plugins/foundry-skills/skills/find-test-data/SKILL.md +++ b/plugins/foundry-skills/skills/find-test-data/SKILL.md @@ -13,16 +13,19 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs -- Read artifact `freeform-summary`. Produced by `interview-to-freeform-summary`, `summarize-paper`. Source summary from summarize-paper / interview-to-freeform-summary; mine its sample-data, public-data-candidate, accession, and data-sizing guidance — this is where the source's dataset evidence lives (the design briefs strip it). -- Read artifact `summary-nextflow`. Schema: summary-nextflow. Produced by `summarize-nextflow`. Source summary from summarize-nextflow; mine its test candidate set, explicit selection, and sample-data evidence when running the NEXTFLOW → GALAXY pipeline. -- Read artifact `summary-cwl`. Schema: summary-cwl. Produced by `summarize-cwl`. Source summary from summarize-cwl; mine its test-data / sample-data evidence when running the CWL → GALAXY pipeline. -- Read artifact `freeform-galaxy-interface`. Produced by `freeform-summary-to-galaxy-interface`. Galaxy interface brief from freeform-summary-to-galaxy-interface pinning input labels, collection shapes, and datatypes for the PAPER / INTERVIEW → GALAXY pipelines. +- Read artifact `workflow-brief`. Optional; absence is allowed and must be reported honestly. Schema: workflow-brief-schema. Produced by `freeform-summary-to-workflow-brief`, `nextflow-summary-to-workflow-brief`. Reviewed Workflow Brief when running the brief-driven Galaxy path; mine only its selected input and fixture requirements. +- Read artifact `freeform-summary`. Optional; absence is allowed and must be reported honestly. Produced by `interview-to-freeform-summary`, `summarize-paper`. Source summary from summarize-paper / interview-to-freeform-summary; mine its sample-data, public-data-candidate, accession, and data-sizing guidance — this is where the source's dataset evidence lives (the design briefs strip it). +- Read artifact `summary-nextflow`. Optional; absence is allowed and must be reported honestly. Schema: summary-nextflow. Produced by `summarize-nextflow`. Source summary from summarize-nextflow; mine its test candidate set, explicit selection, and sample-data evidence when running the NEXTFLOW → GALAXY pipeline. +- Read artifact `summary-cwl`. Optional; absence is allowed and must be reported honestly. Schema: summary-cwl. Produced by `summarize-cwl`. Source summary from summarize-cwl; mine its test-data / sample-data evidence when running the CWL → GALAXY pipeline. +- Read artifact `freeform-galaxy-interface`. Produced by `freeform-summary-to-galaxy-interface`. Galaxy interface brief from freeform-summary-to-galaxy-interface pinning input labels, collection shapes, and datatypes for the direct PAPER / INTERVIEW → GALAXY pipelines. +- Read artifact `workflow-brief-galaxy-interface`. Produced by `workflow-brief-to-galaxy-interface`. Galaxy interface brief from workflow-brief-to-galaxy-interface pinning input labels, collection shapes, and datatypes for a Workflow-Brief-driven Galaxy path. - Read artifact `nextflow-galaxy-interface`. Produced by `nextflow-summary-to-galaxy-interface`. Galaxy interface brief from nextflow-summary-to-galaxy-interface pinning input labels, collection shapes, and datatypes for the NEXTFLOW → GALAXY pipeline. - Read artifact `cwl-galaxy-interface`. Produced by `cwl-summary-to-galaxy-interface`. Galaxy interface brief from cwl-summary-to-galaxy-interface pinning input labels, collection shapes, and datatypes for the CWL → GALAXY pipeline. +- Read artifact `test-data-refs`. Optional; absence is allowed and must be reported honestly. Schema: test-data-refs. Produced by `cwl-to-test-data`, `find-test-data`, `nextflow-to-test-data`, `paper-to-test-data`. Refs from the chain's source-specific leg (nextflow-to-test-data, cwl-to-test-data, paper-to-test-data) when the pipeline has one. Keep its resolved entries unchanged and search only for its unresolved inputs. Absent when this Mold is the chain's first leg. ## Outputs -- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Test data matched from IWC fixtures or public sources, expressed as URLs/paths plus expected shapes for downstream test authoring. +- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Schema: test-data-refs. Test data matched from IWC fixtures or public sources, expressed as URLs/paths plus expected shapes for downstream test authoring. Carries forward, unchanged, every entry an earlier leg resolved. ## Required Tools @@ -30,7 +33,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load Upfront -- None declared. +- `references/schemas/test-data-refs.schema.json`: Schema file copied verbatim into the bundle. Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning. ## Load On Demand @@ -39,22 +42,23 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- None declared. +- Before starting, check that the installed validator carries the test-data-refs schema this bundle was cast with: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `test-data-refs.json` before returning it: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json`. This checks artifact `test-data-refs` against the test-data-refs schema. ## Procedure -Resolve concrete test data for the workflow's inputs. Read the interface brief for each input's Galaxy shape and datatype, **and the source summary for the data the source itself names** — then search IWC fixtures and public sources for data that matches. Emit `test-data-refs.json`: one entry per input, each carrying a URL or path plus the expected shape, ready for implement-galaxy-workflow-test to stage. +Resolve concrete test data for the workflow's inputs. Read the interface brief for each input's Galaxy shape and datatype, the reviewed Workflow Brief when supplied, and any retained source summary for data the source itself names — then search IWC fixtures and public sources for data that matches. Emit `test-data-refs.json`: one entry per input, each carrying a URL or path plus the expected shape, ready for implement-galaxy-workflow-test to stage. -This skill is the first leg of the harness's `test-data-resolution` branch. It resolves what it can and reports gaps; the harness routes any unresolved input to the `user-supplied` fallthrough. Deciding to ask the user is a harness concern, not this skill's — its job is an honest, source-backed match. +This skill is the search leg of the harness's `test-data-resolution` branch: it runs after the source-specific leg when the pipeline has one, and first otherwise. It resolves what it can and reports gaps; the harness routes any unresolved input to the `user-supplied` fallthrough. Deciding to ask the user is a harness concern, not this skill's — its job is an honest, source-backed match. ### Sequence -1. **Enumerate inputs and their required shape.** From the interface brief, list each workflow input: label, Galaxy collection shape (File / list / paired / list:paired / record), and datatype. This is the *target shape* every match must satisfy. -2. **Mine the source summary for named data.** The interface and data-flow briefs are design artifacts — they deliberately drop dataset provenance. The source summary (`freeform-summary` / `summary-nextflow` / `summary-cwl`) is where the source names its data: sample-data locations, accessions, public-data candidates, fallback bundles, and sizing guidance ("one chromosome", "precomputed count matrix", "small subset"). Pull every candidate dataset and every data-sizing instruction the source gives. +1. **Enumerate inputs and their required shape.** From the interface brief, list each workflow input: label, Galaxy collection shape (File / list / paired / list:paired / record), and datatype. This is the *target shape* every match must satisfy. If an earlier leg's `test-data-refs` is present, keep each `resolved: true` entry exactly as written, `resolved_by` included, and work only the unresolved labels through steps 2–5. +2. **Mine the contract and available evidence for named data.** The interface and data-flow briefs are design artifacts — they deliberately drop dataset provenance. Read fixture requirements from `workflow-brief` when present, then consult any retained source summary (`freeform-summary` / `summary-nextflow` / `summary-cwl`) for sample-data locations, accessions, public-data candidates, fallback bundles, and sizing guidance. A hand-authored brief may have no summary; search from its selected input requirements rather than inventing provenance. 3. **Match each named candidate against the required shape — and don't stop at a shape mismatch.** Check each candidate against step 1's target shape and datatype. A candidate that is the wrong *shape* (e.g. raw signal / reads named when the input is a count matrix) is **not** a resolution — but it is also **not** the end of the search. When the source's named candidates don't fit, follow the source's own guidance to the right-shape public artifact: if the source says the input is a precomputed count matrix, find the canonical public count matrix for that study/domain (GEO/ENCODE/ArrayExpress series, a published supplementary table) rather than reporting "no data." "Named candidates are the wrong shape" ≠ "no data exists." 4. **Search IWC fixtures and public sources.** Prefer existing IWC test data for the same domain — it already conforms to iwc-test-data-conventions (remote URL, recorded hash, known collection layout); a near-neighbour IWC `-tests.yml` is the strongest source. Otherwise resolve the right-shape public dataset found in step 3, sized for a fast test run. 5. **"Small" is a documented subset of a real source, not a fabricated stand-in.** When the source asks for a small fixture (one chromosome, selected loci, a few samples) and only a full real dataset exists, that input is **resolved**: record the real source URL plus the data-import-boundary prep needed to reach the small shape (row-subset by key, column/sample split into the collection's element identifiers). The prep is a note on the ref, not an analysis step and not an excuse to mark the input unresolved. Resolve the *data*; leave analysis parameters (factors, thresholds, reference levels, top-N) to the design skills — they are not this skill's to decide. -6. **Emit refs.** Write one `test-data-refs.json` entry per input: the URL/path, the expected Galaxy shape, datatype, element identifiers when it is a collection, integrity hash when known, and any subset/split prep. Per galaxy-workflow-testability-design, make sure each entry maps to an addressable input label. +6. **Emit refs.** Write one `test-data-refs.json` entry per input: the URL/path, the expected Galaxy shape, datatype, element identifiers when it is a collection, integrity hash when known, and any subset/split prep. Per galaxy-workflow-testability-design, make sure each entry maps to an addressable input label. Shape each entry per test-data-refs, set `resolved_by: find-test-data` on the entries you resolve, and validate the file with `foundry validate-test-data-refs` before returning. 7. **Report genuine gaps only.** Mark `resolved: false` with a reason **only** when steps 2–5 turn up no real source of the right shape — not merely because the source's first-named candidate was the wrong shape or because a real source needs a documented subset. These honest gaps are what the harness hands to `user-supplied`. ### No fabrication diff --git a/plugins/foundry-skills/skills/find-test-data/_provenance.json b/plugins/foundry-skills/skills/find-test-data/_provenance.json index bf3b584..cde812e 100644 --- a/plugins/foundry-skills/skills/find-test-data/_provenance.json +++ b/plugins/foundry-skills/skills/find-test-data/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "find-test-data", "path": "content/molds/find-test-data/index.md", - "revision": 5, - "content_hash": "dab7a03bc9d3382c07b335054c5e1d4b63ddc9ba50a8a703a5895573d44318be", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "revision": 8, + "content_hash": "854238aee01711044de686f74e789e843f19613e6740722b7d95e68d7ad89e03", + "commit": "2f8792a62ca53f074605ddbcfb25e75871f52b24" }, - "cast_at": "2026-09-15T23:16:21.017Z", + "cast_at": "2026-10-22T03:17:18.861Z", "refs": [ { "kind": "research", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Confirm the collection shapes and labels the resolved data must populate so downstream tests can address them.", "trigger": "When mapping an input label in the data-flow brief to a concrete file or collection of files.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -36,8 +36,24 @@ "evidence": "corpus-observed", "purpose": "Match IWC test-data conventions — Zenodo/remote-URL first, SHA-1 integrity, per-input collection layout — when selecting or describing candidate fixtures.", "trigger": "When choosing where test data should come from or describing the shape a candidate file must have.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[test-data-refs]]", + "src": "package://@galaxy-foundry/gxwf-foundry#testDataRefsSchema", + "dst": "references/schemas/test-data-refs.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning.", + "verification": "Run the test-data-resolution chain on a pinned scenario and confirm each leg's test-data-refs.json validates and implement-galaxy-workflow-test stages from it.", + "license": "MIT", + "src_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "dst_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", "source": "deterministic" } ], @@ -47,13 +63,25 @@ "id": "test-data-refs", "kind": "json", "default_filename": "test-data-refs.json", - "description": "Test data matched from IWC fixtures or public sources, expressed as URLs/paths plus expected shapes for downstream test authoring." + "schema": "[[test-data-refs]]", + "description": "Test data matched from IWC fixtures or public sources, expressed as URLs/paths plus expected shapes for downstream test authoring. Carries forward, unchanged, every entry an earlier leg resolved." } ], "consumes": [ + { + "id": "workflow-brief", + "description": "Reviewed Workflow Brief when running the brief-driven Galaxy path; mine only its selected input and fixture requirements.", + "optional": true, + "inherited_schema": "[[workflow-brief-schema]]", + "producers": [ + "freeform-summary-to-workflow-brief", + "nextflow-summary-to-workflow-brief" + ] + }, { "id": "freeform-summary", "description": "Source summary from [[summarize-paper]] / [[interview-to-freeform-summary]]; mine its sample-data, public-data-candidate, accession, and data-sizing guidance — this is where the source's dataset evidence lives (the design briefs strip it).", + "optional": true, "producers": [ "interview-to-freeform-summary", "summarize-paper" @@ -62,6 +90,7 @@ { "id": "summary-nextflow", "description": "Source summary from [[summarize-nextflow]]; mine its test candidate set, explicit selection, and sample-data evidence when running the NEXTFLOW → GALAXY pipeline.", + "optional": true, "inherited_schema": "[[summary-nextflow]]", "producers": [ "summarize-nextflow" @@ -70,6 +99,7 @@ { "id": "summary-cwl", "description": "Source summary from [[summarize-cwl]]; mine its test-data / sample-data evidence when running the CWL → GALAXY pipeline.", + "optional": true, "inherited_schema": "[[summary-cwl]]", "producers": [ "summarize-cwl" @@ -77,11 +107,18 @@ }, { "id": "freeform-galaxy-interface", - "description": "Galaxy interface brief from [[freeform-summary-to-galaxy-interface]] pinning input labels, collection shapes, and datatypes for the PAPER / INTERVIEW → GALAXY pipelines.", + "description": "Galaxy interface brief from [[freeform-summary-to-galaxy-interface]] pinning input labels, collection shapes, and datatypes for the direct PAPER / INTERVIEW → GALAXY pipelines.", "producers": [ "freeform-summary-to-galaxy-interface" ] }, + { + "id": "workflow-brief-galaxy-interface", + "description": "Galaxy interface brief from [[workflow-brief-to-galaxy-interface]] pinning input labels, collection shapes, and datatypes for a Workflow-Brief-driven Galaxy path.", + "producers": [ + "workflow-brief-to-galaxy-interface" + ] + }, { "id": "nextflow-galaxy-interface", "description": "Galaxy interface brief from [[nextflow-summary-to-galaxy-interface]] pinning input labels, collection shapes, and datatypes for the NEXTFLOW → GALAXY pipeline.", @@ -95,6 +132,18 @@ "producers": [ "cwl-summary-to-galaxy-interface" ] + }, + { + "id": "test-data-refs", + "description": "Refs from the chain's source-specific leg ([[nextflow-to-test-data]], [[cwl-to-test-data]], [[paper-to-test-data]]) when the pipeline has one. Keep its resolved entries unchanged and search only for its unresolved inputs. Absent when this Mold is the chain's first leg.", + "optional": true, + "inherited_schema": "[[test-data-refs]]", + "producers": [ + "cwl-to-test-data", + "find-test-data", + "nextflow-to-test-data", + "paper-to-test-data" + ] } ] } diff --git a/plugins/foundry-skills/skills/find-test-data/_verify.json b/plugins/foundry-skills/skills/find-test-data/_verify.json index 869d020..ef641aa 100644 --- a/plugins/foundry-skills/skills/find-test-data/_verify.json +++ b/plugins/foundry-skills/skills/find-test-data/_verify.json @@ -24,6 +24,46 @@ "validate-summary-nextflow", "{artifact_path}" ] + }, + { + "artifact_id": "test-data-refs", + "direction": "input", + "kind": "json", + "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", + "validator_bin": "foundry", + "args": [ + "validate-test-data-refs", + "--schema-sha256", + "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "{artifact_path}" + ] + }, + { + "artifact_id": "workflow-brief", + "direction": "input", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "validator_bin": "foundry", + "args": [ + "validate-workflow-brief", + "{artifact_path}" + ] + }, + { + "artifact_id": "test-data-refs", + "direction": "output", + "kind": "json", + "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", + "validator_bin": "foundry", + "args": [ + "validate-test-data-refs", + "--schema-sha256", + "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "{artifact_path}" + ] } ] } diff --git a/plugins/foundry-skills/skills/find-test-data/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/find-test-data/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/find-test-data/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/find-test-data/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/find-test-data/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/find-test-data/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/find-test-data/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/find-test-data/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/find-test-data/references/schemas/test-data-refs.schema.json b/plugins/foundry-skills/skills/find-test-data/references/schemas/test-data-refs.schema.json new file mode 100644 index 0000000..45df249 --- /dev/null +++ b/plugins/foundry-skills/skills/find-test-data/references/schemas/test-data-refs.schema.json @@ -0,0 +1,291 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://galaxyproject.org/foundry/schemas/test-data-refs.schema.json", + "$comment": "Canonical source: packages/gxwf-foundry/src/schemas/test-data-refs/test-data-refs.schema.json in galaxyproject/foundry. Mold frontmatter cites this schema via [[test-data-refs]] wiki-links; the cast pipeline imports the `testDataRefsSchema` runtime export and serializes it into cast bundles. $defs.Fixture is a verbatim copy of the galaxy-workflow-test-plan Fixture; a test keeps them equal.", + "title": "Galaxy workflow test-data refs", + "description": "Test data for each Galaxy workflow input, written by the test-data-resolution chain (a source-specific Mold, then find-test-data) and read by implement-galaxy-workflow-test. One ref per workflow input; an input no leg could resolve stays in the list as an honest gap.", + "$ref": "#/$defs/TestDataRefs", + "$defs": { + "TestDataRefs": { + "type": "object", + "additionalProperties": false, + "required": [ + "refs" + ], + "properties": { + "refs": { + "type": "array", + "items": { + "$ref": "#/$defs/Ref" + }, + "description": "One entry per workflow input label. A later leg of the chain keeps every entry an earlier leg resolved, unchanged, and replaces only unresolved ones." + } + } + }, + "Ref": { + "type": "object", + "additionalProperties": false, + "required": [ + "workflow_label", + "collection_shape", + "datatype", + "resolved", + "resolved_by", + "reason", + "fixture" + ], + "properties": { + "workflow_label": { + "type": "string", + "minLength": 1, + "description": "Workflow input this ref feeds: its label in the Galaxy interface brief, or its input port in the CWL design." + }, + "collection_shape": { + "type": [ + "string", + "null" + ], + "description": "Galaxy collection shape when the input is a collection, e.g. list, paired, list:paired, record; null for a single dataset." + }, + "datatype": { + "type": [ + "string", + "null" + ], + "description": "Galaxy datatype/extension, e.g. fastqsanger.gz, fasta, tabular; null when undecided." + }, + "resolved": { + "type": "boolean", + "description": "true when the ref points at real data of the required shape; false for an honest gap." + }, + "resolved_by": { + "type": [ + "string", + "null" + ], + "description": "Name of the Mold that resolved this input, e.g. nextflow-to-test-data or find-test-data; null while unresolved." + }, + "reason": { + "type": [ + "string", + "null" + ], + "description": "Why the input is unresolved; null when resolved." + }, + "fixture": { + "$ref": "#/$defs/Fixture", + "description": "Where the data lives. For a collection whose elements carry their own fixtures, storage and location may be null." + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + }, + "description": "Collection elements, nested for nested shapes (list:paired → sample → forward/reverse). Identifiers must match the staged test file." + }, + "prep": { + "type": [ + "string", + "null" + ], + "description": "Data-import-boundary prep needed to reach the Galaxy shape from the source data: a subset, split, or concatenation. A note on the ref, not an analysis step." + } + }, + "allOf": [ + { + "if": { + "properties": { + "resolved": { + "const": false + } + } + }, + "then": { + "properties": { + "resolved_by": { + "type": "null" + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "properties": { + "storage": { + "const": "unresolved" + }, + "location": { + "type": "null" + } + } + } + } + }, + "else": { + "properties": { + "resolved_by": { + "type": "string", + "minLength": 1 + }, + "reason": { + "type": "null" + } + }, + "anyOf": [ + { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + } + }, + { + "if": { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + "then": { + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + } + } + } + } + } + } + ] + }, + "Element": { + "type": "object", + "additionalProperties": false, + "required": [ + "identifier" + ], + "properties": { + "identifier": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "$ref": "#/$defs/Fixture" + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + } + } + } + }, + "DataElement": { + "description": "An element of a resolved collection: it carries a located fixture, or nested elements that each do.", + "anyOf": [ + { + "required": [ + "fixture" + ], + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + }, + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + }, + "Fixture": { + "type": "object", + "additionalProperties": false, + "required": [ + "storage", + "location", + "checksum", + "provenance" + ], + "properties": { + "storage": { + "type": [ + "string", + "null" + ], + "enum": [ + "remote-url", + "in-repo", + "cvmfs-string", + "generated-toy", + "unresolved", + null + ], + "description": "Where the fixture data lives or should live: remote-url (Zenodo/EBI/SRA location), in-repo (committed test-data path), cvmfs-string (data-table/built-in-index bare string), generated-toy (to be synthesized), or unresolved (not yet sourced)." + }, + "location": { + "type": [ + "string", + "null" + ], + "description": "Concrete URL, repo-relative path, or data-table string when known; null when the fixture still needs resolution." + }, + "checksum": { + "type": [ + "string", + "null" + ], + "description": "Integrity hash when available, e.g. SHA-1:; null when unknown." + }, + "provenance": { + "type": "string", + "description": "Where this fixture comes from or how it was chosen, e.g. a translated source test-data reference, a Zenodo record, or a note that test-data-resolution must supply it." + } + } + } + } +} diff --git a/plugins/foundry-skills/skills/freeform-summary-to-cwl-design/_provenance.json b/plugins/foundry-skills/skills/freeform-summary-to-cwl-design/_provenance.json index e88006d..e39a440 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-cwl-design/_provenance.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-cwl-design/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/freeform-summary-to-cwl-design/index.md", "revision": 1, "content_hash": "bdc96941ec75c70907bac77ab58b94da70923cb81ab28da3badd3410ce38a678", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "21765674b50fa1552557043bbd4709ae15ab4c3f" }, - "cast_at": "2026-09-13T20:58:27.416Z", + "cast_at": "2026-09-20T17:29:30.447Z", "refs": [], "artifacts": { "produces": [ diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/SKILL.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/SKILL.md index 87352f0..59a72de 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/SKILL.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/SKILL.md @@ -15,7 +15,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `freeform-summary`. Produced by `interview-to-freeform-summary`, `summarize-paper`. Free-form source summary emitted by summarize-paper or interview-to-freeform-summary; consumed alongside the Galaxy interface brief. - Read artifact `freeform-galaxy-interface`. Produced by `freeform-summary-to-galaxy-interface`. Preceding Galaxy interface brief from freeform-summary-to-galaxy-interface that pins inputs, outputs, and labels. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/_provenance.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/_provenance.json index bc2fb70..de07d1d 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/_provenance.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/freeform-summary-to-galaxy-data-flow/index.md", "revision": 3, "content_hash": "22613db4e710f58b29bfd2d6fe1d49f167afe451e09afefb2efd755b4a6e4da1", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "94ce4ea20107680b467d03e8889b63689774e325" }, - "cast_at": "2026-09-13T20:58:27.463Z", + "cast_at": "2026-11-10T10:05:12.482Z", "refs": [ { "kind": "pattern", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Ground collection-shape choices in curated, corpus-observed operation and recipe patterns.", "trigger": "When selecting collection cleanup, reshape, identifier, or collection-tabular bridge patterns.", - "src_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", - "dst_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", "source": "deterministic" }, { @@ -36,8 +36,8 @@ "evidence": "corpus-observed", "purpose": "Ground conditional-branch and optional-step choices in curated, corpus-observed Galaxy when/pick_value patterns.", "trigger": "When data-flow translation needs optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", - "src_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", - "dst_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", "source": "deterministic" }, { @@ -51,8 +51,8 @@ "evidence": "corpus-observed", "purpose": "Ground genomic-interval operation choices in curated, corpus-observed Galaxy interval recipes.", "trigger": "When the workflow operates on genomic intervals (BED/GFF/VCF coordinate features) and data-flow translation needs overlap, merge, coverage, windowing, masking, or set-algebra steps.", - "src_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", - "dst_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", "source": "deterministic" }, { @@ -66,8 +66,8 @@ "evidence": "corpus-observed", "purpose": "Ground tabular bridge and table-operation choices in curated, corpus-observed operation patterns.", "trigger": "When data-flow translation needs filtering, joining, aggregation, pivoting, or tabular-collection bridges.", - "src_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", - "dst_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", "source": "deterministic" }, { @@ -81,8 +81,8 @@ "evidence": "hypothesis", "purpose": "Keep the data-flow brief separate from gxformat2 templating and concrete step implementation.", "verification": "Promote after two worked freeform-to-Galaxy translations preserve this Mold boundary without moving fields.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -111,8 +111,8 @@ "evidence": "hypothesis", "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's wiring and collection decisions settle, and append data-flow obligations the narrative leaves open — a described transformation with no named tool, an implied intermediate the source never states, an ordering the prose doesn't fix.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" } ], @@ -167,7 +167,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md index bd4ddc7..b00b1d5 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md @@ -13,9 +13,9 @@ tags: - topic/collection-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy collection transformation patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." related_notes: - "[[iwc-transformations-survey]]" - "[[iwc-conditionals-survey]]" @@ -25,10 +25,13 @@ related_patterns: - "[[reshape-relabel-remap-by-collection-axis]]" - "[[fan-in-bundle-consume-and-flatten]]" - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" - "[[sync-collections-by-identifier]]" - "[[harmonize-by-sortlist-from-identifiers]]" - "[[regex-relabel-via-tabular]]" - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" - "[[collection-swap-nesting-with-apply-rules]]" - "[[collection-split-identifier-via-rules]]" - "[[collection-build-list-paired-with-apply-rules]]" @@ -42,45 +45,46 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: collection patterns -This is the runtime-facing map for Galaxy collection transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. -## Cleanup +## Enter or leave a collection -- [[collection-cleanup-after-mapover-failure]] — use `__FILTER_EMPTY_DATASETS__` or `__FILTER_FAILED_DATASETS__` after map-over when empty or errored elements would break downstream steps. -- [[collection-unbox-singleton]] — use `__EXTRACT_DATASET__` with `which: first` when a known one-element collection must become a dataset. +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. -## Map-over Lifecycle Recipes +## Keep mapped siblings aligned -- [[manifest-to-mapped-collection-lifecycle]] — build collection elements from manifest/table rows, map a tool over them, then relabel or reshape outputs. -- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped outputs, sync siblings from surviving identifiers, then gate final reports. -- [[reshape-relabel-remap-by-collection-axis]] — correct the mapped axis after domain fan-out using Apply Rules and deterministic relabeling. -- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection consumer, then flatten or aggregate results. +First identify what changed. Membership, order, and labels are separate decisions. -## Identifiers +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. -- [[sync-collections-by-identifier]] — membership sync: extract identifiers from one collection and filter or relabel a sibling collection. -- [[harmonize-by-sortlist-from-identifiers]] — order sync: sort one sibling collection by another collection's identifier order. -- [[regex-relabel-via-tabular]] — label rewrite: derive new element identifiers in tabular form and apply them with `__RELABEL_FROM_FILE__`. -- [[relabel-via-rules-and-find-replace]] — relabel inside a structural reshape, currently grounded in the influenza fan-out pattern. +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. -## Structural Reshape +## Change the collection axes -- [[collection-flatten-after-fanout]] — collapse nested collection output to a flat list when the outer axis no longer matters. -- [[collection-build-named-bundle]] — assemble individual outputs into a named collection bundle for publishing or downstream fan-in. -- [[collection-swap-nesting-with-apply-rules]] — use Apply Rules to swap `list:list` axes. -- [[collection-split-identifier-via-rules]] — use Apply Rules regex columns to derive nested list identifiers from one identifier string. -- [[collection-build-list-paired-with-apply-rules]] — use Apply Rules to promote identifier columns into `list:paired` structure. +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. -## Bridges +## Follow a map-over lifecycle -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. ## See also diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md index 23fae29..2dfe890 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md @@ -12,9 +12,9 @@ tags: - topic/galaxy-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy when and pick_value conditional patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." related_notes: - "[[iwc-conditionals-survey]]" related_patterns: @@ -30,29 +30,33 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: conditionals patterns -This is the runtime-facing map for Galaxy conditional workflow choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. -## Direct Gates +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. -- [[conditional-run-optional-step]] — expose or derive a boolean, connect it as `inputs.when`, and use `when: $(inputs.when)` to skip optional steps. -- [[conditional-gate-on-nonempty-result]] — compute a boolean from empty/non-empty dataset or collection state before gating downstream reporting/export. The MGnify recipe is corpus-backed but clunky pending verified-pattern workflow work. +## Skip work without replacing its output -## Routes and Fallbacks +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. -- [[conditional-route-between-alternative-outputs]] — run one of several `when`-gated alternatives, then merge possible outputs with `pick_value`. -- [[conditional-transform-or-pass-through]] — optionally transform a value, then use `pick_value` to choose transformed output or original fallback. +## Continue with one value -## Not Conditionals +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. -- [[collection-cleanup-after-mapover-failure]] — use collection filters when empty or failed mapped elements should be dropped or replaced. This is collection-state cleanup, not a `when` gate. -- `__FILTER_NULL__` — catalog capability for null outputs after conditional steps, but no IWC-backed pattern candidate in the conditionals survey. Zero uptake is not itself an anti-pattern call. +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. ## See Also - [[iwc-conditionals-survey]] — conditionals survey and evidence trail. -- [[galaxy-collection-patterns]] — companion MOC for collection transforms and cleanup. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md index 8123493..de9ec07 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md @@ -13,8 +13,8 @@ tags: - topic/interval-transform status: draft created: 2026-06-10 -revised: 2026-08-03 -revision: 2 +revised: 2026-09-23 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." related_notes: - "[[iwc-interval-operations-survey]]" @@ -34,6 +34,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md index 445ed6c..591b3f5 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md @@ -13,8 +13,8 @@ tags: - topic/tabular-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-10-15 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." related_notes: - "[[iwc-tabular-operations-survey]]" @@ -40,38 +40,39 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: tabular patterns -This is the runtime-facing map for Galaxy tabular transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation pages are the actionable references. +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. -## Row And Column Operations +## Rows and columns in one table -- [[tabular-filter-by-column-value]] — keep/drop rows by string column value with `Filter1`. -- [[tabular-filter-by-regex]] — keep/drop rows by regex or pattern matching. -- [[tabular-cut-and-reorder-columns]] — project and reorder columns with `Cut1`. -- [[tabular-compute-new-column]] — compute row-wise values into a new column. +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. -## Joins And Aggregation +## Combine or summarize tables -- [[tabular-join-on-key]] — join two tabular datasets by key columns. -- [[tabular-group-and-aggregate-with-datamash]] — group and aggregate rows with `datamash_ops`. -- [[tabular-sql-query]] — use SQL when filtering, joining, and projection are clearer as one query. +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. -## Text-Processing Recipes +## Text-processing recipes -- [[tabular-prepend-header]] — add a header row with an awk/text-processing step. -- [[tabular-synthesize-bed-from-3col]] — build BED from three-column tabular inputs. -- [[tabular-split-taxonomy-string]] — split taxonomy-like strings into useful columns. -- [[tabular-relabel-by-row-counter]] — synthesize row labels from row order. +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. -## Bridges +## Between tables and collections -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. ## See also diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/SKILL.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/SKILL.md index cdf21d6..a0d1775 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/SKILL.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/SKILL.md @@ -14,7 +14,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs - Read artifact `freeform-summary`. Produced by `interview-to-freeform-summary`, `summarize-paper`. Free-form source summary emitted by summarize-paper or interview-to-freeform-summary; methods, tools, sample data, references, and workflow intent with explicit uncertainty. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/_provenance.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/_provenance.json index 7abed33..0a8dd30 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/_provenance.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/freeform-summary-to-galaxy-interface/index.md", "revision": 3, "content_hash": "de0cb02532205ad18f4aa1ab2881cac725a3191b448468a8a11d0663ff1ec97d", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "94ce4ea20107680b467d03e8889b63689774e325" }, - "cast_at": "2026-09-13T20:58:27.690Z", + "cast_at": "2026-11-10T10:05:12.495Z", "refs": [ { "kind": "research", @@ -73,8 +73,8 @@ "evidence": "corpus-observed", "purpose": "Choose stable workflow input/output labels and promoted checkpoint outputs that future tests can address.", "trigger": "When deciding labels, public outputs, checkpoint outputs, or fixture-compatible collection inputs.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -88,8 +88,8 @@ "evidence": "hypothesis", "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's input, output, and label decisions settle, and append interface obligations the narrative never settles — an output the source names but never specifies, a parameter stated only qualitatively, a collection shape the prose doesn't determine.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" } ], @@ -137,7 +137,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/SKILL.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/SKILL.md index ed2c0fd..fa9ed59 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/SKILL.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/SKILL.md @@ -18,7 +18,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `freeform-galaxy-data-flow`. Produced by `freeform-summary-to-galaxy-data-flow`. Galaxy data-flow brief from freeform-summary-to-galaxy-data-flow that pins abstract operations and collection choices. - Read artifact `iwc-comparison-notes`. Produced by `compare-against-iwc-exemplar`. Structural diff guidance from compare-against-iwc-exemplar (run on the design brief); steers the skeleton toward IWC-aligned structure before per-step authoring. Carries an inline gxformat2 excerpt of the nearest exemplar. - Read artifact `iwc-exemplar-gxformat2`. Produced by `compare-against-iwc-exemplar`. Cleaned gxformat2 view of the nearest IWC exemplar's relevant subgraph from compare-against-iwc-exemplar; pattern-match the draft's input/collection shapes, map-over wiring, output promotion, and post-job actions against this concrete idiom. Absent when no nearest exemplar was found. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -27,8 +27,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli @@ -41,6 +41,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load On Demand +- `references/cli/draft-next-step.json`: CLI command reference packaged as a sidecar. Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop. Use when: deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step. - `references/cli/draft-validate.json`: CLI command reference packaged as a sidecar. Validate the emitted draft against draft-contract rules (sentinel form, topology, _plan_* placement) before handing off. Use when: after writing or modifying the draft workflow file. - `references/patterns/galaxy-collection-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded collection pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges. - `references/patterns/galaxy-conditionals-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded conditional pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_provenance.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_provenance.json index 8e4b551..64bdfe5 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_provenance.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_provenance.json @@ -4,12 +4,28 @@ "mold": { "name": "freeform-summary-to-galaxy-template", "path": "content/molds/freeform-summary-to-galaxy-template/index.md", - "revision": 6, - "content_hash": "0a95f8c04465d48131ace523e0b27af53f1ade6e525a9173c5ff1e936ddc3366", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 7, + "content_hash": "a9a4393c49eec26be0c7c67925e04960a04ea07d5d7ee6ecd415c2491bcac467", + "commit": "db8801f2f0a31746e35cf399dd96cdbb8aa36dcf" }, - "cast_at": "2026-09-13T20:58:27.750Z", + "cast_at": "2026-11-10T10:22:51.627Z", "refs": [ + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[draft-next-step]]", + "src": "content/cli/gxwf/draft-next-step.md", + "dst": "references/cli/draft-next-step.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop.", + "trigger": "When deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step.", + "verification": "Cast the skill, emit a draft with a drafty step inside a subworkflow, confirm draft-next-step returns that step's full path.", + "src_hash": "d4fe7945945a57846fe8b168e7bacba04f739de1d86bc613ae6f0e5879ac2f55", + "dst_hash": "8638645f5974d8072dc3c63e522b7ab25d562f8d6217ea93b61f0a29f371d6f7", + "source": "deterministic" + }, { "kind": "cli-command", "mode": "sidecar", @@ -37,8 +53,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded collection pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges.", - "src_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", - "dst_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", "source": "deterministic" }, { @@ -52,8 +68,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded conditional pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", - "src_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", - "dst_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", "source": "deterministic" }, { @@ -67,8 +83,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded genomic-interval pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for interval overlap, merge, coverage, windowing, masking, or set-algebra on coordinate features.", - "src_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", - "dst_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", "source": "deterministic" }, { @@ -82,8 +98,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded tabular pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for tabular filtering, projection, joins, aggregation, text-processing recipes, or tabular-collection bridges.", - "src_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", - "dst_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", "source": "deterministic" }, { @@ -135,8 +151,8 @@ "purpose": "Respect the handoff from the freeform-to-Galaxy interface and data-flow briefs to the gxformat2 skeleton.", "trigger": "When translating abstract nodes, unresolved tool needs, and placeholder transformations into template TODOs.", "verification": "Promote after two worked freeform-to-Galaxy templates preserve the design-brief/template split without schema changes.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -150,8 +166,8 @@ "evidence": "corpus-observed", "purpose": "Group the settled step set into titled stage frames (the gxformat2 `comments:` array) so the skeleton carries the analysis-stage narrative IWC authors annotate by hand. Schema-legal and optional.", "trigger": "After topology is settled and the skeleton can be partitioned into named analysis stages (inputs, per-stage step clusters, parameter-derivation knots, visualization/outputs).", - "src_hash": "a206d5e08a8caaeaa42d9eadcd78eb0e2724c5756be681ca4b543cf0f7269176", - "dst_hash": "a206d5e08a8caaeaa42d9eadcd78eb0e2724c5756be681ca4b543cf0f7269176", + "src_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", + "dst_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", "source": "deterministic" }, { @@ -165,8 +181,8 @@ "evidence": "hypothesis", "purpose": "Emit the gxformat2 draft superset: TODO tool_id, optional state / tool_shed_repository, and per-step _plan_state / _plan_context planning fields.", "verification": "Promote after a downstream per-step implementation Mold consumes _plan_state and _plan_context without round-tripping back through the source summary.", - "src_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", - "dst_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", + "src_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", + "dst_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", "source": "deterministic" }, { @@ -180,8 +196,8 @@ "evidence": "corpus-observed", "purpose": "Choose stable workflow input/output labels, testable checkpoint outputs, and fixture-compatible workflow interfaces while drafting the skeleton.", "trigger": "When the template decides workflow inputs, workflow outputs, promoted checkpoints, or collection output identifiers that future tests will need to address.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -195,8 +211,8 @@ "evidence": "hypothesis", "purpose": "Close the open entries the settled topology discharges, and append a blocking entry for any settled step whose declared output no wired input can supply — the computability gap gxwf validation cannot see, raised here rather than left for the per-step loop to hit.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -211,8 +227,8 @@ "purpose": "Output contract: the emitted gxformat2 draft conforms to [[galaxy-workflow-draft]]. Cast bundles the JSON Schema so the skill carries its output shape alongside the [[draft-validate]] CLI checks.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", - "dst_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } @@ -290,7 +306,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_required_tools.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_required_tools.json index 54fd1c9..1a8a44d 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_required_tools.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/_required_tools.json @@ -3,13 +3,14 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": [ + "gxwf draft-next-step", "gxwf draft-validate" ] } diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/cli/draft-next-step.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/cli/draft-next-step.json new file mode 100644 index 0000000..296abe7 --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/cli/draft-next-step.json @@ -0,0 +1,42 @@ +{ + "type": "cli-command", + "tool": "gxwf", + "command": "draft-next-step", + "summary": "Pick the next drafty step a harness should work on, or report no remaining work; deterministic topological + alphabetical tiebreak.", + "source_path": "content/cli/gxwf/draft-next-step.md", + "source_revision": 1, + "package": "@galaxy-tool-util/cli", + "description": "Pick the next step a downstream agent should work on (or report no remaining work)", + "synopsis": "gxwf draft-next-step [options] ", + "args": [ + { + "raw": "file", + "name": "file", + "required": true, + "variadic": false, + "description": "Draft workflow file (.gxwf.yml)" + } + ], + "options": [ + { + "flags": "--format ", + "name": "format", + "description": "Force input format: format2 (default; native is rejected)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--output-format ", + "name": "outputFormat", + "description": "Output format: json (default) or markdown", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false, + "defaultValue": "json" + } + ], + "body": "# `gxwf draft-next-step`\n\nDeterministic next-step picker for the per-step authoring loop. Wraps `nextDraftStep` from `@galaxy-tool-util/schema`. Native (.ga) input is rejected — drafts are format2-only.\n\nThe harness loop reads as: `while (next = gxwf draft-next-step ).draft: invoke per-step skill on next.step`.\n\n## Output\n\nDefault output is pretty JSON — the agent-loop wire format. Same input → byte-identical output. `--output-format markdown` renders a human-glance checklist instead.\n\nJSON fields:\n\n- `draft` (bool) — `true` while a drafty step remains, `false` at the terminal cases below.\n- `step` (string array) — the picked step's **path**: a single id at top level (`[\"merge assembled transcripts\"]`), or the full path through containing subworkflow steps when the pick is nested. The leaf (last element) is the step to work on.\n- `work` (string array) — the chosen step's remaining `TODO[...]` sentinels followed by its `_plan_*` fields, in checklist order.\n\nAt the terminal cases the object is just `{ \"draft\": false }` — no `step` or `work` keys.\n\nTwo terminal cases both surface as `draft: false`:\n\n- The workflow's class is already `GalaxyWorkflow` (concretization complete, class promoted).\n- The class is still `GalaxyWorkflowDraft` but every step is fully resolved (no `TODO_*`, no `_plan_*`). The harness's next move is [[draft-extract]] (which calls `promoteFullyConcreteDrafts` to flip the class) followed by terminal [[validate]].\n\n## Examples\n\n```bash\ngxwf draft-next-step workflow.gxwf.yml\ngxwf draft-next-step workflow.gxwf.yml --output-format markdown\n```\n\nHarness loop sketch:\n\n```bash\nwhile [ \"$(gxwf draft-next-step workflow.gxwf.yml | jq -r .draft)\" = \"true\" ]; do\n step=$(gxwf draft-next-step workflow.gxwf.yml | jq -r '.step | last')\n # invoke per-step skill on step, which mutates workflow.gxwf.yml in place\ndone\n```\n\n## Gotchas\n\n- Pick order is **topological** (a step waits until its `in:` sources are concretized) with **alphabetical tiebreak** on step id at the same level. Same input across runs returns the same pick — safe to call repeatedly without state.\n- Descends into draft subworkflows; the returned `step` array is the full path through containing subworkflow steps (leaf last).\n- A `draft: false` result does **not** mean the workflow has passed [[validate]] yet. It means the per-step authoring loop is done; the harness still owns the promote + terminal-validate pass.\n- This command does not mutate the workflow file. It's a pure observer.\n- Insertion order of step dicts does not affect output — semantically equal workflows produce identical picks." +} diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md index efd7c87..9256dbf 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md @@ -4,12 +4,13 @@ tags: - target/galaxy status: draft created: 2026-06-12 -revised: 2026-06-12 -revision: 1 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-comments-survey]]" - "[[gxformat2-schema]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" summary: "How to annotate a gxformat2 workflow with editor comments: one titled frame per analysis stage, populate contains_steps, color decorative." diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md index 2a66f1a..9fc6e51 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-06 -revised: 2026-05-10 -revision: 2 +revised: 2026-11-10 +revision: 6 related_notes: - "[[gxformat2-schema]]" - "[[galaxy-data-flow-draft-contract]]" @@ -17,6 +17,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[implement-galaxy-tool-step]]" - "[[advance-galaxy-draft-step]]" @@ -117,14 +118,14 @@ steps: ## Topology repair escalation -The template Mold settles topology, but it can miss a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. +The template Mold settles topology, but a settled draft can still be unable to give a correct run. The common case is a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. When one surfaces, the per-step loop does not author topology to fix it. It **escalates**: -- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. -- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and re-wires the affected region: one producer step, a small sub-path, or honest narrowing of the output. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. +- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs, or that the chosen wrapper can't produce it. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. Earlier Molds and the loop raise blocking entries too. +- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and makes the smallest structural edits that close them. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. -This is **repair**, distinct from **settling**: it is narrow (one named region), reactive (only a detected computability gap triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each escalation must strictly reduce that count, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. +This is **repair**, distinct from **settling**: it is narrow (the entries bound it), reactive (an open blocking entry triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each repair must leave fewer than it found, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. ## Open work diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md index bd4ddc7..b00b1d5 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md @@ -13,9 +13,9 @@ tags: - topic/collection-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy collection transformation patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." related_notes: - "[[iwc-transformations-survey]]" - "[[iwc-conditionals-survey]]" @@ -25,10 +25,13 @@ related_patterns: - "[[reshape-relabel-remap-by-collection-axis]]" - "[[fan-in-bundle-consume-and-flatten]]" - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" - "[[sync-collections-by-identifier]]" - "[[harmonize-by-sortlist-from-identifiers]]" - "[[regex-relabel-via-tabular]]" - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" - "[[collection-swap-nesting-with-apply-rules]]" - "[[collection-split-identifier-via-rules]]" - "[[collection-build-list-paired-with-apply-rules]]" @@ -42,45 +45,46 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: collection patterns -This is the runtime-facing map for Galaxy collection transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. -## Cleanup +## Enter or leave a collection -- [[collection-cleanup-after-mapover-failure]] — use `__FILTER_EMPTY_DATASETS__` or `__FILTER_FAILED_DATASETS__` after map-over when empty or errored elements would break downstream steps. -- [[collection-unbox-singleton]] — use `__EXTRACT_DATASET__` with `which: first` when a known one-element collection must become a dataset. +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. -## Map-over Lifecycle Recipes +## Keep mapped siblings aligned -- [[manifest-to-mapped-collection-lifecycle]] — build collection elements from manifest/table rows, map a tool over them, then relabel or reshape outputs. -- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped outputs, sync siblings from surviving identifiers, then gate final reports. -- [[reshape-relabel-remap-by-collection-axis]] — correct the mapped axis after domain fan-out using Apply Rules and deterministic relabeling. -- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection consumer, then flatten or aggregate results. +First identify what changed. Membership, order, and labels are separate decisions. -## Identifiers +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. -- [[sync-collections-by-identifier]] — membership sync: extract identifiers from one collection and filter or relabel a sibling collection. -- [[harmonize-by-sortlist-from-identifiers]] — order sync: sort one sibling collection by another collection's identifier order. -- [[regex-relabel-via-tabular]] — label rewrite: derive new element identifiers in tabular form and apply them with `__RELABEL_FROM_FILE__`. -- [[relabel-via-rules-and-find-replace]] — relabel inside a structural reshape, currently grounded in the influenza fan-out pattern. +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. -## Structural Reshape +## Change the collection axes -- [[collection-flatten-after-fanout]] — collapse nested collection output to a flat list when the outer axis no longer matters. -- [[collection-build-named-bundle]] — assemble individual outputs into a named collection bundle for publishing or downstream fan-in. -- [[collection-swap-nesting-with-apply-rules]] — use Apply Rules to swap `list:list` axes. -- [[collection-split-identifier-via-rules]] — use Apply Rules regex columns to derive nested list identifiers from one identifier string. -- [[collection-build-list-paired-with-apply-rules]] — use Apply Rules to promote identifier columns into `list:paired` structure. +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. -## Bridges +## Follow a map-over lifecycle -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. ## See also diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md index 23fae29..2dfe890 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md @@ -12,9 +12,9 @@ tags: - topic/galaxy-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy when and pick_value conditional patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." related_notes: - "[[iwc-conditionals-survey]]" related_patterns: @@ -30,29 +30,33 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: conditionals patterns -This is the runtime-facing map for Galaxy conditional workflow choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. -## Direct Gates +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. -- [[conditional-run-optional-step]] — expose or derive a boolean, connect it as `inputs.when`, and use `when: $(inputs.when)` to skip optional steps. -- [[conditional-gate-on-nonempty-result]] — compute a boolean from empty/non-empty dataset or collection state before gating downstream reporting/export. The MGnify recipe is corpus-backed but clunky pending verified-pattern workflow work. +## Skip work without replacing its output -## Routes and Fallbacks +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. -- [[conditional-route-between-alternative-outputs]] — run one of several `when`-gated alternatives, then merge possible outputs with `pick_value`. -- [[conditional-transform-or-pass-through]] — optionally transform a value, then use `pick_value` to choose transformed output or original fallback. +## Continue with one value -## Not Conditionals +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. -- [[collection-cleanup-after-mapover-failure]] — use collection filters when empty or failed mapped elements should be dropped or replaced. This is collection-state cleanup, not a `when` gate. -- `__FILTER_NULL__` — catalog capability for null outputs after conditional steps, but no IWC-backed pattern candidate in the conditionals survey. Zero uptake is not itself an anti-pattern call. +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. ## See Also - [[iwc-conditionals-survey]] — conditionals survey and evidence trail. -- [[galaxy-collection-patterns]] — companion MOC for collection transforms and cleanup. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md index 8123493..de9ec07 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md @@ -13,8 +13,8 @@ tags: - topic/interval-transform status: draft created: 2026-06-10 -revised: 2026-08-03 -revision: 2 +revised: 2026-09-23 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." related_notes: - "[[iwc-interval-operations-survey]]" @@ -34,6 +34,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md index 445ed6c..591b3f5 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md @@ -13,8 +13,8 @@ tags: - topic/tabular-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-10-15 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." related_notes: - "[[iwc-tabular-operations-survey]]" @@ -40,38 +40,39 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: tabular patterns -This is the runtime-facing map for Galaxy tabular transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation pages are the actionable references. +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. -## Row And Column Operations +## Rows and columns in one table -- [[tabular-filter-by-column-value]] — keep/drop rows by string column value with `Filter1`. -- [[tabular-filter-by-regex]] — keep/drop rows by regex or pattern matching. -- [[tabular-cut-and-reorder-columns]] — project and reorder columns with `Cut1`. -- [[tabular-compute-new-column]] — compute row-wise values into a new column. +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. -## Joins And Aggregation +## Combine or summarize tables -- [[tabular-join-on-key]] — join two tabular datasets by key columns. -- [[tabular-group-and-aggregate-with-datamash]] — group and aggregate rows with `datamash_ops`. -- [[tabular-sql-query]] — use SQL when filtering, joining, and projection are clearer as one query. +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. -## Text-Processing Recipes +## Text-processing recipes -- [[tabular-prepend-header]] — add a header row with an awk/text-processing step. -- [[tabular-synthesize-bed-from-3col]] — build BED from three-column tabular inputs. -- [[tabular-split-taxonomy-string]] — split taxonomy-like strings into useful columns. -- [[tabular-relabel-by-row-counter]] — synthesize row labels from row order. +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. -## Bridges +## Between tables and collections -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. ## See also diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json index 3a39f43..dd3a94a 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json @@ -75,7 +75,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_version": { "anyOf": [ @@ -108,7 +108,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "out": { "anyOf": [ @@ -309,7 +309,7 @@ ] }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "release": { "anyOf": [ @@ -412,22 +412,28 @@ "items": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" } ] }, @@ -437,22 +443,28 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" }, { "type": "string" @@ -927,7 +939,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "label": { "anyOf": [ @@ -1040,7 +1052,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_state": { "anyOf": [ @@ -1856,7 +1868,9 @@ "text", "File", "data", - "collection" + "collection", + "color", + "directory_uri" ], "type": "string" } @@ -1952,10 +1966,10 @@ ] }, "restrictions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "suggestions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "type": { "enum": [ @@ -1963,6 +1977,9 @@ "string" ], "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" } }, "required": [ @@ -1971,6 +1988,96 @@ "type": "object" }, "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { "additionalProperties": false, "properties": { "default": { @@ -2016,7 +2123,7 @@ "required": [], "type": "object" }, - "Shared10": { + "Shared11": { "additionalProperties": false, "properties": { "default": { @@ -2124,7 +2231,7 @@ ], "type": "object" }, - "Shared11": { + "Shared12": { "additionalProperties": false, "properties": { "default": { @@ -2230,7 +2337,7 @@ "required": [], "type": "object" }, - "Shared12": { + "Shared13": { "additionalProperties": false, "properties": { "default": { @@ -2337,7 +2444,7 @@ ], "type": "object" }, - "Shared13": { + "Shared14": { "additionalProperties": false, "properties": { "default": { @@ -2424,7 +2531,143 @@ ], "type": "object" }, - "Shared14": { + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { "anyOf": [ { "items": { @@ -2457,7 +2700,7 @@ } ] }, - "Shared15": { + "Shared18": { "anyOf": [ { "type": "null" @@ -2488,11 +2731,11 @@ } ] }, - "Shared16": { + "Shared19": { "anyOf": [ { "items": { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, "type": "array" }, @@ -2500,7 +2743,7 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, { "type": "string" @@ -2522,7 +2765,7 @@ } ] }, - "Shared17": { + "Shared20": { "anyOf": [ { "type": "null" @@ -2610,7 +2853,7 @@ "$ref": "#/$defs/Shared1" }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "uuid": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/SKILL.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/SKILL.md index f20e2a1..aa994aa 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/SKILL.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/SKILL.md @@ -42,7 +42,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. +- Before starting, check that the installed validator carries the galaxy-workflow-test-plan schema this bundle was cast with: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. ## Procedure diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_provenance.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_provenance.json index 4e30ebf..a814662 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_provenance.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/freeform-summary-to-galaxy-test-plan/index.md", "revision": 2, "content_hash": "9c1d56e625a5dd26cb7a82082ac8f4eeb7db287a626bf00e73457c8dd491ad6d", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "a59d6025d612ce7013b09fa1ccb58f60507ccc6a" }, - "cast_at": "2026-09-13T20:58:27.957Z", + "cast_at": "2026-10-26T16:18:41.856Z", "refs": [ { "kind": "research", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Choose which workflow outputs and promoted checkpoints make meaningful assertions when no upstream test evidence exists.", "trigger": "When deciding which outputs to assert, which checkpoints to promote, and which labels the plan should assume.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -36,8 +36,8 @@ "evidence": "corpus-observed", "purpose": "Distinguish accepted IWC-style test shortcuts from assertion smells while synthesizing assertions from intent.", "trigger": "When considering existence-only, size-only, image-dimension, or tolerant output checks, or recording an omission.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -51,8 +51,8 @@ "evidence": "corpus-observed", "purpose": "Describe job-input fixtures, remote-URL-first locations, hashes, and collection input shapes the plan should record as fixture provenance.", "trigger": "When recording job inputs or fixture provenance, or deciding whether a fixture belongs in test-data, Zenodo, ENA/SRA, or CVMFS.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -66,8 +66,8 @@ "evidence": "corpus-observed", "purpose": "Pick the assertion family and tolerance that fit each synthesized expected output by output type.", "trigger": "When turning an expected output into assertion intent and a tolerance.", - "src_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", - "dst_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", "source": "deterministic" }, { @@ -81,8 +81,8 @@ "evidence": "corpus-observed", "purpose": "Keep the plan addressable by stable labels and artifacts Planemo can connect back to invocations, jobs, and outputs.", "trigger": "When recording the labels and checkpoints the downstream test must address.", - "src_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", - "dst_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", + "src_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "dst_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", "source": "deterministic" }, { @@ -97,8 +97,8 @@ "purpose": "Output contract: the emitted plan conforms to [[galaxy-workflow-test-plan]]. Cast bundles the JSON Schema so the skill carries its output shape; validate with `foundry validate-galaxy-workflow-test-plan`.", "verification": "Cast the skill, run on a representative paper/interview-derived summary, confirm the emitted YAML validates and downstream [[implement-galaxy-workflow-test]] consumes it without re-reading the source summary.", "license": "MIT", - "src_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", - "dst_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", + "src_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "dst_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "source": "deterministic" }, { @@ -114,8 +114,8 @@ "trigger": "When naming an assertion family or compare operator for a synthesized expected output.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", - "dst_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", + "src_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "dst_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_verify.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_verify.json index 5d06f4b..14be176 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_verify.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-workflow-test-plan", + "--schema-sha256", + "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md index 6d0fef7..bd1160e 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-11 -revision: 6 +revised: 2026-10-21 +revision: 10 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,231 +16,109 @@ related_notes: - "[[validate-tests]]" - "[[iwc-tabular-operations-survey]]" - "[[galaxy-discover-datasets]]" -summary: "Decision and idiom guide for picking planemo workflow-test assertions: which family per output type, how to size tolerances, when to validate." +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." --- -# Planemo asserts: idiom and decision guide +# Choosing Planemo workflow-test assertions -Companion to [[iwc-test-data-conventions]] (input shapes), [[galaxy-workflow-testability-design]] (workflow structure before test YAML exists), and [[iwc-shortcuts-anti-patterns]] (what's accepted vs smell). This note is forward-looking: when authoring a new `-tests.yml`, which assertion family fits which output, and what the recommended tolerances and operators are. +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. -The **vocabulary itself is not restated here** — every assertion's parameter list, types, defaults, required fields, and Python docstring is rendered from the test-format JSON Schema at [[tests-format]]. Assertion names below deep-link into that page (e.g. [[tests-format#has_text_model|has_text]] jumps straight to that `$def`). +## 1. Choose by the property, then the output type -## 1. Choose by output type +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | -The single most useful decision table. Pick the row that matches the file format the workflow emits; default to the recommended assertion family. +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. -| Output type | Default assertion family | Why | Fallback | -|---|---|---|---| -| **Plain text reports / logs** (FastQC summary, MultiQC text section) | [[tests-format#has_text_model|has_text]] (substring on a known stable token) + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Substring is robust across versions; line count catches truncation | [[tests-format#has_text_matching_model|has_text_matching]] (regex) when token text changes per run | -| **HTML reports** (MultiQC HTML, custom dashboards) | [[tests-format#has_text_model|has_text]] against stable section names | HTML embeds timestamps and asset hashes; byte-diff is hopeless | [[tests-format#has_size_model|has_size]] + large `delta` as last resort | -| **Tabular** (TSV, CSV, BED-like) | [[tests-format#has_n_columns_model|has_n_columns]] + [[tests-format#has_text_model|has_text]] for headers + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Schema-shape + content probes; tolerant to row reordering | [[tests-format#has_line_matching_model|has_line_matching]] for a known canonical row | -| **VCF** | `compare: diff` with `lines_diff: 6` | The `lines_diff: 6` constant matches the typical VCF header preamble that embeds `##fileDate=` and `##source=` | [[tests-format#has_text_matching_model|has_text_matching]] for known variants | -| **BAM** | [[tests-format#has_size_model|has_size]] + [[tests-format#has_archive_member_model|has_archive_member]] (BAM is a gzipped block format) | Headers contain `@PG` lines with command lines and Galaxy job IDs; never byte-diff | Convert to SAM in a downstream step and assert on text | -| **FASTA** (deterministic — assemblies, consensus) | `file:` exact comparison or [[tests-format#has_text_model|has_text]] for known sequence | Output is byte-stable when the upstream tool is deterministic | [[tests-format#has_n_lines_model|has_n_lines]] if line wrapping varies | -| **FASTA** (non-deterministic — RepeatModeler libraries) | `compare: sim_size` with large `delta:` | Family content varies run-to-run | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | -| **FASTQ** (rare as workflow output) | [[tests-format#has_n_lines_model|has_n_lines]] (must be multiple of 4) | Quality scores are read-id-dependent | `compare: sim_size` | -| **JSON** (deterministic — config dumps, params) | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] / [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] | Surgical: assert on the property you care about, ignore the rest | `compare: diff` if the JSON is fully canonical | -| **JSON** (stochastic — HyPhy stats, MCMC results) | `has_text: text: "{"` (existence-only) | Embedded floats break any structural assertion; see [[iwc-shortcuts-anti-patterns]] §1 | [[tests-format#has_h5_keys_model|has_h5_keys]]-equivalent for top-level structural keys | -| **HDF5 / AnnData** | [[tests-format#has_h5_keys_model|has_h5_keys]] + [[tests-format#has_h5_attribute_model|has_h5_attribute]] for known structure | Asserts shape, not values — values are float-noisy | [[tests-format#has_size_model|has_size]] with `delta:` | -| **XML** | [[tests-format#is_valid_xml_model|is_valid_xml]] + [[tests-format#has_element_with_path_model|has_element_with_path]] + `element_text_is`/`element_text_matches` | Surgical structural probes; byte-diff fails on whitespace/attribute-order | [[tests-format#has_n_elements_with_path_model|has_n_elements_with_path]] for cardinality | -| **PNG / image plots** | [[tests-format#has_image_width_model|has_image_width]] + [[tests-format#has_image_height_model|has_image_height]] + [[tests-format#has_size_model|has_size]] with `delta:` (5–10%) | Catches "rendered something with the right dimensions"; corpus norm | [[tests-format#has_image_center_of_mass_model|has_image_center_of_mass]] if mask/segmentation outputs are deterministic | -| **TIFF / multipage images** | [[tests-format#has_image_frames_model|has_image_frames]] + [[tests-format#has_image_channels_model|has_image_channels]] + [[tests-format#has_size_model|has_size]] with `delta:` | Same logic, channel/frame structure matters | — | -| **Archives** (zip, tar.gz) | `has_archive_member: path: "regex"` with nested `asserts:` | Asserts on a specific member; archive timestamps never byte-stable | [[tests-format#has_size_model|has_size]] with `delta:` | -| **GFF / GTF** | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` + [[tests-format#has_text_model|has_text]] for known feature | Tools reorder freely; hard to byte-diff | `has_n_columns: 9` (GFF spec) | -| **Cool / HiC matrices** | `compare: sim_size` with multi-MB `delta:` | Binary, run-to-run variance | [[tests-format#has_archive_member_model|has_archive_member]] for HDF5 component | +## 2. Compare operators and expected artifacts -When in doubt: start with [[tests-format#has_size_model|has_size]] + `delta_frac: 0.1`. It catches the catastrophic failure mode (empty / 10x bigger output). Then add a content probe. +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. -### 1a. If the assertion is too weak, revisit the workflow output +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. -Assertion choice sometimes reveals a workflow-design problem. If the only available output can support only a size check or image-dimension smoke test, check whether the workflow should expose a stronger checkpoint before settling for the weak assertion. +## 3. Set tolerances from the failure boundary -- Scanpy's plot outputs use size and image-dimension assertions, but the same workflow also exposes AnnData HDF5 checkpoints and a cluster-count table (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end uses coarse size bands for coverage/mapped-read outputs, but expression/count tables get stronger regex and exact-line checks (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. -Rule: if a translated workflow exposes only weakly assertable final reports, consult [[galaxy-workflow-testability-design]] and consider promoting a table/text/HDF5 checkpoint before writing the final assertions. +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. -## 2. The `compare:` operators +## 4. Text assertions and repeated checks -Top-level on a `file:` output assertion. Vocabulary, in decreasing strictness: - -- **`diff`** (default). Byte-for-byte equality with optional `lines_diff:` tolerance for a fixed number of header lines. Use only when the upstream tool is deterministic on fixed inputs *and* the output has no embedded timestamps, command lines, version banners, or hash-ordered Python-dict-style keys. -- **`re_match` / `re_match_multiline`**. Each line of the expected fixture is a regex that must match the corresponding output line. Useful when a few fields per row are timestamped but the rest is canonical. Rare in the corpus. -- **`contains`**. The expected fixture is a substring of the output. Cheap; weak. Prefer `asserts: has_text` for new code unless you genuinely have a multi-line block to assert as a whole. -- **`sim_size`**. Output file size matches the fixture's size within `delta:` (bytes) or `delta_frac:` (fraction). Use when the output is necessarily non-deterministic but its rough size is reproducible (RepeatModeler libraries, HiC matrices, Bayesian sampler outputs). - -Picking `lines_diff:`: count the mutable header lines in the output format. VCF: ~6 (`##fileformat`, `##fileDate`, `##source`, `##reference`, contig/info lines vary). SAM/text headers: count `@HD`/`@PG`/`@CO` lines. Set `lines_diff:` to the count exactly — looser values mask real diffs. - -## 3. Tolerance picking - -**`delta:`** is bytes (for [[tests-format#has_size_model|has_size]] and `compare: sim_size`) or absolute count (for [[tests-format#has_n_lines_model|has_n_lines]], [[tests-format#has_n_columns_model|has_n_columns]], [[tests-format#has_image_width_model|has_image_width]], etc.). Suffix multipliers documented in the schema — `1K`, `1M`, `1G` work. - -**`delta_frac:`** is a fraction (0.1 = 10%). Use when expected size scales with input volume. Three IWC tests use it (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`); the rest use absolute `delta:`. - -**Picking magnitudes** (from corpus survey in [[iwc-shortcuts-anti-patterns]] §2): - -- Image dimensions: `delta: 25–30` pixels (5% of typical matplotlib defaults). -- Image file size: `delta: 5K–60K` (5–10% of file size). -- Small text reports: `delta: 1K–10K`. -- HTML reports: `delta: 25K–100K`. -- BAM files: `delta: 1M–10M`. -- RepeatModeler / Bayesian sampler outputs: `delta: 30K–90M` (extreme, but justified by the underlying nondeterminism). - -**Heuristic for new outputs:** `delta_frac: 0.1` is a defensible default. Tighten if the output proves more deterministic than expected. - -## 4. Text family — [[tests-format#has_text_model|has_text]] vs [[tests-format#has_text_matching_model|has_text_matching]] vs [[tests-format#has_line_model|has_line]] vs [[tests-format#has_line_matching_model|has_line_matching]] vs [[tests-format#has_n_lines_model|has_n_lines]] - -All five are common; choose by what you're verifying. - -- **[[tests-format#has_text_model|has_text]]** — output contains the substring `text:`. Anywhere in the output, any number of times. Add `n:` / `min:` / `max:` to constrain occurrence count. Add `delta:` to allow slack on the count. -- **[[tests-format#has_text_matching_model|has_text_matching]]** — output matches the regex `expression:`. Use sparingly; prefer literal [[tests-format#has_text_model|has_text]] when you can. -- **[[tests-format#has_line_model|has_line]]** — output has at least one *line* matching `line:` exactly. Use when line boundaries matter (e.g. asserting on a specific row in a table). -- **[[tests-format#has_line_matching_model|has_line_matching]]** — same but with regex. -- **[[tests-format#has_n_lines_model|has_n_lines]]** — assert the line count is `n:` ± `delta:`. - -A common combination in IWC: `has_n_lines: n: 100, delta: 5` + `has_text: text: "expected_token"` — line-count sanity-check plus a content marker. This catches both truncation and content drift in one assertion pair. - -`negate: true` is supported on every assertion. Used for the "this output should NOT contain X" case. - -## 5. Collection output assertions (`element_tests:`) - -For Galaxy collection outputs, the test format keys element assertions by element identifier: +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. ```yaml -my_collection_output: - element_tests: - sample_1: +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: asserts: - has_text: - text: "expected" - sample_2: - file: test-data/expected_sample_2.txt -``` - -Optional `attributes:` at the collection level can assert on the produced collection shape: - -```yaml -my_collection_output: - attributes: {collection_type: list:list} - element_tests: - ... + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 ``` -Nested collections: outer `element_tests:` keyed by outer identifier; inner uses `elements:` (note plural, no `_tests` suffix on the inner). See [[iwc-test-data-conventions]] §2f for the live example. - -For a list-of-files where every element should pass the same minimal check, the existence-probe pattern (`has_text: "{"` for JSON; `has_size: min: 100` for any non-empty binary) is widely used and accepted in IWC. - -## 6. The validate-against-workflow inner loop - -A `-tests.yml` file can be **structurally invalid** in two distinct ways: - -1. **Schema-invalid** — wrong field names, wrong nesting, wrong types. Caught by the test-format JSON Schema. -2. **Workflow-incoherent** — schema-valid YAML, but the input/output labels don't match the actual workflow. Renaming an output in the `.ga` and forgetting to update its sibling `-tests.yml` produces this case. Planemo will surface it as an "output not found" error at test-runtime, but only after a full workflow run. - -The `@galaxy-tool-util/schema` npm package ships **two** validators that catch both cases statically — no Galaxy or Planemo invocation needed: +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. -- **`validateTestsFile(yaml)`** — runs the file against `tests.schema.json` (AJV). Reports schema violations with paths. -- **`checkTestsAgainstWorkflow(workflow, tests)`** — cross-checks a `.ga` / format2 workflow against a tests file: missing input labels, missing output labels, type incompatibilities (e.g. test supplies a `File` for a parameter typed `int`). +## 5. Collection and structured-output examples -Both are pure-JS, take milliseconds, and have no Galaxy dependency. Wire them into the inner authoring loop: +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" ``` -edit -tests.yml - → validateTestsFile() # schema gate - → checkTestsAgainstWorkflow(.ga, tests) # coherence gate - → planemo workflow_test_on_invocation # assertion gate (no full re-run) - → planemo test # full integration (slow) -``` - -The first two gates short-circuit cheap mistakes before a slow planemo run. They are the static-validation equivalent of `gxwf` for tests, and the `implement-galaxy-workflow-test` mold should reference them as its primary inner-loop tooling. Source: galaxy-tool-util-ts package, `src/test-format/index.ts` exports. - -## 7. Authoring loop — generation, then refinement - -Reviewer convention is to **generate** the initial `-tests.yml` rather than hand-write it. Two planemo subcommands cover this: - -- **`planemo workflow_test_init --from_invocation `** ([[planemo-workflow_test_init]]) — given a successful Galaxy invocation ID, emit a `-tests.yml` with a `job:` block that captures all inputs (with SHA-1 hashes) and an `outputs:` block with `file:` references to the actual outputs (downloaded into `test-data/`). Hand-tighten the assertions afterward. -- **`planemo workflow_test_on_invocation `** ([[planemo-workflow_test_on_invocation]]) — re-evaluate an edited `-tests.yml` against a saved invocation without re-running the workflow. The fast inner loop for assertion iteration; complements the static gates in §6. - -Together these cut the assertion-iteration cost dramatically. An agent should: - -1. Run the workflow once on usegalaxy.* (or local) to get a known-good invocation. -2. `--from_invocation` to bootstrap the test file. -3. Replace the autogenerated `file:` exact-comparison assertions with assertion-family-appropriate alternatives per §1. -4. [[planemo-workflow_test_on_invocation]] after each edit; full [[planemo-test]] at the end. - -## 8. What the schema gives you for free - -When the test-format schema lands as a Foundry-rendered note, the agent can consult any assertion's `$def` directly for: parameter types, defaults, required fields, the `that` discriminator constant, and the original Python docstring (carried through as `description`). This note does **not** restate that vocabulary — it complements it with the corpus-grounded *which-and-when*. - -What's still missing from the schema and worth keeping in research notes: - -- This decision table (§1) — output-type → assertion family. -- Tolerance magnitudes (§3) — corpus-derived defaults. -- The `validateTestsFile` / `checkTestsAgainstWorkflow` integration story (§6). -- Anti-pattern flags — see [[iwc-shortcuts-anti-patterns]]. -## 9. Common combinations (recipes) +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. -Six recipes worth memorizing. +## 6. Validate labels and schema before execution -**Stable text report (FastQC summary, simple stats).** -```yaml -my_report: - asserts: - has_n_lines: { n: 12, delta: 2 } - has_text: { text: "Total Sequences" } -``` - -**MultiQC HTML report.** -```yaml -multiqc_report: - asserts: - has_text: { text: "Filtered Reads" } - has_text: { text: "FastQC" } -``` +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: -**VCF (pinned tool, fixed reference).** -```yaml -called_variants: - file: test-data/expected.vcf - compare: diff - lines_diff: 6 +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga ``` -**Stochastic JSON (HyPhy-style).** -```yaml -hyphy_meme: - element_tests: - geneA: { asserts: { has_text: { text: "{" } } } - geneB: { asserts: { has_text: { text: "{" } } } -``` +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. -**Matplotlib plot.** -```yaml -umap_plot: - asserts: - has_size: { size: 68416, delta: 6000 } - has_image_width: { width: 601, delta: 30 } - has_image_height: { height: 429, delta: 25 } -``` +## 7. Create and refine the test -**AnnData (HDF5).** -```yaml -clustered_anndata: - asserts: - has_h5_keys: { keys: "obs/louvain" } - has_h5_keys: { keys: "var/highly_variable" } - has_h5_keys: { keys: "uns/rank_genes_groups" } - has_size: { size: 12000000, delta: 1500000 } -``` +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. -## 10. Cross-references +## 8. Source of the exact syntax -- [[galaxy-workflow-testability-design]] — decide which workflow outputs and checkpoints to expose before choosing assertions. -- [[iwc-test-data-conventions]] — input-side conventions (job inputs, collection shapes, `hashes:`, CVMFS). -- [[iwc-shortcuts-anti-patterns]] — accepted-vs-smell catalog and corpus prevalence; this note's mirror image. -- Test-format schema (`@galaxy-tool-util/schema` npm package) — authoritative vocabulary; will be vendored into a Foundry-rendered schema note. See `content/meta/casting.md` for the casting story. -- Planemo test-format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- [[galaxy-xsd]] — Galaxy XSD assertion source of truth, vendored from upstream. -- Tightening of the schema and Pydantic source: [galaxyproject/galaxy#22566](https://github.com/galaxyproject/galaxy/pull/22566). -- TS schema sync into npm: [jmchilton/galaxy-tool-util-ts#75](https://github.com/jmchilton/galaxy-tool-util-ts/pull/75). +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md index 59b3c25..72583a5 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-11 -revision: 3 +revised: 2026-10-26 +revision: 4 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[galaxy-tool-job-failure-reference]]" @@ -23,6 +23,8 @@ sources: - "~/projects/repositories/planemo/planemo/galaxy/activity.py" - "~/projects/repositories/planemo/planemo/galaxy/invocations" - "~/projects/repositories/planemo/planemo/galaxy/config.py" + - "~/projects/repositories/planemo/planemo/galaxy/serve.py" + - "~/projects/repositories/planemo/planemo/engine/galaxy.py" summary: "Reference for Planemo workflow test/run architecture, Galaxy modes, API polling, and noisy failure boundaries." --- @@ -94,6 +96,20 @@ Important operations: For workflows, Planemo invokes Galaxy using input labels/names. Stable generated labels are therefore important for both test execution and debugging. +### Workflow Import Failures + +Planemo imports the workflow while it brings Galaxy up for the run (`config.install_workflows()`, called from `_serve()` in `planemo/galaxy/serve.py` for managed Galaxy and from `ExternalGalaxyEngine.ensure_runnables_served` in `planemo/engine/galaxy.py` for external Galaxy, as of planemo 0.75.47). For gxformat2 it converts the workflow to a native dict client-side first, so Galaxy's own gxformat2 conversion never runs. That is outside the per-test `execute()` wrapper that turns exceptions into an `ErrorRunResponse`. When Galaxy rejects the import request, BioBlend raises `ConnectionError`, the test command exits non-zero, and Planemo writes neither the `--test_output_json` report nor the HTML report. No workflow, history, or invocation id exists. + +The evidence then lives in three places: + +| Evidence | What it holds | +|---|---| +| Upload response | HTTP status and body. A 400 usually names the failing field or validation rule. A 500 body rarely explains the cause. | +| Galaxy server log | The traceback for the failing request, the only place a 500's cause appears. Managed Galaxy streams its log into the Planemo terminal output and keeps it on disk. External Galaxy keeps it on the server. | +| Planemo terminal log | The BioBlend exception and the point in the run where it was raised. | + +Re-importing the same converted dict as the same user on the same server reproduces the failure without the rest of the Planemo run, which makes it the cheap loop for isolating a trigger. + Workflow testability design guidance lives in [[galaxy-workflow-testability-design]]. This architecture note only records why Planemo makes labels and workflow-level outputs operationally important. ## Structured Artifacts @@ -121,6 +137,7 @@ Planemo transforms and summarizes Galaxy failure information. These are likely i - Polling may summarize “at least one job is in error” while detailed job evidence is printed elsewhere or stored separately. - Missing outputs become assertion/output problems, which can conflate label mismatch, workflow output omission, optional output absence, failed invocation, or output download issue. - Output download failures may be logged but not always propagated as the most specific failure. +- A rejected workflow import aborts the run before any structured report is written, so terminal and server logs are the only record. - External Galaxy without an admin key may defer missing-tool problems until runtime. - `allow_tool_state_corrections=True` can let Galaxy adjust tool state during invocation, which is useful but can mask definition drift. - `--no_wait` is not suitable for debug conclusions because invocation creation can succeed before jobs fail. @@ -135,7 +152,7 @@ For [[run-workflow-test]]: For [[debug-galaxy-workflow-output]]: -- Start from structured Planemo output. +- Start from structured Planemo output. When there is none because the workflow import failed, start from the upload response and the Galaxy server log. - For missing-output failures, check label drift and omitted workflow-level outputs before assuming tool failure. - If the failure is job-level, inspect Galaxy job APIs described in [[galaxy-tool-job-failure-reference]]. - If the failure is invocation-level, inspect Galaxy invocation APIs described in [[galaxy-workflow-invocation-failure-reference]]. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json index 1539281..82e4529 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json @@ -358,7 +358,7 @@ "properties": { "family": { "type": "string", - "description": "tests-format assertion family or compare operator this intent maps to, by name, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys, file, compare:diff, compare:sim_size. Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." + "description": "tests-format assertion family, compare operator, or output-attribute field this intent maps to, by name. Assertion families go inside the output's `asserts:`, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys. Compare operators and output-attribute fields sit beside `asserts:`: file, compare:diff, compare:sim_size, element_count (collection outputs), count (nested sub-collections), ftype, checksum, metadata. Do not use collection_type (state it in ExpectedOutput.collection_shape) or element_tests (a non-null element_identifier implies it). Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." }, "intent": { "type": "string", @@ -390,15 +390,16 @@ "string", "null" ], - "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions." + "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions. Address an element of a nested collection by joining identifiers outermost first with `/`, e.g. WT_REP1/forward in a list:paired." }, "evidence": { "type": "string", "enum": [ "test-evidence", + "fixture-derived", "intent" ], - "description": "Whether this assertion was translated from upstream test evidence or synthesized from intent." + "description": "Basis of the expected value: test-evidence means translated from an upstream assertion, checksum, or expected file; fixture-derived means measured from upstream test fixture bytes where no upstream assertion exists (e.g. snapshot-only nf-tests); intent means synthesized from workflow intent, wrapper XML, or step state with no upstream anchor." }, "confidence": { "type": "string", diff --git a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/tests-format.schema.json b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/tests-format.schema.json index 8de90e7..99d551f 100644 --- a/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/tests-format.schema.json +++ b/plugins/foundry-skills/skills/freeform-summary-to-galaxy-test-plan/references/schemas/tests-format.schema.json @@ -1110,6 +1110,18 @@ "default": "Collection", "title": "Class" }, + "count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Count" + }, "element_tests": { "anyOf": [ { @@ -1171,6 +1183,30 @@ ], "default": null, "title": "Elements" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Maximum Count" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum Count" } }, "title": "TestCollectionCollectionElementAssertions", @@ -1366,6 +1402,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestCollectionDatasetElementAssertions", @@ -1679,6 +1728,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestDataOutputAssertions", diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/SKILL.md b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/SKILL.md new file mode 100644 index 0000000..4f84e31 --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/SKILL.md @@ -0,0 +1,81 @@ +--- +name: freeform-summary-to-workflow-brief +description: "Turn a freeform source summary and selected scope into a Workflow Brief without Galaxy design assumptions." +--- + +# freeform-summary-to-workflow-brief + +Follow the procedure below and use the artifact/reference sections as the runtime contract. + +## When To Use + +- Turn a freeform source summary and selected scope into a Workflow Brief without Galaxy design assumptions. + +## Inputs + +- Read artifact `freeform-summary`. Produced by `interview-to-freeform-summary`, `summarize-paper`. Source evidence supplied by the preceding summarizer; preserve uncertainty and the caller’s selected scope. + +## Outputs + +- Write artifact `workflow-brief` as `workflow-brief.md`. Format: `markdown`. Schema: workflow-brief-schema. Expert-editable workflow intent and agent environment, with optional details and explicit workflow or environment blockers. + +## Required Tools + +- **`foundry`** (foundry). `npm install -g @galaxy-foundry/gxwf-foundry`. + Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. + Check: `foundry --help`. + Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md + +## Load Upfront + +- `references/notes/workflow-brief-design.md`: Research note copied verbatim into the bundle. Apply the brief boundaries: evidence and expert intent, no inferred Galaxy design, and separate workflow/environment blockers. +- `references/schemas/workflow-brief-schema.schema.json`: Schema file copied verbatim into the bundle. Carry the Markdown section declaration and its structural validator. + +## Load On Demand + +- `references/cli/check-workflow-brief.json`: CLI command reference packaged as a sidecar. Report declared blockers separately from structural validity. Use when: after structural validation and before reporting whether the brief is clear of declared blockers. +- `references/cli/validate-workflow-brief.json`: CLI command reference packaged as a sidecar. Validate the emitted brief before returning it. Use when: after writing workflow-brief.md. + +## Validation + +- Before starting, check that the installed validator carries the workflow-brief-schema schema this bundle was cast with: run `foundry validate-workflow-brief --schema-sha256 8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `workflow-brief.md` before returning it: run `foundry validate-workflow-brief --schema-sha256 8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0 workflow-brief.md` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-workflow-brief --schema-sha256 8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0 workflow-brief.md`. This checks artifact `workflow-brief` against the workflow-brief-schema schema. + +## Procedure + +Read the narrative or interview-derived `freeform-summary` and emit a Workflow Brief for expert review. + +### Evidence and scope + +Read the source summary and the caller's stated objective and selection. Identify which scientific work is included and excluded; do not copy every source analysis into scope. If the selection is too ambiguous to proceed, state that uncertainty explicitly in Workflow/Blockers rather than choosing an analysis for the expert. + +Write `workflow-brief.md` with `## Workflow`, `### Objective`, and `## Agent Environment`. Add optional subsections only when evidence or expert intent supports them. Identify the source and caller instructions, distinguish source facts from expert requirements, and keep scientific inputs, outputs, and acceptance criteria high-level. Missing optional detail does not require filler sections. + +The writing agent must not infer Galaxy datatypes, collections, interface labels, available tools, or wrapper versions. It must not write step graphs, concrete tool states, or final test declarations. Named source tools and formats are evidence, not Galaxy choices. Explicit expert-provided Galaxy requirements may be recorded with attribution. + +### Agent environment + +Use supplied environment evidence or harmless availability/version probes to describe expected `gxwf`, `foundry`, and `planemo` tooling, agent constraints, container preferences, and Docker/Singularity/Apptainer usability. Distinguish requested requirements, observed facts, and unknowns. Do not claim a container engine works based only on an executable being on PATH. Do not install tools or change the environment as part of brief production. + +Place known issues preventing work in `### Blockers` under the appropriate parent. Put other unresolved choices in Workflow/Open questions, each with a proposed answer and its evidence. Omit or leave Blockers empty when there are none. Any text there, including “none”, is blocking. + +### Validate and hand off + +Run `foundry validate-workflow-brief workflow-brief.md`. Fix structural failures without inventing scientific intent. Run `foundry check-workflow-brief workflow-brief.md --json` and retain its result in the run report. Exit `4` means a valid but blocked brief: return that brief and its blockers for expert/environment resolution. Do not suppress the blockers to obtain a clear check. + +Return the brief for expert editing and review. Producing a brief does not authorize Galaxy design or implementation, and a clear static check does not replace review or environment preflight. Keep source evidence available to the later consumer. + +## Feedback Mode + +- Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. +- When enabled, read `_feedback.md` before doing the work and use its registered `foundry-feedback.ledger.yml` protocol. +- Preserve harness-owned run and phase state. Append only concrete observations about a canonical Foundry source asset or a related project that this run showed to be at fault; do not put ordinary workflow requirements in this ledger. +- Before reporting completion, make one explicit pass over the work you just did. Do not ask yourself whether anything was unclear — recall what happened: where you guessed at something the instructions should have settled, needed information this bundle does not carry, hit an instruction that contradicted another or contradicted the artifacts in front of you, used a packaged reference that did not cover your case, or did something the procedure never describes. +- Append an entry for each such event that clears the protocol's bar. If none do, append nothing and report `no feedback` explicitly. Silence and a clean pass are not the same thing, and nothing downstream can tell them apart unless you say which one it was. +- Pass the same ledger path to any subagent used for this work, and merge updates serially so one writer cannot overwrite another. + +## Runtime Notes + +- Do not read Foundry source files at runtime; use only files packaged in this skill bundle and user-supplied artifacts. +- Preserve declared artifact filenames unless the user or harness supplies explicit paths. +- Carry unresolved assumptions into the output artifact instead of silently inventing missing source evidence. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_feedback.md b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_feedback.md new file mode 100644 index 0000000..61a4bd4 --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_feedback.md @@ -0,0 +1,143 @@ +# Foundry feedback ledger + +The `foundry-feedback-ledger` is an opt-in runtime artifact for observations about the Foundry +assets used during a real cast-skill or pipeline run. It records actionable gaps, defects, +friction, and wishes about Molds and their authored references. It does not record unmet +requirements in the workflow being built; those belong in the open-requirements ledger. + +Every entry is destined for `galaxyproject/foundry`. When a run shows that a related project is +at fault — Galaxy, a tool wrapper, the IWC corpus, a harness CLI — that observation still belongs +here, recorded with the analysis that points at the upstream. The Foundry is where the run +context lives, so it is where the issue is tracked; a maintainer forwards from there. Never tell +a user to take an observation to another tracker instead. + +The registered filename is `foundry-feedback.ledger.yml`. Feedback-aware callers pass the same +file through every skill invocation in a run. + +## Ledger shape + +```yaml +run: + pipeline: nextflow-to-galaxy + run_slug: sarek-galaxy + status: running + phases: + - { n: 1, kind: mold, skill: summarize-nextflow, status: done, feedback_checked: true } + - { n: 2, kind: mold, skill: nextflow-summary-to-galaxy-interface, status: running } +entries: + - id: channel-shape-note-silent-on-optional-outputs + raised_by: nextflow-summary-to-galaxy-interface + observed_in: + mold: nextflow-summary-to-galaxy-interface + path: content/molds/nextflow-summary-to-galaxy-interface/index.md + revision: 6 + content_hash: 82d8... + foundry_head: d9e0b09 + subject: + kind: research + label: nextflow-to-galaxy-channel-shape-mapping + locator: content/research/nextflow-to-galaxy-channel-shape-mapping/index.md + content_hash: a18c... + kind: gap + severity: major + what: "The note does not cover optional outputs." + expected: "State how an absent optional channel maps to the Galaxy interface." + evidence: "Interface design stopped at the optional output decision." + status: open + issue: null +``` + +Entry `kind` is `gap`, `defect`, `friction`, or `wish`. Severity is `blocker`, `major`, or +`minor`. Status is `open`, `filed`, `duplicate`, or `wontfix`. + +## Run lifecycle + +For a pipeline run, the harness creates the file before phase 1 with `run.status: running`, an +empty `entries` list, and the complete top-level phase roster from its assembly record. All +phases begin `pending`. The harness changes a phase to `running` immediately before invocation +and to `done` on success. A phase that terminates the run and the run itself become `failed`; a +graceful user stop makes the run `cancelled`. A hard interruption naturally leaves the run +`running`, which is distinguishable from success without a recovery write. + +A phase also carries `feedback_checked`. The harness sets it true only when that phase reported a +feedback outcome — entry ids it appended, or an explicit `no feedback`. It is absent or false +otherwise, including for a phase that simply never said. This is what separates a run that looked +for friction and found none from a run that never looked. + +Loop phases remain one row and increment `iterations` after each successful iteration. Branch +phases remain one row and record the chosen path in `selected`. After every intended phase is +done, the harness sets `run.status: complete`. + +For a standalone skill invocation, the first feedback-aware skill creates a ledger with +`pipeline: null`, the caller's run slug when one exists, and one phase row for itself. It owns +that row's lifecycle as well as any entry it appends. + +No ledger means feedback mode was off. An empty `entries` list is evidence of a clean run only +when `run.status` is `complete` **and** every phase is `feedback_checked`. A complete run with +unchecked phases is complete but unreviewed; an empty incomplete ledger is neither. + +## When to append + +Appending is a deliberate pass, not a background reflex. Before a skill reports completion it +recalls what actually happened during its work — where it guessed at something the instructions +should have settled, needed information its bundle does not carry, hit an instruction that +contradicted another or contradicted the artifacts in front of it, used a reference that did not +cover the case, or did something its procedure never describes. Those are recallable events. "Was +anything unclear?" is not a useful question to ask at that moment and reliably answers itself no. + +The pass ends in one of two outcomes, and the outcome is always stated: entries appended, or +`no feedback`. A skill that says nothing has not reported a clean pass. + +Two entry shapes are admissible. Both require that `what` names a concrete problem, `expected` +states a proposed correction, and the entry is not already present in this ledger for the same +subject and correction. + +**A Foundry source asset.** The observation concerns Foundry-authored content, implementation, or +a registry that could be changed directly, and `subject.locator` canonically identifies it. This +is the ordinary case and the one the clustering and hash checks are built for. + +**A related project.** The run showed that a project the Foundry drives or cites is at fault — +Galaxy, a Tool Shed wrapper, the IWC corpus, `gxwf`, planemo. Use `subject.kind: related-project`, +name the project and the specific component in `label`, and put its public identity in +`locator` (a repository URL, a tool id, an IWC workflow id). There is no `content_hash`. State +in `what` and `evidence` what the run actually observed, and in `expected` what the upstream +would have to do. The entry is still filed against the Foundry. + +Do not use feedback entries for ordinary workflow requirements, tool-selection uncertainty, or +an unsupported preference with nothing concrete to change. + +## Canonical identity and provenance + +For a Foundry subject, use a repository-relative authored path as `subject.locator`; for a +package-export schema, use `package://#`. Generated skills and harnesses are +never subjects; identify the Mold, Pipeline, caster, assembler, or registry that must change. + +Initial subject kinds are `mold`, `pipeline`, `pattern`, `source-pattern`, `research`, `schema`, +`prompt`, `cli-tool`, `cli-command`, `meta`, `implementation`, and `registry`, plus +`related-project` for the upstream case above. `label` is for display only; the locator is the +identity and clustering key. + +A `related-project` subject is the one kind whose locator is not a Foundry path: it is public +upstream identity, carries no content hash, and is never checked against Foundry main. + +For the running Mold, copy `mold.name`, `mold.path`, `mold.revision`, and `mold.content_hash` +from `_provenance.json` into `observed_in`. Record its `mold.commit` value as `foundry_head`, but +never use that commit as exact content identity: it can be null or predate the bytes in the cast. +For a packaged reference, copy its `refs[].src` and `refs[].src_hash` into the subject locator and +content hash. When the subject is the Mold itself, use its authored path and +`observed_in.content_hash`. + +## Safe updates + +Read the complete ledger immediately before changing it. Preserve run state and all entries you +do not own. Append a complete entry in one rewrite; do not leave a partial YAML document. The +harness is the sole writer of pipeline lifecycle fields, while each skill writes only entries. + +Keep evidence concise and safe to upstream. Never copy credentials, tokens, private repository +URLs, proprietary source text, patient or participant data, or user-identifying filesystem +paths into the ledger. Replace a run-directory prefix with `/` and summarize sensitive +evidence instead of quoting it. + +The reporting skill may set an open entry to `filed`, `duplicate`, or `wontfix` and record the +resolved issue URL after the corresponding disposition is confirmed. It never deletes an entry; +the ledger remains the run audit trail. diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_provenance.json b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_provenance.json new file mode 100644 index 0000000..ee59db1 --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_provenance.json @@ -0,0 +1,96 @@ +{ + "provenance_schema_version": 4, + "cast_target": "claude", + "mold": { + "name": "freeform-summary-to-workflow-brief", + "path": "content/molds/freeform-summary-to-workflow-brief/index.md", + "revision": 2, + "content_hash": "c4ed7645e511fb513295d1c5a031a3ad9dee4210522ac4880dd577bbe5931f24", + "commit": "35f525ac6afd41a0e964d635e09661cc17317b13" + }, + "cast_at": "2026-10-23T14:41:04.005Z", + "refs": [ + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[check-workflow-brief]]", + "src": "content/cli/foundry/check-workflow-brief.md", + "dst": "references/cli/check-workflow-brief.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Report declared blockers separately from structural validity.", + "trigger": "After structural validation and before reporting whether the brief is clear of declared blockers.", + "verification": "Check the committed fixtures and review a worked run for faithful evidence and unchanged brief input.", + "src_hash": "7fd8bccdd0f5a2a62cc51e00589e9e263ce633a4c9a75ba3753d4d01ae822895", + "dst_hash": "a5ad5bcca89d0e0db6f2e4503bf48cfec962c5282e89d2d9a3b3738eda11e8b3", + "source": "deterministic" + }, + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[validate-workflow-brief]]", + "src": "content/cli/foundry/validate-workflow-brief.md", + "dst": "references/cli/validate-workflow-brief.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Validate the emitted brief before returning it.", + "trigger": "After writing workflow-brief.md.", + "verification": "Check the committed fixtures and review a worked run for faithful evidence and unchanged brief input.", + "src_hash": "85992477f91f2131695e7d9ec5d798f99e4800b276aaefd1d763aeb2b10ea105", + "dst_hash": "4e4a4cb44d6d34f90080ecc6f9cdaff68abce96babfdcf289acfa7337cf8b737", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[workflow-brief-design]]", + "src": "content/research/workflow-brief-design/index.md", + "dst": "references/notes/workflow-brief-design.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Apply the brief boundaries: evidence and expert intent, no inferred Galaxy design, and separate workflow/environment blockers.", + "verification": "Check the committed fixtures and review a worked run for faithful evidence and unchanged brief input.", + "src_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "dst_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[workflow-brief-schema]]", + "src": "package://@galaxy-foundry/gxwf-foundry#workflowBriefSchema", + "dst": "references/schemas/workflow-brief-schema.schema.json", + "used_at": "both", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Carry the Markdown section declaration and its structural validator.", + "src_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "dst_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "source": "deterministic" + } + ], + "artifacts": { + "produces": [ + { + "id": "workflow-brief", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "description": "Expert-editable workflow intent and agent environment, with optional details and explicit workflow or environment blockers." + } + ], + "consumes": [ + { + "id": "freeform-summary", + "description": "Source evidence supplied by the preceding summarizer; preserve uncertainty and the caller’s selected scope.", + "producers": [ + "interview-to-freeform-summary", + "summarize-paper" + ] + } + ] + } +} diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_required_tools.json b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_required_tools.json new file mode 100644 index 0000000..b6d5500 --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_required_tools.json @@ -0,0 +1,16 @@ +[ + { + "tool": "foundry", + "origin": "npm", + "package": "@galaxy-foundry/gxwf-foundry", + "invoke": "foundry", + "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", + "availability_check": "foundry --help", + "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", + "source": "implied", + "implied_by": [ + "foundry check-workflow-brief", + "foundry validate-workflow-brief" + ] + } +] diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_verify.json b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_verify.json new file mode 100644 index 0000000..5b98097 --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/_verify.json @@ -0,0 +1,19 @@ +{ + "verify_schema_version": 1, + "entries": [ + { + "artifact_id": "workflow-brief", + "direction": "output", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "validator_bin": "foundry", + "args": [ + "validate-workflow-brief", + "--schema-sha256", + "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "{artifact_path}" + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/cli/check-workflow-brief.json b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/cli/check-workflow-brief.json new file mode 100644 index 0000000..f0c765e --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/cli/check-workflow-brief.json @@ -0,0 +1,31 @@ +{ + "type": "cli-command", + "tool": "foundry", + "command": "check-workflow-brief", + "summary": "Check Workflow Brief structure and stop on explicit workflow or agent environment blockers.", + "source_path": "content/cli/foundry/check-workflow-brief.md", + "source_revision": 1, + "package": "@galaxy-foundry/gxwf-foundry", + "description": "Check Workflow Brief structure and explicit workflow/environment blockers.", + "synopsis": "foundry check-workflow-brief [options] ", + "args": [ + { + "raw": "brief.md", + "name": "brief.md", + "required": true, + "variadic": false + } + ], + "options": [ + { + "flags": "--json", + "name": "json", + "description": "Emit machine-readable structure and blocker diagnostics", + "takesArgument": false, + "optionalArgument": false, + "negatable": false, + "defaultValue": false + } + ], + "body": "# `foundry check-workflow-brief`\n\nRead a [[workflow-brief-design]] document and check its structure and declared blockers.\n\n## Output\n\nExit `0` means structurally valid with no declared blockers. Structural failure exits `3`; a valid brief with nonempty Blockers under Workflow or Agent Environment exits `4`. Input read failures exit `1`.\n\nWithout `--json`, ready status prints on stdout; errors and blockers print on stderr. With `--json`, stdout contains `valid`, `ready`, `errors`, and `blockers`. Blockers identify their parent section, heading line, and Markdown body. Input read errors still print on stderr.\n\n## Examples\n\n```sh\nfoundry check-workflow-brief workflow-brief.md\nfoundry check-workflow-brief workflow-brief.md --json\n```\n\n## Gotchas\n\n- Any content in `### Blockers` blocks work, including “none”. Omit or leave the section empty when there are no blockers. Comments and separators alone are empty.\n- The two Blockers sections are scoped to their respective Workflow and Agent Environment parents.\n- A clear check establishes document shape and absence of declared blockers. Expert review, semantic completeness, and observed tooling/container availability still need their own checks.\n- This command reads the document without editing it." +} diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/cli/validate-workflow-brief.json b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/cli/validate-workflow-brief.json new file mode 100644 index 0000000..cef0d8c --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/cli/validate-workflow-brief.json @@ -0,0 +1,31 @@ +{ + "type": "cli-command", + "tool": "foundry", + "command": "validate-workflow-brief", + "summary": "Validate the title and required sections of a Markdown Workflow Brief.", + "source_path": "content/cli/foundry/validate-workflow-brief.md", + "source_revision": 4, + "package": "@galaxy-foundry/gxwf-foundry", + "description": "Validate a Workflow Brief Markdown document (experimental RFC).", + "synopsis": "foundry validate-workflow-brief [options] [brief.md]", + "args": [ + { + "raw": "brief.md", + "name": "brief.md", + "required": false, + "variadic": false + } + ], + "options": [ + { + "flags": "--schema-sha256 ", + "name": "schemaSha256", + "description": "Exit 4 unless this validator's schema has this sha256 (a cast bundle's _verify.json pin); with no artifact, check only the pin", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + } + ], + "body": "# `foundry validate-workflow-brief`\n\nValidate a Markdown document against the section contract described in [[workflow-brief-design]].\n\n## Output\n\nPrints `: valid` to stdout on success (exit `0`). Missing, duplicate, or empty required sections produce diagnostics on stderr and exit `3`. Input read failures exit `1`. With `--schema-sha256 ` it first compares the hash of the schema it carries against ``, the pin a cast bundle records in `_verify.json`: a mismatch exits `4` without validating, because the validator and the bundle disagree about the contract (see [[schema-packages]]). Given the pin and no artifact, it checks only the pin.\n\n## Examples\n\n```sh\nfoundry validate-workflow-brief workflow-brief.md\n```\n\n## Gotchas\n\n- This checks the document title, required sections, their content, and parent-scoped subsections. It does not verify semantic completeness, expert approval, or environment readiness.\n- Headings inside code fences, block quotes, and HTML comments do not satisfy section requirements.\n- A valid brief may contain blockers. Use `foundry check-workflow-brief` to stop on workflow or environment blockers.\n- Additional sections, prose, tables, and explicit unknowns are allowed. Heading case and section order are flexible." +} diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/notes/workflow-brief-design.md b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/notes/workflow-brief-design.md new file mode 100644 index 0000000..8bc1e6d --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/notes/workflow-brief-design.md @@ -0,0 +1,96 @@ +--- +type: research +title: "Workflow Brief" +tags: + - meta +status: draft +created: 2026-09-17 +revised: 2026-10-19 +revision: 6 +summary: "Markdown Workflow Brief definition and section contract for scope, constraints, agent environment, and explicit blockers." +--- + +# Workflow Brief + +A Workflow Brief is an expert-editable document describing what a workflow should achieve, what it should leave out, and the constraints under which an agent will build and run it. It is an input to design and implementation. The draft workflow captures the proposed steps, wiring, and tool state. + +## Document shape + +The brief is Markdown with a title and two sections: `## Workflow` and `## Agent Environment`. The workflow needs an objective; most details are optional. The agent environment must contain a description, which can explicitly record what still needs to be checked. + +The `workflowBriefSchema` declaration uses the shared engine described in [[markdown-document-contract]] and lives in `packages/gxwf-foundry/src/workflow-brief.ts`. An illustrative document lives in `packages/gxwf-foundry/test/fixtures/workflow-brief/read-alignment.md`. + +| Heading | Required | Content | +| --- | --- | --- | +| `## Workflow` | Yes | Scientific intent and scope. | +| `### Objective` | Yes, under Workflow | Scientific goal and desired outcome. | +| `### Sources` | No | Papers, interviews, repositories, relevant source pins, and evidence availability. | +| `### Scope` | No | Optional `#### Included` and `#### Excluded` describe selected work and boundaries. | +| `### Inputs and outputs` | No | Scientific roles and source-supported formats or organization. | +| `### Requirements and preferences` | No | Expert or source requirements, their strength, rationale, and evidence. | +| `### Acceptance criteria` | No | High-level behaviors that establish success. | +| `### Open questions` | No | Unresolved choices that do not prevent work, each with a proposed answer and its evidence. | +| `### Decisions` | No | Expert decisions and rationale. | +| `### Related artifacts` | No | Links to source evidence and other relevant inputs. | +| `### Blockers` | No, under Workflow | Unresolved issues that prevent design or implementation. Any content blocks work. | +| `## Agent Environment` | Yes | Agent tooling and operating conditions. | +| `### Tooling` | No | Expected Foundry tooling and versions, and what is known about availability. | +| `### Constraints` | No | Agent workspace, network, installation, resource, and access restrictions. | +| `### Containerization` | No | Container preference and Docker, Singularity, or Apptainer availability. | +| `### Blockers` | No, under Agent Environment | Environment issues that prevent work. Any content blocks work. | + +Headings are case-insensitive; section order is flexible and additional sections are allowed. Recognized sections are unique within their parent. If present, ordinary sections need content beyond headings, comments, or separators. Blockers may be empty. Scope's Included and Excluded subsections are optional. + +## Workflow content + +The writing agent must record evidence and expert intent without making Galaxy design choices. It must not assume Galaxy datatypes, dataset collection structures, tool availability, interface labels, or concrete tool versions. Source formats and explicitly named source tools can be cited as evidence; they do not establish a Galaxy mapping or an installed wrapper. An expert may supply specific Galaxy requirements here, and the agent must identify them as expert-provided rather than inferred. + +The agent must not write a step graph, concrete tool state, or final test declarations into the brief. Interface design, data-flow design, tool discovery, and test development belong to the downstream design and implementation Molds. Inputs and outputs stay at scientific roles and source-supported descriptions. Acceptance criteria stay high-level; detailed fixtures and assertions belong to test development. + +Requirements and preferences describe what the expert or source asks for. Agent Environment Constraints describe restrictions on the agent's work. Uncertainty that prevents proceeding belongs in Blockers. Record every other unresolved choice in Open questions with a proposed answer and the evidence for it, such as source behavior, the expert's wording, or the available test data. Propose the answer the evidence best supports. Where expert intent and source behavior conflict, propose the expert's intent and cite the conflict. When the evidence is thin, still propose an answer, mark it low confidence, and say what evidence is missing. The expert accepts a proposal by leaving it unedited and overrides it by editing the answer. + +Write each open question as a bullet with `Proposed:` and `Evidence:` sub-bullets. Add `Confidence: low` and `Missing:` when the evidence is thin: + +```md +- Should runs of one sample be merged before trimming? + - Proposed: yes, merge them as the source does. + - Evidence: the source merges sample-sheet rows sharing a sample ID. The selection does not mention merging. + - Confidence: low + - Missing: whether the expert's samples ever span several runs. +``` + +## Agent environment + +Tooling records whether expected tools such as `gxwf`, `foundry`, and `planemo` are available at the required versions. Distinguish intended requirements from observed availability. Containerization records the preference and whether Docker, Singularity, or Apptainer is actually usable, rather than assuming that an executable name proves a working engine. + +A separate preflight result should record observed versions, container availability, source/reference access, writable paths, timestamps, and diagnostics. A requested installation strategy is not evidence of an installed executable. + +## Validation and blockers + +```sh +foundry validate-workflow-brief workflow-brief.md +foundry check-workflow-brief workflow-brief.md --json +``` + +Validation checks the title, required sections, unique recognized headings, content, and parent-scoped subsections. Example headings in fences, quotes, lists, or comments do not satisfy requirements. Success exits `0`, structural failures exit `3`, and input read failures exit `1`. + +The static readiness check adds a check for `### Blockers` under both Workflow and Agent Environment. A structurally valid document with content in either section exits `4`; a structurally valid document without blockers exits `0`. Omit or leave Blockers empty when there are no blockers. Do not write “none” there: any content, including that word, is blocking. Comments and separators alone are empty. The JSON result includes `valid`, `ready`, `errors`, and blocker bodies with their parent and source line. + +A clear static check only establishes structure and the absence of declared blockers. It does not establish semantic completeness, expert approval, or successful environment preflight. The harness must stop on declared blockers and apply its own review and preflight checks before implementation. + +The exported `WorkflowBrief` and `WorkflowBriefSection` types describe the parsed document. `WorkflowBriefReadinessResult` describes the static check. These checks read the document without changing it. + +## Lifecycle + +1. A source-to-brief producer assembles evidence, proposed scope, and unanswered questions without Galaxy design assumptions. +2. The expert edits the brief and settles blockers needed before the next stage. +3. The harness checks structure, declared blockers, review, and environment readiness. +4. Brief-to-Galaxy consumes the brief and referenced evidence, then runs design, draft, implementation, and test development. + +Brief-to-Galaxy and every design or implementation step must leave the brief unchanged. Record durable workflow knowledge in `open-requirements.ledger.yml`: evidence as discoveries, authorized choices as decisions, unmet needs as obligations, and changes to expert intent as proposed brief-change recommendations. Ordinary phase progress remains harness state. Updating the brief belongs to a separate expert editing step; an implementation agent must not resolve a blocker by rewriting its input. + +A hand-authored brief is a first-class input. Implementation should work in a fresh session using the document and its referenced artifacts. + +## Open design questions + +- Should expert questions live here or link to a separate decision record? diff --git a/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/schemas/workflow-brief-schema.schema.json b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/schemas/workflow-brief-schema.schema.json new file mode 100644 index 0000000..92c431e --- /dev/null +++ b/plugins/foundry-skills/skills/freeform-summary-to-workflow-brief/references/schemas/workflow-brief-schema.schema.json @@ -0,0 +1,103 @@ +{ + "name": "Workflow Brief", + "title": "required", + "sections": [ + { + "heading": "Workflow", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Objective", + "min": 1, + "max": 1, + "content": true + }, + { + "heading": "Sources", + "max": 1, + "content": true + }, + { + "heading": "Scope", + "max": 1, + "content": true, + "sections": [ + { + "heading": "Included", + "max": 1, + "content": true + }, + { + "heading": "Excluded", + "max": 1, + "content": true + } + ] + }, + { + "heading": "Inputs and outputs", + "max": 1, + "content": true + }, + { + "heading": "Requirements and preferences", + "max": 1, + "content": true + }, + { + "heading": "Acceptance criteria", + "max": 1, + "content": true + }, + { + "heading": "Open questions", + "max": 1, + "content": true + }, + { + "heading": "Decisions", + "max": 1, + "content": true + }, + { + "heading": "Related artifacts", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + }, + { + "heading": "Agent Environment", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Tooling", + "max": 1, + "content": true + }, + { + "heading": "Constraints", + "max": 1, + "content": true + }, + { + "heading": "Containerization", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/SKILL.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/SKILL.md index 8c56385..b0f4608 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/SKILL.md +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/SKILL.md @@ -1,6 +1,6 @@ --- name: implement-galaxy-tool-step -description: "Convert an abstract step into a concrete gxformat2 step using a tool summary." +description: "Convert an abstract step into a concrete gxformat2 step using a tool summary or an authored user-defined tool." --- # implement-galaxy-tool-step @@ -9,13 +9,14 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## When To Use -- Convert an abstract step into a concrete gxformat2 step using a tool summary. +- Convert an abstract step into a concrete gxformat2 step using a tool summary or an authored user-defined tool. ## Inputs - Read artifact `galaxy-tool-summary`. Schema: galaxy-tool-summary. Produced by `summarize-galaxy-tool`. Galaxy tool summary manifest from summarize-galaxy-tool conforming to galaxy-tool-summary; binds the abstract step to a concrete tool's ports via the embedded `parsed_tool` and generated `input_schemas`. -- Read artifact `galaxy-workflow-draft`. Schema: galaxy-workflow-draft. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. gxformat2 skeleton being filled in step by step; the step replaces a placeholder in this draft. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `galaxy-user-tool-definition`. Produced by `author-galaxy-tool-wrapper`. `GalaxyUserTool` YAML from author-galaxy-tool-wrapper, in place of a tool summary when the discover-or-author branch fell through to authoring. The step embeds it under `run:`. +- Read artifact `galaxy-workflow-draft`. Schema: galaxy-workflow-draft. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-template`. gxformat2 skeleton being filled in step by step; the step replaces a placeholder in this draft. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -24,14 +25,14 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli ## Load Upfront -- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Read the entries the design tier recorded against this step before implementing it, and append a blocking entry when the step's declared output cannot be computed from its wired inputs — the fall-through to topology repair, in place of fabricating the missing evidence. +- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Read the entries the design tier recorded against this step before implementing it, and append a blocking entry when the step's declared output cannot be computed from its wired inputs or the chosen wrapper can't produce it — the fall-through to topology repair, in place of fabricating the output or authoring around a wrapper that exists. - `references/schemas/galaxy-tool-summary.schema.json`: Schema file copied verbatim into the bundle. Bind the abstract step against the deterministic tool summary manifest emitted upstream — read `parsed_tool` for ports/datatypes and `input_schemas.workflow_step_linked` for valid step `state` shape. - `references/schemas/galaxy-workflow-draft.schema.json`: Schema file copied verbatim into the bundle. In/out contract: the draft this Mold reads and mutates in place conforms to galaxy-workflow-draft. Cast bundles the JSON Schema alongside the draft-validate CLI checks. @@ -43,9 +44,13 @@ Follow the procedure below and use the artifact/reference sections as the runtim - `references/notes/galaxy-collection-semantics.yml`: Companion file copied verbatim into the bundle. Sibling of `references/notes/galaxy-collection-semantics.md`; read it where that note directs. - `references/notes/galaxy-collection-tools.md`: Research note copied verbatim into the bundle. Insert built-in Galaxy collection-operation steps when a direct tool connection cannot express the needed shape. Use when: a step needs collection construction, filtering, extraction, zipping, unzipping, flattening, merging, or relabeling. - `references/notes/galaxy-tool-job-failure-reference.md`: Research note copied verbatim into the bundle. Preserve concrete tool/job failure evidence while implementing step labels, tool ids, output labels, and collection wiring. Use when: a selected wrapper has explicit failure semantics, dynamic outputs, non-default stdio rules, strict-shell behavior, or runtime-only failure risk. +- `references/notes/galaxy-user-tool-workflow-binding.md`: Research note copied verbatim into the bundle. Bind a step to an authored `GalaxyUserTool` by embedding the definition under `run:`. Use when: the step's tool is an authored `GalaxyUserTool` from author-galaxy-tool-wrapper rather than a Tool Shed or built-in wrapper. - `references/notes/galaxy-workflow-testability-design.md`: Research note copied verbatim into the bundle. Preserve testable output labels and collection element identifiers while replacing abstract steps with concrete gxformat2 steps. Use when: a concrete step changes output labels, emits collection outputs, creates a diagnostic checkpoint, or makes a final output too weakly assertable. +- `references/notes/gxformat2-schema.md`: Research note copied verbatim into the bundle. Name the step part of a connection for the draft's step form, and write a dataset `rename:` value. Use when: a list-form step carries both `id` and `label`, or when setting a dataset `rename:`. +- `references/notes/gxformat2.schema.json`: Companion file copied verbatim into the bundle. Sibling of `references/notes/gxformat2-schema.md`; read it where that note directs. - `references/notes/nextflow-operators-to-galaxy-collection-recipes.md`: Research note copied verbatim into the bundle. Turn operator-derived abstract transforms into concrete Galaxy wiring, collection operations, or review requests. Use when: a concrete step implements behavior traced to map, join, groupTuple, branch, mix, combine, or multiMap. -- `references/notes/nextflow-to-galaxy-channel-shape-mapping.md`: Research note copied verbatim into the bundle. Check whether a concrete tool input/output can preserve the intended source-derived Galaxy collection shape. Use when: implementing concrete steps for source-derived File/list/paired/list:paired/list:list inputs or outputs. +- `references/notes/nextflow-to-galaxy-channel-shape-mapping.md`: Research note copied verbatim into the bundle. Check whether a concrete tool input/output can preserve the intended source-derived Galaxy collection shape. Use when: implementing concrete steps for source-derived File/list/paired/list:paired/list:list inputs or outputs, or when connecting a per-element computed value to a mapped step's tool parameter. +- `references/schemas/parsed-tool.schema.json`: Schema file copied verbatim into the bundle. Field-level shape of the `ParsedTool` payload under the summary's `parsed_tool`, which galaxy-tool-summary leaves as a placeholder: port names, datatypes, and a conditional's `whens[]` (`discriminator`, `is_default_when`) in declaration order. Use when: reading a field of `parsed_tool`, including a conditional's case index or default case. ## Validation @@ -53,25 +58,29 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Procedure -Replace one abstract step in the gxformat2 draft with a concrete tool step, using the upstream tool summary. One invocation resolves exactly the chosen step's `TODO_*` / `_plan_*` slots into a concrete `tool_id`, `tool_version`, `state`, and wrapper-determined port names, and returns the mutated draft. This is the "Implement" leaf of the per-step loop owned by advance-galaxy-draft-step. +Replace one abstract step in the gxformat2 draft with a concrete tool step, using the upstream tool summary or, for a tool authored on fallthrough, its `GalaxyUserTool` definition. One invocation resolves exactly the chosen step's `TODO_*` / `_plan_*` slots into a concrete tool binding, `state`, and wrapper-determined port names, and returns the mutated draft. This is the "Implement" leaf of the per-step loop owned by advance-galaxy-draft-step. Single step in scope. This skill owns the chosen step and the wiring that connects it to ports already in the draft. It does not redesign topology and does not unwind earlier iterations — cross-step rework is the orchestrator's call. ### Sequence 1. **Read the step's plan.** From the galaxy-workflow-draft, take the chosen step's deferred evidence: `_plan_state`, `_plan_context`, `_plan_in`, `_plan_out`, and any `TODO_*` slots the template or data-flow brief left for this phase. -2. **Bind to the tool summary.** Read the galaxy-tool-summary manifest: `parsed_tool` gives concrete input/output port names and datatypes; shape the step's `state` against `input_schemas.workflow_step_linked`. Set `tool_version`, and set `tool_id` — confirming or correcting an identity-pinned id rather than re-deriving a good pin from scratch. For a built-in/stock tool the `tool_id` is the bare id (`Filter1`, `Cut1`, collection ops) and `tool_version` must come from the summary's cached pin — never invent a stock version; the summary already resolved it against the shed via summarize-galaxy-tool. If `input_schemas` is `null`, consult `warnings[]` for why before binding by hand. -3. **Wire ports.** Connect the step's inputs to their upstream producers and its outputs to downstream consumers per the `_plan_in` / `_plan_out` intent, using real wrapper port names. Preserve collection mapping and reduction semantics (galaxy-collection-semantics); for a source-derived shape, check the chosen input/output can actually carry the intended File / list / paired / list:paired shape (nextflow-to-galaxy-channel-shape-mapping). +2. **Bind to the tool summary.** Read the galaxy-tool-summary manifest: `parsed_tool` gives concrete input/output port names and datatypes; shape the step's `state` against `input_schemas.workflow_step_linked`. Set `tool_version`, and set `tool_id` — confirming or correcting an identity-pinned id rather than re-deriving a good pin from scratch. For a Tool Shed wrapper the `tool_id` is the full versioned GUID, `/repos////`, ending in the same version as `tool_version`; a template's identity pin is often versionless, so append the resolved version. Galaxy invokes workflows with exact tool versions by default (planemo does), and a versionless shed id then fails invocation as "required tools are not installed" although static validation passes. For a built-in/stock tool the `tool_id` is the bare id (`Filter1`, `Cut1`, collection ops) and `tool_version` must come from the summary's cached pin — never invent a stock version; the summary already resolved it against the shed via summarize-galaxy-tool. If `input_schemas` is `null`, consult `warnings[]` for why before binding by hand. Read a conditional's cases from `parsed_tool`, not the wrapper XML (parsed-tool): its index in `whens[]` is the case index, and `is_default_when` marks the default, which need not be case 0. + + **Authored user-defined tool.** When the wrapper is a `galaxy-user-tool-definition` from author-galaxy-tool-wrapper, there is no tool summary and no toolbox id to bind. Follow galaxy-user-tool-workflow-binding: embed the `galaxy-user-tool.yml` document unchanged under the step's `run:`, and remove `tool_id`, `tool_version`, and `tool_shed_repository` along with any `TODO` in them. Take `in:` keys from the definition's data and collection input names, list every output a consumer uses under `out:` by its output name, and set non-data values in `state` by input name. Never name the tool by `tool_uuid`, `tool_id`, or `content_id` alone, and do not register the tool to obtain a `uuid`. +3. **Wire ports.** Connect the step's inputs to their upstream producers and its outputs to downstream consumers per the `_plan_in` / `_plan_out` intent, using real wrapper port names. Key a parameter nested in a conditional or section by its `|`-joined path (`input_type|i`), and a repeat instance with `_N` (`pick_from_0|value`), giving each instance its own entry in the repeat in `state` (gxformat2-schema). Preserve collection mapping and reduction semantics (galaxy-collection-semantics); for a source-derived shape, check the chosen input/output can actually carry the intended File / list / paired / list:paired shape (nextflow-to-galaxy-channel-shape-mapping). 4. **Close shape gaps.** When a direct tool connection cannot express the needed shape, insert a built-in collection-operation step (galaxy-collection-tools); for identifier-derived reshaping — regex parsing, nesting swaps, paired assignment — use Apply Rules (galaxy-apply-rules-dsl); for a transform traced to a Nextflow operator (map, join, groupTuple, branch, mix, combine, multiMap), turn it into concrete wiring or a review request via nextflow-operators-to-galaxy-collection-recipes. A built-in step inserted here is itself a stock tool: resolve its concrete `tool_version` through the galaxy-tool-cache flow (summarize-galaxy-tool on the bare id) rather than guessing — or leave it draft-tier for the next loop iteration to realize. -5. **Preserve testability.** Keep output labels and collection element identifiers stable and addressable (galaxy-workflow-testability-design). Do not rename a labeled output, drop a checkpoint, or make a final output too weakly assertable just to satisfy this step's wiring. +5. **Preserve testability.** Keep output labels and collection element identifiers stable and addressable (galaxy-workflow-testability-design). Do not rename a labeled output, drop a checkpoint, or make a final output too weakly assertable just to satisfy this step's wiring. A dataset `rename:` names the output after an input with `#{input}`, or uses a workflow parameter with `${label}`. Tool-XML label variables such as `${on_string}` are not rename variables; Galaxy treats them as untyped run-time parameters (gxformat2-schema). 6. **Validate.** Run draft-validate `--concrete` over the mutated draft: it checks draft-contract rules and gates the extracted concrete subset — including the step just implemented — against full gxformat2. On green, return the draft for the next loop iteration; on red, route the diagnostic back to whichever decision above it implicates. ### Failure ownership -When the wrapper can't cleanly carry what the plan needs — wrong datatype, missing parameter, unsupported collection shape — record where a later failure should be investigated: tool/job failure, data-flow mistake, template wiring mistake, wrapper mismatch, or test/assertion issue. Consult galaxy-tool-job-failure-reference when the selected wrapper has explicit failure semantics (exit-code rules, stdio regex, strict-shell, dynamic outputs); implement the step's labels and wiring so that evidence survives to runtime rather than being erased by the concretization. +When the wrapper carries what the plan needs but a datatype, parameter or collection shape could still fail at runtime, record where a later failure should be investigated: tool/job failure, data-flow mistake, template wiring mistake, wrapper mismatch, or test/assertion issue. Consult galaxy-tool-job-failure-reference when the selected wrapper has explicit failure semantics (exit-code rules, stdio regex, strict-shell, dynamic outputs); implement the step's labels and wiring so that evidence survives to runtime rather than being erased by the concretization. When the step's declared output cannot be computed from the inputs wired to it — the connection graph says the ports connect, but no wired input carries the evidence the output needs — do not fabricate it. Append a **blocking** entry to the open-requirements-ledger and fall through to repair-galaxy-draft-topology rather than emitting a step that validates but can't run. A missing Tool Shed wrapper is not this case; that gap routes through the discover-or-author branch. +When the chosen wrapper runs the step's work but can't produce a declared output, and no step 4 insert closes it, treat it the same way. Set `step` on the blocking entry, name the wrapper and version in `missing`, and quote the summary's ports or option that show the limit. Leave that output's `TODO_*` port unbound, so the step stays drafty for repair, and don't ask for a replacement wrapper. A wrapper that doesn't run the step's work at all was the wrong pick: return to wrapper resolution instead. + ## Feedback Mode - Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/_provenance.json b/plugins/foundry-skills/skills/implement-galaxy-tool-step/_provenance.json index a35228a..8de99be 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/_provenance.json +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "implement-galaxy-tool-step", "path": "content/molds/implement-galaxy-tool-step/index.md", - "revision": 9, - "content_hash": "197be2d9b27741c3f258092f8392654369d1ca234d7fad492387ff69d656292b", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 15, + "content_hash": "2009d579cf2a68f8b31d1a69dd751f3f4d80fe3f19f3606f8cac2dda37aa343b", + "commit": "61da115975d155a36face0e83d5c28f638da7fca" }, - "cast_at": "2026-09-13T20:58:28.102Z", + "cast_at": "2026-11-10T18:57:18.395Z", "refs": [ { "kind": "cli-command", @@ -37,8 +37,8 @@ "evidence": "corpus-observed", "purpose": "Implement identifier-derived collection reshaping via Apply Rules.", "trigger": "When collection element identifiers need regex parsing, nesting-level swaps, regrouping, or paired identifier assignment.", - "src_hash": "7e166c0495b4a5366f49fed4d002419a27cb33fb8eb431374878c1890a5f95a6", - "dst_hash": "7e166c0495b4a5366f49fed4d002419a27cb33fb8eb431374878c1890a5f95a6", + "src_hash": "cd61c61b4613694c685d51cc7dea9c997b3350be04135f387135eed9d9052910", + "dst_hash": "cd61c61b4613694c685d51cc7dea9c997b3350be04135f387135eed9d9052910", "source": "deterministic" }, { @@ -89,8 +89,8 @@ "evidence": "corpus-observed", "purpose": "Insert built-in Galaxy collection-operation steps when a direct tool connection cannot express the needed shape.", "trigger": "When a step needs collection construction, filtering, extraction, zipping, unzipping, flattening, merging, or relabeling.", - "src_hash": "457439b3c3bb23ecf9732d7c7d6e0b8fae976b6e9e26f1b6af2d818843fba601", - "dst_hash": "457439b3c3bb23ecf9732d7c7d6e0b8fae976b6e9e26f1b6af2d818843fba601", + "src_hash": "ca52bc372ad0aa6dc6f0a036a5b5ee499090a36bd3e31cd0215a393c19271531", + "dst_hash": "ca52bc372ad0aa6dc6f0a036a5b5ee499090a36bd3e31cd0215a393c19271531", "source": "deterministic" }, { @@ -104,8 +104,24 @@ "evidence": "corpus-observed", "purpose": "Preserve concrete tool/job failure evidence while implementing step labels, tool ids, output labels, and collection wiring.", "trigger": "When a selected wrapper has explicit failure semantics, dynamic outputs, non-default stdio rules, strict-shell behavior, or runtime-only failure risk.", - "src_hash": "bf6530fca24f40780836e92d8d263f789038a6e0a722351ab8e8c9aa42646e2c", - "dst_hash": "bf6530fca24f40780836e92d8d263f789038a6e0a722351ab8e8c9aa42646e2c", + "src_hash": "188511486605e05d6d6ffb555b9c31287a1b6bd94c1f67dd8e483ce258415bfa", + "dst_hash": "188511486605e05d6d6ffb555b9c31287a1b6bd94c1f67dd8e483ce258415bfa", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-user-tool-workflow-binding]]", + "src": "content/research/galaxy-user-tool-workflow-binding/index.md", + "dst": "references/notes/galaxy-user-tool-workflow-binding.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Bind a step to an authored `GalaxyUserTool` by embedding the definition under `run:`.", + "trigger": "When the step's tool is an authored `GalaxyUserTool` from [[author-galaxy-tool-wrapper]] rather than a Tool Shed or built-in wrapper.", + "verification": "Promote after a worked run binds an authored tool this way and the extracted workflow imports into Galaxy with the tool resolved and runs.", + "src_hash": "cfdf62f982349c02cdfa5f3214f3feb9f54cf664b880e438294e5748f9002902", + "dst_hash": "cfdf62f982349c02cdfa5f3214f3feb9f54cf664b880e438294e5748f9002902", "source": "deterministic" }, { @@ -119,10 +135,47 @@ "evidence": "corpus-observed", "purpose": "Preserve testable output labels and collection element identifiers while replacing abstract steps with concrete gxformat2 steps.", "trigger": "When a concrete step changes output labels, emits collection outputs, creates a diagnostic checkpoint, or makes a final output too weakly assertable.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[gxformat2-schema]]", + "src": "content/research/gxformat2-schema/index.md", + "dst": "references/notes/gxformat2-schema.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Name the step part of a connection for the draft's step form, and write a dataset `rename:` value.", + "trigger": "When a list-form step carries both `id` and `label`, or when setting a dataset `rename:`.", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", + "dst_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[gxformat2-schema]]", + "src": "content/research/gxformat2-schema/gxformat2.schema.json", + "dst": "references/notes/gxformat2.schema.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Name the step part of a connection for the draft's step form, and write a dataset `rename:` value.", + "trigger": "When a list-form step carries both `id` and `label`, or when setting a dataset `rename:`.", + "companion_of": "references/notes/gxformat2-schema.md", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "2c4b3df730b70d9c648105ced438fc08be306a41f2aff7d4bd2897f72f94d509", + "dst_hash": "2c4b3df730b70d9c648105ced438fc08be306a41f2aff7d4bd2897f72f94d509", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, { "kind": "research", "mode": "verbatim", @@ -148,9 +201,9 @@ "load": "on-demand", "evidence": "corpus-observed", "purpose": "Check whether a concrete tool input/output can preserve the intended source-derived Galaxy collection shape.", - "trigger": "When implementing concrete steps for source-derived File/list/paired/list:paired/list:list inputs or outputs.", - "src_hash": "32efd59cc86563b145819cda38f0692bdd3144aac10ae831dc6d8accd9dcda8d", - "dst_hash": "32efd59cc86563b145819cda38f0692bdd3144aac10ae831dc6d8accd9dcda8d", + "trigger": "When implementing concrete steps for source-derived File/list/paired/list:paired/list:list inputs or outputs, or when connecting a per-element computed value to a mapped step's tool parameter.", + "src_hash": "e8d9515c9ae7da29188e8744410d8459e7ee9acd48af550216076fcf7a02b73b", + "dst_hash": "e8d9515c9ae7da29188e8744410d8459e7ee9acd48af550216076fcf7a02b73b", "source": "deterministic" }, { @@ -162,10 +215,10 @@ "used_at": "runtime", "load": "upfront", "evidence": "hypothesis", - "purpose": "Read the entries the design tier recorded against this step before implementing it, and append a blocking entry when the step's declared output cannot be computed from its wired inputs — the fall-through to topology repair, in place of fabricating the missing evidence.", + "purpose": "Read the entries the design tier recorded against this step before implementing it, and append a blocking entry when the step's declared output cannot be computed from its wired inputs or the chosen wrapper can't produce it — the fall-through to topology repair, in place of fabricating the output or authoring around a wrapper that exists.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -194,8 +247,26 @@ "purpose": "In/out contract: the draft this Mold reads and mutates in place conforms to [[galaxy-workflow-draft]]. Cast bundles the JSON Schema alongside the [[draft-validate]] CLI checks.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", - "dst_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[parsed-tool]]", + "src": "package://@galaxy-tool-util/schema#parsedToolSchema", + "dst": "references/schemas/parsed-tool.schema.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Field-level shape of the `ParsedTool` payload under the summary's `parsed_tool`, which [[galaxy-tool-summary]] leaves as a placeholder: port names, datatypes, and a conditional's `whens[]` (`discriminator`, `is_default_when`) in declaration order.", + "trigger": "When reading a field of `parsed_tool`, including a conditional's case index or default case.", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "3966d0c425e561891113f5c71a2d36a42c00a78cb744bb8970c577b46abcff9b", + "dst_hash": "3966d0c425e561891113f5c71a2d36a42c00a78cb744bb8970c577b46abcff9b", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } @@ -225,6 +296,13 @@ "summarize-galaxy-tool" ] }, + { + "id": "galaxy-user-tool-definition", + "description": "`GalaxyUserTool` YAML from [[author-galaxy-tool-wrapper]], in place of a tool summary when the discover-or-author branch fell through to authoring. The step embeds it under `run:`.", + "producers": [ + "author-galaxy-tool-wrapper" + ] + }, { "id": "galaxy-workflow-draft", "description": "gxformat2 skeleton being filled in step by step; the step replaces a placeholder in this draft.", @@ -236,7 +314,8 @@ "freeform-summary-to-galaxy-template", "implement-galaxy-tool-step", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-template" ] }, { @@ -259,7 +338,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/_required_tools.json b/plugins/foundry-skills/skills/implement-galaxy-tool-step/_required_tools.json index 54fd1c9..18f1089 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/_required_tools.json +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/_required_tools.json @@ -3,9 +3,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/_verify.json b/plugins/foundry-skills/skills/implement-galaxy-tool-step/_verify.json index 119dc53..2d1ff36 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/_verify.json +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-tool-summary", + "--schema-sha256", + "bf77d0735410fa0028bc9bea08edaff3016580508bf27851fdcae65d038f2d94", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-apply-rules-dsl.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-apply-rules-dsl.md index 735041e..c211586 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-apply-rules-dsl.md +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-apply-rules-dsl.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-02 -revision: 2 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-collection-tools]]" - "[[galaxy-collection-semantics]]" @@ -22,1564 +22,146 @@ related_notes: - "[[relabel-via-rules-and-find-replace]]" - "[[iwc-transformations-survey]]" sources: - - "https://github.com/jmchilton/galaxy-agentic-collection-transform (initial research seed)" - - "https://github.com/galaxyproject/galaxy/blob/main/lib/galaxy/util/rules_dsl.py" - - "https://github.com/galaxyproject/galaxy/blob/main/lib/galaxy/util/rules_dsl_spec.yml" -summary: "Reference for Galaxy's Apply Rules DSL: rule operations, mapping operations, composition patterns, pitfalls." + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/util/rules_dsl.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/util/rules_dsl_spec.yml" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/collections.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tools/apply_rules.xml" +summary: "How Apply Rules derives columns from collection metadata, filters rows, and maps them into new collection structure." --- -Reference for Galaxy's Apply Rules DSL — the rule grammar consumed by `__APPLY_RULES__` (see [[galaxy-collection-tools]] for the surrounding tool catalog and [[galaxy-collection-semantics]] for collection mapping/reduction semantics). +# Galaxy Apply Rules DSL -**Key principle:** rules transform collection metadata (identifiers, indices, tags) as tabular data; mapping operations turn the resulting columns back into collection structure. +Galaxy's `__APPLY_RULES__` reorganizes a collection by deriving columns from element identifiers and tags, transforming those columns, and mapping them into a new collection. It can relabel, filter, sort, flatten, or change nesting without processing file contents. The output is named `output`. -**Sources of truth in Galaxy:** -- `lib/galaxy/util/rules_dsl.py` — rule implementation -- `lib/galaxy/util/rules_dsl_spec.yml` — test spec covering every rule type -- `lib/galaxy/managers/collections.py` — collection building from rules -- [PR #5819](https://github.com/galaxyproject/galaxy/pull/5819) — original implementation +Use a dedicated collection tool when it expresses the operation directly, such as flattening or extracting an element. Apply Rules fits transformations that depend on parsed identifiers, tag values, or several operations together. [[galaxy-collection-tools]] lists the alternatives. -This note is the consumer-facing companion to those files. Verify against the spec YAML when in doubt. +## A rule set inside a workflow -## Rules DSL Architecture +The tool parameter named `rules` holds an object with two lists: `rules` performs the transformations and `mapping` defines the output. Mapping entries contain a `type` and a `columns` array. A sibling `tool_state.mapping` or a bare `tool_state.rules` array is not this serialized shape. -### Core Concepts - -**Data Model:** -``` -data: [[cell values]] # 2D array of strings (tabular data) -sources: [source objects] # Metadata for each row (identifiers, indices, tags) -``` - -**Initial State Example:** -```python -# Input: list:paired with elements [sample1/forward, sample1/reverse, sample2/forward, sample2/reverse] - -data = [[], [], [], []] # Empty rows, one per dataset -sources = [ - {"identifiers": ["sample1", "forward"], "indices": [0, 0], "dataset": , "tags": []}, - {"identifiers": ["sample1", "reverse"], "indices": [0, 1], "dataset": , "tags": []}, - {"identifiers": ["sample2", "forward"], "indices": [1, 0], "dataset": , "tags": []}, - {"identifiers": ["sample2", "reverse"], "indices": [1, 1], "dataset": , "tags": []}, -] -``` - -**Execution Flow:** -1. Collection metadata extracted to tabular format -2. Rules applied sequentially to transform data -3. Mapping operations convert transformed data to new collection - -**Example:** -``` -Input collection: list [i1, i2] - -Initial state: - data: [["value1"], ["value2"]] - sources: [ - {"identifiers": ["i1"], "indices": [0]}, - {"identifiers": ["i2"], "indices": [1]} - ] - -After rules: - data: [["value1", "i1"], ["value2", "i2"]] # Added identifier column - -After mapping: - Output collection: list [i1, i2] -``` - ---- - -## Rule Operations - -Rules are applied **sequentially** in the order specified. Each rule transforms the data table. - -### 1. Column Addition Rules - -#### add_column_basename - -**Purpose:** Extract basename from file paths - -**Parameters:** -- `target_column` (int): Column containing paths - -**Example:** -```yaml -rules: - - type: add_column_basename - target_column: 0 -``` - -**Transformation:** -``` -Input: [["/path/to/moo.txt"], ["moo.txt"]] -Output: [["/path/to/moo.txt", "moo.txt"], ["moo.txt", "moo.txt"]] -``` - -**Use cases:** -- Extract filenames from full paths -- Create identifiers from uploaded file paths -- Normalize identifiers across different upload methods - ---- - -#### add_column_regex - -**Purpose:** Capture regex groups or perform replacements - -**Parameters:** -- `target_column` (int): Column to process -- `expression` (string): Regular expression pattern -- `replacement` (string, optional): Replacement template with `\1`, `\2` for groups -- `group_count` (int, optional): Number of groups to capture as separate columns -- `allow_unmatched` (bool, default: false): If false, errors on unmatched rows - -**Mode 1: Simple capture (default)** -```yaml -rules: - - type: add_column_regex - target_column: 0 - expression: '(o)+' -``` -``` -Input: [["foo"], ["cow"]] -Output: [["foo", "oo"], ["cow", "o"]] -``` - -**Mode 2: Replacement** -```yaml -rules: - - type: add_column_regex - target_column: 0 - expression: '(o+)' - replacement: 'the os \1' -``` -``` -Input: [["foo"], ["cow"]] -Output: [["foo", "the os oo"], ["cow", "the os o"]] -``` - -**Mode 3: Multiple groups** -```yaml -rules: - - type: add_column_regex - target_column: 0 - expression: '.*(o)(o)' - group_count: 2 -``` -``` -Input: [["foo"], ["boo"]] -Output: [["foo", "o", "o"], ["boo", "o", "o"]] -``` - -**Mode 4: Allow unmatched** -```yaml -rules: - - type: add_column_regex - target_column: 0 - expression: '(o)+' - allow_unmatched: true -``` -``` -Input: [["foo"], ["cow"], ["cat"]] -Output: [["foo", "oo"], ["cow", "o"], ["cat", ""]] -``` - -**Use cases:** -- Extract sample names from filenames (e.g., `sample_(\w+)_R1.fastq`) -- Parse structured identifiers (e.g., `TCGA-(\w+)-(\d+)`) -- Clean up identifiers (remove prefixes/suffixes) -- Extract metadata embedded in filenames - -**Common patterns:** -```yaml -# Extract sample ID from "sample_123_R1.fastq" -expression: 'sample_(\w+)_R\d' - -# Extract prefix before underscore -expression: '([^_]+)_.*' - -# Extract everything before last dot -expression: '(.+)\.[^.]+$' -``` - ---- - -#### add_column_substr - -**Purpose:** Extract or remove fixed-length substrings - -**Parameters:** -- `target_column` (int): Column to process -- `substr_type` (enum): Operation type - - `keep_prefix`: Keep first N characters - - `keep_suffix`: Keep last N characters - - `drop_prefix`: Remove first N characters - - `drop_suffix`: Remove last N characters -- `length` (int): Number of characters - -**Examples:** -```yaml -# Keep first 2 characters -rules: - - type: add_column_substr - target_column: 0 - substr_type: keep_prefix - length: 2 -``` -``` -Input: [["foo"], ["cow"], ["ba"], ["d"]] -Output: [["foo", "fo"], ["cow", "co"], ["ba", "ba"], ["d", "d"]] -``` - -```yaml -# Drop last 2 characters -rules: - - type: add_column_substr - target_column: 0 - substr_type: drop_suffix - length: 2 -``` -``` -Input: [["foo"], ["cow"], ["ba"], ["d"]] -Output: [["foo", "f"], ["cow", "c"], ["ba", ""], ["d", ""]] -``` - -**Use cases:** -- Remove common prefixes/suffixes -- Extract barcodes from fixed positions -- Truncate long identifiers - ---- - -#### add_column_rownum - -**Purpose:** Add sequential row numbers - -**Parameters:** -- `start` (int): Starting number (0 or 1) - -**Example:** -```yaml -rules: - - type: add_column_rownum - start: 1 -``` -``` -Input: [["foo"], ["cow"], ["ba"], ["d"]] -Output: [["foo", "1"], ["cow", "2"], ["ba", "3"], ["d", "4"]] -``` - -**Use cases:** -- Create numerical identifiers -- Track original row order after sorting -- Generate replicate numbers - ---- - -#### add_column_value - -**Purpose:** Add constant value to all rows - -**Parameters:** -- `value` (string): Constant value - -**Example:** -```yaml -rules: - - type: add_column_value - value: "control" -``` -``` -Input: [["foo"], ["cow"]] -Output: [["foo", "control"], ["cow", "control"]] -``` - -**Use cases:** -- Add condition labels (treatment/control) -- Add constant metadata -- Create separator columns for concatenation - ---- - -#### add_column_concatenate - -**Purpose:** Combine two columns into one - -**Parameters:** -- `target_column_0` (int): First column -- `target_column_1` (int): Second column - -**Example:** -```yaml -rules: - - type: add_column_concatenate - target_column_0: 0 - target_column_1: 1 -``` -``` -Input: [["sample", "001"], ["sample", "002"]] -Output: [["sample", "001", "sample001"], ["sample", "002", "sample002"]] -``` - -**Use cases:** -- Combine sample ID + replicate number -- Build hierarchical identifiers -- Create unique identifiers from multiple parts - -**Common pattern - add separator:** -```yaml -rules: - - type: add_column_value - value: "_" - - type: add_column_concatenate - target_column_0: 0 - target_column_1: 2 # The "_" column - - type: add_column_concatenate - target_column_0: 3 - target_column_1: 1 # Result + second original column -``` - ---- - -#### add_column_metadata - -**Purpose:** Extract metadata from source objects - -**Parameters:** -- `value` (enum): Metadata type - - `identifier0`, `identifier1`, `identifier2`, ... - - `index0`, `index1`, `index2`, ... - - `tags` - -**Identifier extraction:** -```yaml -rules: - - type: add_column_metadata - value: identifier0 # Outermost identifier -``` -``` -Input: [["moo"], ["meow"], ["bark"]] -Sources: [{"identifiers": ["cow"]}, {"identifiers": ["cat"]}, {"identifiers": ["dog"]}] -Output: [["moo", "cow"], ["meow", "cat"], ["bark", "dog"]] -``` - -**Multiple levels:** -```yaml -rules: - - type: add_column_metadata - value: identifier0 # Outer identifier - - type: add_column_metadata - value: identifier1 # Inner identifier -``` -``` -Sources: [ - {"identifiers": ["sample1", "forward"]}, - {"identifiers": ["sample1", "reverse"]} -] -Output: [["data", "sample1", "forward"], ["data", "sample1", "reverse"]] -``` - -**Index extraction:** -```yaml -rules: - - type: add_column_metadata - value: index0 - - type: add_column_metadata - value: index1 -``` -``` -Sources: [ - {"indices": [0, 0]}, # First sample, forward - {"indices": [0, 1]}, # First sample, reverse - {"indices": [1, 0]}, # Second sample, forward - {"indices": [1, 1]} # Second sample, reverse -] -Output: [ - ["samp1for", "0", "0"], - ["samp1rev", "0", "1"], - ["samp2for", "1", "0"], - ["samp2rev", "1", "1"] -] -``` - -**Tags extraction:** -```yaml -rules: - - type: add_column_metadata - value: tags -``` -``` -Sources: [ - {"identifiers": ["cow"], "tags": ["farm"]}, - {"identifiers": ["dog"], "tags": ["house", "firestation"]} -] -Output: [["moo", "farm"], ["bark", "firestation,house"]] # Sorted, comma-joined -``` - -**Use cases:** -- Access collection structure metadata -- Build identifiers from nested collections -- Use positional indices for numerical IDs -- Extract tags for grouping/filtering - ---- - -#### add_column_group_tag_value - -**Purpose:** Extract specific group tag value - -**Parameters:** -- `value` (string): Group tag name (e.g., "condition", "type") -- `default_value` (string): Value if tag not present - -**Example:** -```yaml -rules: - - type: add_column_group_tag_value - value: condition - default_value: 'control' -``` -``` -Sources: [ - {"tags": ["group:condition:treated"]}, - {"tags": ["group:condition:control"]}, - {"tags": []} # No condition tag -] -Output: [["data", "treated"], ["data", "control"], ["data", "control"]] -``` - -**Multiple tags - first alphabetically wins:** -```yaml -rules: - - type: add_column_group_tag_value - value: where - default_value: 'barn' -``` -``` -Sources: [ - {"tags": ["group:where:house", "group:where:firestation"]} -] -Output: [["data", "firestation"]] # "firestation" < "house" alphabetically -``` - -**Use cases:** -- Group samples by experimental condition -- Extract sample type (single-end/paired-end) -- Use tags for nested collection organization - ---- - -#### add_column_from_sample_sheet_index - -**Purpose:** Retrieve values from sample sheet columns - -**Parameters:** -- `value` (int): Sample sheet column index - -**Example:** -```yaml -rules: - - type: add_column_from_sample_sheet_index - value: 0 - - type: add_column_from_sample_sheet_index - value: 1 -``` -``` -Sources: [ - {"columns": [0, 1]}, - {"columns": [2, 3]} -] -Output: [["moo", 0, 1], ["cow", 2, 3]] -``` - -**Use cases:** -- Extract metadata from uploaded sample sheets -- Access additional columns beyond identifiers -- Incorporate external metadata - ---- - -### 2. Filter Rules - -Filters **remove rows** from the data table based on conditions. - -#### add_filter_regex - -**Purpose:** Keep/remove rows matching pattern - -**Parameters:** -- `target_column` (int): Column to test -- `expression` (string): Regular expression -- `invert` (bool, default: false): If true, keep non-matching rows - -**Keep matching:** -```yaml -rules: - - type: add_filter_regex - target_column: 0 - expression: '(a+)' - invert: false -``` -``` -Input: [["a", "b", "c"], ["e", "f", "g"]] -Output: [["a", "b", "c"]] -``` - -**Remove matching:** -```yaml -rules: - - type: add_filter_regex - target_column: 2 - expression: '(c+)' - invert: true -``` -``` -Input: [["a", "b", "c"], ["e", "f", "g"]] -Output: [["e", "f", "g"]] -``` - -**Use cases:** -- Filter by sample name pattern -- Remove control samples -- Select specific file types - ---- - -#### add_filter_count - -**Purpose:** Keep/remove first or last N rows - -**Parameters:** -- `count` (int): Number of rows -- `which` (enum): `first` or `last` -- `invert` (bool, default: false): If true, reverse filter - -**Remove first row:** -```yaml -rules: - - type: add_filter_count - count: 1 - which: first - invert: false # Remove first, keep rest -``` -``` -Input: [["a", "b", "c"], ["e", "f", "g"], ["h", "i", "j"]] -Output: [["e", "f", "g"], ["h", "i", "j"]] -``` - -**Keep only last row:** -```yaml -rules: - - type: add_filter_count - count: 1 - which: last - invert: true # Remove all but last -``` -``` -Input: [["a", "b", "c"], ["e", "f", "g"], ["h", "i", "j"]] -Output: [["h", "i", "j"]] -``` - -**Use cases:** -- Remove header rows -- Skip first N samples -- Select specific replicates - ---- - -#### add_filter_empty - -**Purpose:** Remove rows with empty cells - -**Parameters:** -- `target_column` (int): Column to check -- `invert` (bool, default: false): If true, keep only empty - -**Remove empty:** -```yaml -rules: - - type: add_filter_empty - target_column: 0 - invert: false -``` -``` -Input: [["", "b", "c"], ["a", "b", "c"]] -Output: [["a", "b", "c"]] -``` - -**Use cases:** -- Remove rows with missing identifiers -- Clean up sparse data -- Filter failed extractions - ---- - -#### add_filter_matches - -**Purpose:** Exact value matching (case-sensitive) - -**Parameters:** -- `value` (string): Exact value to match -- `target_column` (int): Column to check -- `invert` (bool, default: false): If true, keep non-matching - -**Example:** -```yaml -rules: - - type: add_filter_matches - value: "a" - target_column: 0 - invert: false -``` -``` -Input: [["a", "b", "c"], ["e", "f", "g"], ["h", "i", "j"]] -Output: [["a", "b", "c"]] -``` - -**Important:** Exact match only, no partial matches: -```yaml -rules: - - type: add_filter_matches - value: "a" - target_column: 1 -``` -``` -Input: [["a ", "b", "c"]] # Note space after "a" -Output: [] # No match - "a " != "a" -``` - -**Use cases:** -- Filter by specific sample ID -- Select exact condition matches -- Boolean filtering (match "true"/"false") - ---- - -#### add_filter_compare - -**Purpose:** Numeric comparisons - -**Parameters:** -- `target_column` (int): Column with numeric values -- `value` (number): Comparison value -- `compare_type` (enum): - - `less_than` - - `less_than_equal` - - `greater_than` - - `greater_than_equal` - -**Example:** -```yaml -rules: - - type: add_filter_compare - target_column: 0 - value: 13 - compare_type: less_than -``` -``` -Input: [["1", "moo"], ["10", "cow"], ["13", "rat"], ["20", "dog"]] -Output: [["1", "moo"], ["10", "cow"]] -``` - -**Use cases:** -- Filter by quality scores -- Select samples by replicate number -- Threshold-based filtering - ---- - -### 3. Structural Rules - -#### remove_columns - -**Purpose:** Delete specified columns - -**Parameters:** -- `target_columns` (list[int]): Column indices to remove - -**Example:** -```yaml -rules: - - type: remove_columns - target_columns: [0, 1] -``` -``` -Input: [["a", "b", "c"], ["e", "f", "g"]] -Output: [["c"], ["g"]] -``` - -**Use cases:** -- Clean up intermediate columns -- Remove temporary concatenation columns -- Keep only final identifier columns - ---- - -#### sort - -**Purpose:** Sort rows by column value - -**Parameters:** -- `target_column` (int): Column to sort by -- `numeric` (bool): If true, numeric sort; if false, alphabetic - -**Alphabetic sort:** -```yaml -rules: - - type: sort - numeric: false - target_column: 0 -``` -``` -Input: [["moo", "cow"], ["meow", "cat"], ["bark", "dog"]] -Output: [["bark", "dog"], ["meow", "cat"], ["moo", "cow"]] -``` - -**Note:** Case-sensitive, uppercase sorts before lowercase -``` -Input: [["Dog"], ["cat"], ["cow"]] -Output: [["Dog"], ["cat"], ["cow"]] # "Dog" < "cat" < "cow" -``` - -**Use cases:** -- Alphabetize samples -- Order by numerical IDs -- Group similar identifiers together - ---- - -#### swap_columns - -**Purpose:** Exchange two column positions - -**Parameters:** -- `target_column_0` (int): First column -- `target_column_1` (int): Second column - -**Example:** -```yaml -rules: - - type: swap_columns - target_column_0: 0 - target_column_1: 1 -``` -``` -Input: [["moo", "cow"], ["meow", "cat"]] -Output: [["cow", "moo"], ["cat", "meow"]] -``` - -**Use cases:** -- Reorder identifier columns for mapping -- Fix column order mistakes -- Prepare for specific mapping requirements - ---- - -#### split_columns - -**Purpose:** Create Cartesian product of column groups (split rows) - -**Parameters:** -- `target_columns_0` (list[int]): First column group -- `target_columns_1` (list[int]): Second column group - -**Example:** -```yaml -rules: - - type: split_columns - target_columns_0: [0] - target_columns_1: [1] -``` -``` -Input: [["moo", "cow", "A"], ["meow", "cat", "B"]] -Output: [ - ["moo", "A"], - ["cow", "A"], - ["meow", "B"], - ["cat", "B"] -] -``` - -**How it works:** -- For each row, creates N×M new rows where: - - N = number of columns in group 0 - - M = number of columns in group 1 -- Each new row contains one value from group 0 + one value from group 1 + all other columns - -**Use cases:** -- Split paired-end data into forward/reverse -- Expand multiple samples per row -- Create all combinations for comparisons - ---- - -## Mapping Operations - -Mapping operations define how transformed data columns become collection structure. These are the **final step** that converts tabular data back to collections. - -### Available Mapping Types - -#### list_identifiers - -**Purpose:** Create list structure with specified nesting levels - -**Parameters:** -- `columns` (list[int]): Column indices for identifiers - -**Single column = simple list:** -```yaml -mapping: - - type: list_identifiers - columns: [0] -``` -``` -Data: [["sample1"], ["sample2"]] -Result: list [sample1, sample2] -``` - -**Two columns = nested list:list:** -```yaml -mapping: - - type: list_identifiers - columns: [0, 1] -``` -``` -Data: [["group1", "s1"], ["group1", "s2"], ["group2", "s3"]] -Result: list:list [ - group1 → [s1, s2], - group2 → [s3] -] -``` - -**Three columns = list:list:list:** -```yaml -mapping: - - type: list_identifiers - columns: [0, 1, 2] -``` - -**Nesting logic:** -- Column 0 = outermost identifier -- Column 1 = next level identifier -- Column 2 = innermost identifier -- Groups rows by matching outer identifiers - ---- - -#### paired_identifier - -**Purpose:** Add paired collection level - -**Parameters:** -- `columns` (list[int]): Single column with paired identifier - -**Valid identifier values:** -- `forward`, `f`, `1`, `R1` → becomes `forward` -- `reverse`, `r`, `2`, `R2` → becomes `reverse` - -**Simple paired:** -```yaml -mapping: - - type: paired_identifier - columns: [0] -``` -``` -Data: [["forward"], ["reverse"]] -Result: paired {forward, reverse} -``` - -**Combined with list:** -```yaml -mapping: - - type: list_identifiers - columns: [0] - - type: paired_identifier - columns: [1] -``` -``` -Data: [ - ["sample1", "forward"], - ["sample1", "reverse"], - ["sample2", "forward"], - ["sample2", "reverse"] -] -Result: list:paired [ - sample1 → {forward, reverse}, - sample2 → {forward, reverse} -] -``` - ---- - -#### paired_or_unpaired_identifier - -**Purpose:** Add paired_or_unpaired collection level (allows unpaired single datasets) - -**Parameters:** -- `columns` (list[int]): Single column with paired/unpaired identifier - -**Valid identifier values:** -- All paired values above, plus: -- `unpaired`, `u` → becomes `unpaired` - -**Example:** -```yaml -mapping: - - type: list_identifiers - columns: [0] - - type: paired_or_unpaired_identifier - columns: [1] -``` - -**Note:** If a sample has only `forward` and no `reverse`, it becomes `unpaired` automatically. - ---- - -#### tags - -**Purpose:** Apply tags to collection elements - -**Parameters:** -- `columns` (list[int]): Columns containing tag values - -**Example:** -```yaml -mapping: - - type: list_identifiers - columns: [0] - - type: tags - columns: [1] -``` -``` -Data: [["sample1", "replicate1"], ["sample2", "replicate2"]] -Result: list with tags [ - sample1 (tags: ["replicate1"]), - sample2 (tags: ["replicate2"]) -] -``` - ---- - -#### group_tags - -**Purpose:** Apply group tags (format: `group:name:value`) - -**Parameters:** -- `columns` (list[int]): Columns containing group tag values - -**Example:** -```yaml -mapping: - - type: list_identifiers - columns: [1, 0] # Group by column 1, element ID column 0 - - type: group_tags - columns: [1] # Apply as group tag -``` -``` -Data: [["s1", "treated"], ["s2", "control"]] -Result: list:list with group tags [ - treated → [s1 (tags: ["group:treated"])], - control → [s2 (tags: ["group:control"])] -] -``` - ---- - -## Collection Type Determination - -The output collection type is determined solely by the mapping: - -```python -# From RuleSet.collection_type property: -list_columns = mapping_as_dict.get("list_identifiers", {"columns": []})["columns"] -collection_type = ":".join("list" for c in list_columns) -if "paired_identifier" in mapping_as_dict: - collection_type += ":paired" if collection_type else "paired" -if "paired_or_unpaired_identifier" in mapping_as_dict: - collection_type += ":paired_or_unpaired" if collection_type else "paired_or_unpaired" -``` - -**Examples:** -- `list_identifiers: [0]` → `list` -- `list_identifiers: [0, 1]` → `list:list` -- `list_identifiers: [0]` + `paired_identifier: [1]` → `list:paired` -- `list_identifiers: [0, 1]` + `paired_identifier: [2]` → `list:list:paired` - ---- - -## Complete Example: list:record to list:paired - -This example demonstrates complex transformation combining multiple rule types: - -**Goal:** Convert `list:record` collection where records have "mother" and "child" elements into `list:paired` with "forward" and "reverse". - -```yaml -rules: - - type: add_column_metadata - value: identifier0 # Sample identifier - - type: add_column_metadata - value: identifier1 # Record type (mother/father/child) - - type: add_column_regex - target_column: 2 - expression: 'mother' - replacement: 'forward' - allow_unmatched: true # Leaves others as "" - - type: add_column_regex - target_column: 2 - expression: 'child' - replacement: 'reverse' - allow_unmatched: true - - type: add_column_concatenate - target_column_0: 3 # Result of first regex - target_column_1: 4 # Result of second regex - - type: add_filter_empty - target_column: 5 # Remove rows that didn't match (father) - invert: false - - type: remove_columns - target_columns: [2, 3, 4] # Clean up intermediate columns - -mapping: - - type: list_identifiers - columns: [1, 2] # Sample ID, then forward/reverse -``` - -**Transformation steps:** - -``` -Initial: - data: [["el1"], ["el2"], ["el3"]] - sources: [ - {"identifiers": ["samp1", "mother"]}, - {"identifiers": ["samp1", "father"]}, - {"identifiers": ["samp1", "child"]} - ] - -After add_column_metadata (identifier0, identifier1): - [["el1", "samp1", "mother"], - ["el2", "samp1", "father"], - ["el3", "samp1", "child"]] - -After first regex (mother → forward): - [["el1", "samp1", "mother", "forward"], - ["el2", "samp1", "father", ""], - ["el3", "samp1", "child", ""]] - -After second regex (child → reverse): - [["el1", "samp1", "mother", "forward", ""], - ["el2", "samp1", "father", "", ""], - ["el3", "samp1", "child", "", "reverse"]] - -After concatenate (cols 3+4): - [["el1", "samp1", "mother", "forward", "", "forward"], - ["el2", "samp1", "father", "", "", ""], - ["el3", "samp1", "child", "", "reverse", "reverse"]] - -After filter empty (col 5): - [["el1", "samp1", "mother", "forward", "", "forward"], - ["el3", "samp1", "child", "", "reverse", "reverse"]] - -After remove_columns [2, 3, 4]: - [["el1", "samp1", "forward"], - ["el3", "samp1", "reverse"]] - -Final mapping with list_identifiers [1, 2]: - Result: list:paired [ - samp1 → {forward, reverse} - ] -``` - ---- - -## Rule Composition Patterns - -### Pattern 1: Extract and Flatten - -**Goal:** Flatten `list:paired` → `list` with combined identifiers - -```yaml -rules: - - type: add_column_metadata - value: identifier0 # Outer ID - - type: add_column_metadata - value: identifier1 # Pair ID (forward/reverse) - - type: add_column_concatenate - target_column_0: 1 - target_column_1: 2 # Combine them - -mapping: - - type: list_identifiers - columns: [3] # Use concatenated column -``` - ---- - -### Pattern 2: Group by Tag - -**Goal:** Reorganize by tag value into nested structure - -```yaml -rules: - - type: add_column_metadata - value: identifier0 - - type: add_column_group_tag_value - value: condition # Extract "condition" tag - default_value: "unassigned" - -mapping: - - type: list_identifiers - columns: [1, 0] # Group by condition, then sample ID - - type: group_tags - columns: [1] # Apply as group tags -``` - ---- - -### Pattern 3: Filter and Sort - -**Goal:** Select subset and alphabetize - -```yaml -rules: - - type: add_column_metadata - value: identifier0 - - type: add_filter_regex - target_column: 0 - expression: '^control_' # Only controls - invert: false - - type: sort - numeric: false - target_column: 0 - -mapping: - - type: list_identifiers - columns: [0] -``` - ---- - -### Pattern 4: Parse Filename Structure - -**Goal:** Extract sample info from "sample_123_R1.fastq.gz" format - -```yaml -rules: - - type: add_column_metadata - value: identifier0 # Original filename - - type: add_column_regex - target_column: 0 - expression: 'sample_(\w+)_R(\d)' - group_count: 2 # Sample ID and read number - - type: add_column_value - value: "_R" - - type: add_column_concatenate - target_column_0: 3 - target_column_1: 2 # "_R" + "1" = "_R1" - - type: add_column_concatenate - target_column_0: 1 - target_column_1: 4 # "123" + "_R1" = "123_R1" - - type: remove_columns - target_columns: [0, 2, 3, 4] # Keep only final identifier - -mapping: - - type: list_identifiers - columns: [0] -``` - ---- - -### Pattern 5: Create Paired from Separate Lists - -**Goal:** Combine separate forward/reverse lists into paired - -**Assumption:** Files named like `sample1_R1.fastq`, `sample1_R2.fastq` - -```yaml -rules: - - type: add_column_metadata - value: identifier0 - - type: add_column_regex - target_column: 0 - expression: '(.+)_R([12])' - group_count: 2 # Sample name and read number - - type: add_column_regex - target_column: 2 - expression: '1' - replacement: 'forward' - allow_unmatched: true - - type: add_column_regex - target_column: 2 - expression: '2' - replacement: 'reverse' - allow_unmatched: true - - type: add_column_concatenate - target_column_0: 3 - target_column_1: 4 - - type: sort - numeric: false - target_column: 1 # Ensure pairs adjacent - - type: remove_columns - target_columns: [0, 2, 3, 4] - -mapping: - - type: list_identifiers - columns: [0] # Sample ID - - type: paired_identifier - columns: [1] # forward/reverse -``` - ---- - -## Best Practices - -### 1. Plan Column Layout - -Before writing rules, sketch the transformations: -``` -Col 0: Original identifier -Col 1: Extracted sample ID (regex) -Col 2: Extracted replicate (regex) -Col 3: Separator "_" -Col 4: Concatenate 1+3+2 -Col 5: Final identifier after cleanup -``` - -### 2. Test Incrementally - -Add rules one at a time and verify output: -- Start with metadata extraction -- Add one transformation -- Check result -- Continue - -### 3. Use allow_unmatched Carefully - -Only use when genuinely optional: -```yaml -# BAD - silently fails to extract -- type: add_column_regex - expression: 'wrong_pattern' - allow_unmatched: true - -# GOOD - errors if pattern doesn't match -- type: add_column_regex - expression: 'expected_pattern' - allow_unmatched: false -``` - -### 4. Remove Intermediate Columns - -Clean up before mapping: -```yaml -rules: - - type: add_column_metadata - value: identifier0 - # ... many transformations ... - - type: remove_columns - target_columns: [0, 2, 3] # Remove temp columns - -mapping: - - type: list_identifiers - columns: [0] # Only final column remains -``` - -### 5. Validate with Filters - -Use filters to ensure data quality: -```yaml -rules: - - type: add_column_regex - expression: 'pattern' - allow_unmatched: false # Errors if doesn't match - - type: add_filter_empty - target_column: 1 - invert: false # Remove any that became empty -``` - -### 6. Document Complex Rules - -Add comments explaining logic: -```yaml -rules: - # Extract sample ID from filename "sample_123_R1.fastq" - - type: add_column_regex - target_column: 0 - expression: 'sample_(\w+)_R\d' - - # Remove original filename column - - type: remove_columns - target_columns: [0] -``` - ---- - -## Common Pitfalls - -### Pitfall 1: Column Indices Shift - -**Problem:** After removing columns, indices change - -```yaml -# WRONG -rules: - - type: remove_columns - target_columns: [0] - - type: add_column_regex - target_column: 1 # This is now wrong! Column 1 became 0 -``` - -**Solution:** Remove columns last, or recalculate indices - -### Pitfall 2: Forgetting Invert Logic - -**Problem:** Confusion about filter invert - -```yaml -# Remove matching rows (keep non-matching) -- type: add_filter_regex - expression: 'control_' - invert: false # FALSE means "remove matching" - -# Keep matching rows -- type: add_filter_regex - expression: 'sample_' - invert: true # TRUE means "remove non-matching" = keep matching -``` - -**Clearer thinking:** `invert: false` = "remove matches", `invert: true` = "remove non-matches" - -### Pitfall 3: Regex Escaping - -**Problem:** Special regex characters not escaped - -```yaml -# WRONG - . matches any character -expression: 'file.fastq' - -# RIGHT -expression: 'file\.fastq' - -# For literal parentheses -expression: '\(sample\)' -``` - -### Pitfall 4: Case Sensitivity - -**Problem:** Filters are case-sensitive +This step swaps the axes of a `list:list` input: ```yaml -# Doesn't match "Sample1" -- type: add_filter_matches - value: "sample1" - target_column: 0 +tool_id: __APPLY_RULES__ +in: + input: sample_by_segment +tool_state: + input: { __class__: ConnectedValue } + rules: + rules: + - type: add_column_metadata + value: identifier0 + - type: add_column_metadata + value: identifier1 + mapping: + - type: list_identifiers + columns: [1, 0] ``` -**Solution:** Use regex with case-insensitive flag or normalize case first - -### Pitfall 5: Empty Sources After Filtering - -**Problem:** All rows filtered out - -```yaml -rules: - - type: add_filter_regex - expression: 'nonexistent' - invert: false -# Result: Empty collection! -``` - -**Solution:** Test filters carefully, use `allow_unmatched: true` when appropriate - ---- - -## When to Use / When NOT to Use Apply Rules - -### When to Use Apply Rules - -- Complex identifier parsing (multiple regex extractions) -- Tag-based restructuring (group by experimental condition) -- Conditional filtering combined with restructuring -- Structure transformations not covered by simple tools -- Multiple transformations needed in one step - -### When NOT to Use Apply Rules - -| Operation | Use This Instead | Why | -|-----------|------------------|-----| -| Simple filtering | `__FILTER_FROM_FILE__` | Simpler, clearer intent | -| Basic flattening | `__FLATTEN__` | One-step operation | -| Sort collection | `__SORTLIST__` | Dedicated tool | -| Extract element | `__EXTRACT_DATASET__` | Direct operation | -| Zip two lists | `__ZIP_COLLECTION__` | Simpler syntax | -| Unzip paired | `__UNZIP_COLLECTION__` | Straightforward | -| Relabel identifiers | `__RELABEL_FROM_FILE__` | If mapping from file | - -### Comparison Table - -| Operation | Simple Tool | When to use Apply Rules instead | -|-----------|-------------|--------------------------------| -| Filter | Filter Collection | Need to filter on derived metadata, combine with restructuring | -| Flatten | Flatten Collection | Need control over identifier format, filter simultaneously | -| Relabel | Relabel Identifiers | Need regex-based transformation, derive from existing metadata | -| Sort | Sort Collection | Need to sort by derived values, combine with other operations | -| Restructure | N/A | Full control over nesting structure from any metadata | +The first rule creates column 0 from the outer identifier. The second creates column 1 from the inner identifier. Mapping `[1, 0]` groups by segment, then sample. The executable workflow and test live under `verification/workflows/collection-swap-nesting-with-apply-rules/`. See [[collection-swap-nesting-with-apply-rules]] for the expected collection. -**Key Insight:** Apply Rules is the tool of choice when: -- Multiple transformations needed in one step -- Restructuring based on metadata (tags, identifier patterns) -- Complex identifier manipulation required -- Standard tools don't cover the use case +## Rows, columns, and sources ---- +For collection input, Galaxy starts with one **empty row per leaf dataset**. Source metadata accompanies each row: its identifier path, index path, dataset reference, dataset tags, and any inherited sample-sheet columns. `identifier0` means the outermost identifier. `index0` is its zero-based position. -## Use Case Examples +Column indices are zero-based and refer to the table at that point in execution. Addition rules append columns. Removing columns shifts subsequent indices. Filters and sorting carry source metadata along with the rows, so a renamed row still points to its original dataset. -### Use Case 1: Standard Paired-End RNA-seq +Mapping uses the final table. Unmapped columns can remain, so removing intermediate columns is optional. Plan the output identifier path before writing rules, then track the columns that supply it. -**Files:** `sample1_R1.fastq.gz`, `sample1_R2.fastq.gz`, `sample2_R1.fastq.gz`, `sample2_R2.fastq.gz` +## Rule operations -**Goal:** Create `list:paired` collection +Each rule requires `type` and the parameters listed below, except fields explicitly marked optional. Rules run in order. Names and behavior here follow Galaxy's [implementation](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/util/rules_dsl.py) and [rule specification](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/util/rules_dsl_spec.yml). -```yaml -rules: - - type: add_column_metadata - value: identifier0 - - type: add_column_regex - target_column: 0 - expression: '(.+)_R([12])\.fastq\.gz' - group_count: 2 - - type: add_column_regex - target_column: 2 - expression: '1' - replacement: 'forward' - allow_unmatched: true - - type: add_column_regex - target_column: 2 - expression: '2' - replacement: 'reverse' - allow_unmatched: true - - type: add_column_concatenate - target_column_0: 3 - target_column_1: 4 - - type: sort - target_column: 1 - numeric: false - - type: remove_columns - target_columns: [0, 2, 3, 4] +### Add columns -mapping: - - type: list_identifiers - columns: [0] - - type: paired_identifier - columns: [1] -``` +| Rule | Parameters and result | +|---|---| +| `add_column_metadata` | String `value`: `identifierN`, `indexN`, or `tags`. Appends the identifier, stringified index, or sorted comma-joined dataset tags. The requested nesting level must exist. | +| `add_column_group_tag_value` | String `value` names the key in `group:key:value`. Appends its value. Optional `default_value` defaults to `""`. If multiple tags match, the first after sorting wins. | +| `add_column_from_sample_sheet_index` | Integer `value` selects a source sample-sheet column. Requires that source metadata to exist. | +| `add_column_value` | String `value` appends a constant. | +| `add_column_rownum` | Integer `start` appends successive row numbers as strings, beginning at that value. | +| `add_column_basename` | Integer `target_column` appends everything after the last `/`. | +| `add_column_concatenate` | Integer `target_column_0` and `target_column_1` append their concatenation, without a separator. | +| `add_column_substr` | Integer `target_column`, integer `length`, and `substr_type`: `keep_prefix`, `keep_suffix`, `drop_prefix`, or `drop_suffix`. Appends the resulting substring. | +| `add_column_regex` | Integer `target_column` and string `expression`. Optional `replacement`, `group_count`, and boolean `allow_unmatched`. Behavior below. | ---- +Regex extraction uses Python regular expressions and searches within the cell. Without `replacement` or `group_count`, it appends the **whole match**, not the first capture group. With `group_count`, it appends one column per capture group and requires exactly that many groups. With `replacement`, it appends the expanded template for the match, such as `'\1'`, not a rewritten copy of the whole cell. -### Use Case 2: Remove Control Samples +Unmatched rows fail by default. `allow_unmatched: true` appends empty strings instead. Anchor expressions when the entire identifier must conform, and use YAML single quotes to preserve regex backslashes. [[collection-split-identifier-via-rules]] shows a nesting example. -**Goal:** Filter out samples starting with "control_" +### Filter rows -```yaml -rules: - - type: add_column_metadata - value: identifier0 - - type: add_filter_regex - target_column: 0 - expression: '^control_' - invert: true # Remove matches = keep non-controls +`invert` is a required boolean for the first four filters. Its meaning depends on the operation: -mapping: - - type: list_identifiers - columns: [0] -``` +| Rule | Required parameters | `invert: false` | `invert: true` | +|---|---|---|---| +| `add_filter_regex` | Integer `target_column`, string `expression`, boolean `invert` | Keep regex matches | Keep nonmatches | +| `add_filter_matches` | Integer `target_column`, string `value`, boolean `invert` | Keep exact matches | Keep nonmatches | +| `add_filter_empty` | Integer `target_column`, boolean `invert` | Keep nonempty cells | Keep empty cells | +| `add_filter_count` | Integer `count`, `which: first` or `last`, boolean `invert` | Drop the selected first/last rows | Keep only those rows | ---- +Exact matching is case-sensitive and includes whitespace. Regex filters also search within cells. -### Use Case 3: Group by Treatment Condition +`add_filter_compare` requires integer `target_column`, integer `value`, and `compare_type`: `less_than`, `less_than_equal`, `greater_than`, or `greater_than_equal`. It parses cells as floating-point numbers and keeps rows satisfying the comparison. It has no inversion parameter. -**Goal:** Reorganize by "group:condition:*" tag into nested list +### Reorder or reshape the table -```yaml -rules: - - type: add_column_metadata - value: identifier0 - - type: add_column_group_tag_value - value: condition - default_value: 'unassigned' +| Rule | Parameters and result | +|---|---| +| `sort` | Integer `target_column` and boolean `numeric`. Sorts ascending, numerically if true, otherwise by case-sensitive string order. | +| `remove_columns` | `target_columns`, an array of column indices. Removes those columns. | +| `swap_columns` | Integer `target_column_0` and `target_column_1`. Exchanges their positions. | +| `split_columns` | Arrays `target_columns_0` and `target_columns_1`. Produces two rows per input row, each retaining one selected group and all unselected columns. Both rows retain the same dataset source. | -mapping: - - type: list_identifiers - columns: [1, 0] # Group by condition, then sample - - type: group_tags - columns: [1] -``` +For example, splitting groups `[0]` and `[1]` transforms `["left", "right", "sample"]` into `["left", "sample"]` and `["right", "sample"]`. It does not generate a Cartesian product. ---- +## Map columns into a collection -### Use Case 4: Select Top N by Quality Score +| Mapping type | `columns` and effect | +|---|---| +| `list_identifiers` | One or more columns in outer-to-inner order. `[0]` produces `list`, `[0, 1]` produces `list:list`. | +| `paired_identifier` | One column containing read direction. Adds an innermost `paired` level. | +| `paired_or_unpaired_identifier` | One column containing read direction or `unpaired`. Adds an innermost `paired_or_unpaired` level. | +| `tags` | Columns whose values become dataset tags. Each cell supplies one tag. | +| `group_tags` | Columns whose values become dataset tags prefixed with `group:`. To produce `group:condition:treated`, supply `condition:treated`. | -**Assumption:** Quality score in sample name like "sample_123_q95" +A second list column creates another list level even if its values are `forward` and `reverse`. To create `list:paired`, use both `list_identifiers` and `paired_identifier`. Pair roles are case-insensitive: `f`, `1`, `r1`, and `forward` normalize to `forward`, while `r`, `2`, `r2`, and `reverse` normalize to `reverse`. For mixed paired and single reads, supply `u` or `unpaired` explicitly for single reads. -**Goal:** Keep only samples with quality >= 90 +For flat identifiers `sampleA_R1.fastq.gz` and `sampleA_R2.fastq.gz`, this parameter derives the sample and pair roles directly: ```yaml rules: - - type: add_column_metadata - value: identifier0 - - type: add_column_regex - target_column: 0 - expression: 'sample_\w+_q(\d+)' - - type: add_filter_compare - target_column: 1 - value: 90 - compare_type: greater_than_equal - - type: remove_columns - target_columns: [1] - -mapping: - - type: list_identifiers - columns: [0] + rules: + - type: add_column_metadata + value: identifier0 + - type: add_column_regex + target_column: 0 + expression: '^(.+)_R([12])\.fastq\.gz$' + group_count: 2 + mapping: + - type: list_identifiers + columns: [1] + - type: paired_identifier + columns: [2] ``` ---- - -### Use Case 5: Replicate Structure +Column 1 is `sampleA`, column 2 is `1` or `2`, and the output type is `list:paired`. Each sample must have the intended forward and reverse datasets. [[collection-build-list-paired-with-apply-rules]] describes paired promotion from existing nesting. -**Files:** `treatment_rep1`, `treatment_rep2`, `control_rep1`, `control_rep2` +## Check the result -**Goal:** Create `list:list` [treatment → [rep1, rep2], control → [rep1, rep2]] +Build a complex rule set incrementally. After each addition, inspect the table and retained identifiers before adding the next rule. Keep a column map in comments so later edits can update every affected index. -```yaml -rules: - - type: add_column_metadata - value: identifier0 - - type: add_column_regex - target_column: 0 - expression: '(.+)_rep(\d+)' - group_count: 2 - - type: sort - target_column: 1 - numeric: false - - type: remove_columns - target_columns: [0] - -mapping: - - type: list_identifiers - columns: [0, 1] # Condition, then replicate -``` +Test identifiers, collection type, and dataset membership together. A correct element count can hide swapped samples or incorrect pairing. Check representative contents as well as structure. ---- - -## API Usage +Watch for these failures: -```python -inputs = { - "input": {"src": "hdca", "id": collection_id}, - "rules": { - "rules": [...], - "mapping": [...] - } -} -response = POST /api/tools {"tool_id": "__APPLY_RULES__", "history_id": "...", "inputs": inputs} -``` +- **Duplicate output paths:** two rows mapping to the same full identifier path cause the later dataset to replace the earlier one in the builder. Make the final path unique for every intended dataset. +- **Unexpected filtering:** inspect retained identifiers, especially around `invert` and count filters. Filtering is selection, not validation that every input conforms. +- **All rows removed:** check that filters retain the expected datasets, including cases with no matches. If no row survives, there is no dataset left to map. `allow_unmatched` only affects regex extraction and cannot prevent a filter from removing rows. +- **Regex escaping:** `.` matches any character. To match a literal filename extension, write a single-quoted YAML expression such as '\.fastq$'. To match literal parentheses, use '\(sample\)'. Avoid double-quoted YAML unless you also account for its backslash escaping. +- **Case differences:** exact-match filters distinguish `sample1` from `Sample1`. If case should be ignored, use a regex such as '(?i)^sample1$' with `add_filter_regex`. +- **Missing metadata or bad indices:** check the input nesting and recalculate column numbers after removal or splitting. Remove temporary columns near the end when that makes index tracking easier. +- **Failed extraction or numeric conversion:** fix the identifier contract or select the intended rows before extraction. Use `allow_unmatched` only when an empty result has a defined role. ---- +The [collection builder](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/collections.py#L823-L960) defines grouping and tag assignment. [[galaxy-collection-semantics]] covers downstream mapping and reduction. For a relabeling task, see [[relabel-via-rules-and-find-replace]]. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-collection-tools.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-collection-tools.md index 45e8362..414c205 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-collection-tools.md +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-collection-tools.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-02 -revision: 2 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-collection-semantics]]" - "[[galaxy-apply-rules-dsl]]" @@ -14,424 +14,253 @@ related_notes: - "[[nextflow-operators-to-galaxy-collection-recipes]]" - "[[iwc-transformations-survey]]" sources: - - "https://github.com/jmchilton/galaxy-agentic-collection-transform (initial research seed)" - - "https://github.com/galaxyproject/galaxy/tree/main/lib/galaxy/tools (XML wrappers; source of truth)" -summary: "Catalog of Galaxy's collection-operation tools — purpose, IO, parameters, selection guide. Companion to galaxy-collection-semantics." + - "https://github.com/galaxyproject/galaxy/tree/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tools" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tools/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tools/model_operation_macros.xml" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tools/actions/model_operations.py" +summary: "Catalog of Galaxy collection-operation tools, with versioned inputs, outputs, defaults, mapping behavior, and diagnostic qualifications." --- -Catalog of Galaxy's built-in collection-operation tools (the `__BUILD_LIST__`, `__FILTER_FROM_FILE__`, `__APPLY_RULES__`, … family) — what each tool does, its inputs and outputs, and when to reach for it. Source of truth is the Galaxy XML wrappers under `lib/galaxy/tools/`; this is the high-level catalog. Pairs with [[galaxy-collection-semantics]], which describes the underlying mapping and reduction semantics rather than the user-facing tools. +Galaxy's built-in collection-operation tools assemble, select, reshape, and annotate collections. Their outputs usually copy history dataset associations while sharing the underlying dataset files. They do not rerun scientific analysis or duplicate those files. Some operations read file contents, including identifier tables, tag tables, null markers, and the emptiness check. Creating a large cross product still creates many history objects. -These are **model operations** — they manipulate collection structure without processing file contents, so they're fast and don't grow storage. +The catalog describes the wrappers and implementations at Galaxy commit `a63da1dfd1960360f4aa2fddc6a75396954d750a`. Versions below are the newest wrappers present at that commit, not a promise that every Galaxy server exposes them. Older wrappers can have different inputs. A workflow's concrete `tool_version` and parameter schema must match the installed tool, not merely the newest entry in this catalog. [[galaxy-collection-semantics]] describes collection mapping and reduction, and [[galaxy-apply-rules-dsl]] gives the exact rule vocabulary. -## Tool Categories +## Collection inputs and mapped outputs -### 1. Collection Creation Tools +An input declared as `data` takes one dataset per execution. Selecting a collection for that input invokes Galaxy's normal mapping behavior. Build List and Zip therefore produce one list or pair per mapped execution, with an outer collection collecting the results. They do not accept arbitrary collections as single dataset values or concatenate them directly. -#### Build List (`__BUILD_LIST__`) -**Version:** 1.2.0 +An input declared as `data_collection` consumes a collection of a compatible type. A deeper collection may be mapped over to supply compatible inner collections. The output types in this catalog describe one execution, except where mapped behavior is stated explicitly. Collection order and identifiers both matter when synchronizing inputs. Equal element counts alone do not establish that two datasets represent the same sample. -**Purpose:** Build a new list collection from individual datasets or collections. +## Collection creation -**Inputs:** -- `datasets` (repeat): Input datasets or collections - - `input`: Data input (optional) - - `id_cond/id_select`: Label selection method - - `idx`: Use index (0, 1, 2...) - - `identifier`: Use dataset's existing identifier - - `manual`: Specify custom identifier +### Build list (`__BUILD_LIST__`, 1.2.0) -**Output:** `list` collection +Builds `output`, a `list`, from a repeat named `datasets`. Each repeat has optional dataset input `input` and conditional `id_cond`: -**Use Cases:** -- A: Build collection from individual datasets -- B: Merge collection with individual dataset(s) → nested collection -- C: Merge collections → nested collection (requires equal element counts) +| Parameter | Values and behavior | +|---|---| +| `id_cond/id_select` | `idx` by default, `identifier`, or `manual`. | +| `idx` | Uses the repeat's zero-based position as the element identifier. Omitted optional inputs can leave gaps in the numbering. | +| `identifier` | Uses the incoming element identifier when available, otherwise the dataset name. | +| `id_cond/identifier` | Supplies the label for `manual`. | ---- - -#### Duplicate File to Collection (`__DUPLICATE_FILE_TO_COLLECTION__`) -**Version:** 1.0.0 - -**Purpose:** Create collection of arbitrary size by duplicating an input dataset N times. - -**Inputs:** -- `input`: Dataset to duplicate -- `number`: Integer - number of copies -- `element_identifier`: Base name for identifiers (e.g., "test" → "test 1", "test 2", etc.) - -**Output:** `list` collection - -**Use Case:** Create test collections, broadcasting single dataset to match collection size. - ---- - -### 2. Element Extraction Tools - -#### Extract Dataset (`__EXTRACT_DATASET__`) -**Version:** 1.0.2 - -**Purpose:** Extract a single dataset from a collection. - -**Inputs:** -- `input`: Collection (`list`, `paired`, `paired_or_unpaired`, `record`) -- `which`: Selection method - - `first`: First dataset - - `by_identifier`: Match element identifier (string) - - `by_index`: Select by 0-based index (integer) - -**Output:** Single dataset - -**Note:** For nested collections, collapses inner-most collection into dataset, creating new list at that level. - ---- - -### 3. Filter Tools +Individual datasets become elements of one list. Collection inputs to the dataset parameters are mapped, producing lists inside an outer collection. Multiple mapped inputs must be compatible for Galaxy's paired mapping. This is different from Merge Collections, which appends collection elements. -#### Filter Empty Datasets (`__FILTER_EMPTY_DATASETS__`) -**Version:** 1.1.0 +Choose unique labels. The implementation builds an identifier-keyed dictionary, so a repeated label replaces an earlier entry. Wrapper 1.1.0 has the dataset repeat but lacks the label-selection conditional. -**Purpose:** Remove empty elements from a collection. +### Duplicate file to collection (`__DUPLICATE_FILE_TO_COLLECTION__`, 1.0.0) -**Inputs:** -- `input`: Collection (`list`, `list:paired`) -- `replacement` (optional): Dataset to replace empty elements instead of removing +Takes dataset `input`, integer `number`, and text `element_identifier`. No numeric default is supplied for `number`. Produces `output`, a `list`, with labels formed from the base identifier, a space, and a one-based number. For base `test` and size 2, the identifiers are `test 1` and `test 2`. -**Output:** Filtered collection (same type) +Each element refers to a copy of the input's history association. This can materialize repeated inputs for a test or a synchronized operation, but repeated copies remain the same scientific input. -**Use Case:** Continue multi-sample analysis when downstream tools require content. +## Element extraction ---- +### Extract dataset (`__EXTRACT_DATASET__`, 1.0.2) -#### Filter Failed Datasets (`__FILTER_FAILED_DATASETS__`) -**Version:** 1.0.0 +Consumes `input` with declared types `list`, `paired`, `paired_or_unpaired`, or `record`. Produces dataset `output`, named after the selected element identifier. -**Purpose:** Remove datasets in error (red) state from a collection. +| Parameter | Values and behavior | +|---|---| +| `which/which_dataset` | `first` by default, `by_identifier`, or `by_index`. | +| `which/identifier` | Required for `by_identifier`. The wrapper's sanitizer retains ASCII letters, digits, `_`, `-`, and `#`. | +| `which/index` | Zero-based integer, default 0, for `by_index`. | -**Inputs:** -- `input`: Collection (`list`, `list:paired`) -- `replacement` (optional): Dataset to replace failed elements +An empty collection cannot supply the first dataset. Missing identifiers and invalid indices fail. For a nested input, Galaxy can map over compatible inner collections and collect the extracted datasets at the remaining outer levels. Extraction does not flatten every dataset in the hierarchy. -**Output:** Filtered collection (same type) +## Filtering -**Use Case:** Continue analysis when some samples fail. +### State, empty-content, and null filters ---- +These tools take collection `input`, declared as `list` or `list:paired`, and produce collection `output` with the input type. Their predicates differ: -#### Filter Null Elements (`__FILTER_NULL__`) -**Version:** 1.1.0 +| Tool and version | Criterion for retaining a dataset | +|---|---| +| Filter empty datasets, `__FILTER_EMPTY_DATASETS__`, 1.1.0 | Has data and yields at least one byte from Galaxy's file reader, which handles supported compression. A compressed file containing no decompressed content is empty. | +| Filter failed datasets, `__FILTER_FAILED_DATASETS__`, 1.1.0 | `is_ok` is true. The implementation keeps successful datasets, not simply every state other than red/error. | +| Filter null elements, `__FILTER_NULL__`, 1.1.0 | Is not the recognized `expression.json` null marker. This checks the datatype and `peek == "null"`, or the file content read by the implementation. It does not remove every empty file or arbitrary JSON containing null values. | +| Keep success, `__KEEP_SUCCESS_DATASETS__`, 1.1.0 | `is_ok` is true. Paused datasets can reach the filter and are excluded. Other pending datasets cause an input-not-ready result. | -**Purpose:** Remove null elements (from conditional workflow execution) from a collection. +For `list:paired`, both members must pass. If either fails the predicate, the whole pair is removed. Higher collection levels can be mapped over. -**Inputs:** -- `input`: Collection (`list`, `list:paired`) -- `replacement` (optional): Dataset to replace null elements +All four versions listed expose optional dataset `replacement`. For a simple list, supplying it replaces excluded elements while retaining their identifiers and copying their original tags. Replacement is rejected for `list:paired`. The 1.0.0 wrappers for Empty, Null, and Keep Success do not expose replacement. Filter Failed 1.0.0 does. -**Output:** Filtered collection (same type) +Filter Empty and Filter Null require successful inputs before applying their content predicate. They do not repair errored datasets. Filter Failed waits for ready inputs and does not accept paused inputs in the same way as Keep Success. Keep Success also waits on running or queued inputs, despite wrapper help that suggests still-running elements are removed immediately. -**Use Case:** Clean up collections after conditional workflow branches. +Removing elements changes the sample population and can break alignment with another collection. Preserving an identifier with a replacement preserves structural alignment, but does not establish that the replacement is a meaningful scientific result. ---- +### Filter collection (`__FILTER_FROM_FILE__`, 1.1.0) -#### Keep Success (`__KEEP_SUCCESS_DATASETS__`) -**Version:** 1.1.0 +Takes any collection `input` and a text identifier file in conditional `how`. Matching applies to the collection's top-level element identifiers, with whitespace stripped from file lines. -**Purpose:** Keep only datasets in success (green) state. +| Parameter | Contract | +|---|---| +| `how/how_filter` | `remove_if_absent` by default, or `remove_if_present`. | +| `how/filter_source` | Text dataset containing one identifier per line. | -**Inputs:** -- `input`: Collection (`list`, `list:paired`) -- `replacement` (optional): Dataset to replace unsuccessful elements +`remove_if_absent` keeps elements named in the file. `remove_if_present` excludes them. File entries absent from the collection have no effect. The tool retains input order, rather than adopting file order. -**Output:** Filtered collection (same type) +Outputs `output_filtered` and `output_discarded` partition the input and retain its collection type. Keeping the discarded output makes exclusions inspectable. The 1.0.0 wrapper exposes the same parameter names. -**Use Case:** Similar to Filter Failed but positive selection approach. +## Structure transformations ---- +### Flatten collection (`__FLATTEN__`, 1.0.0) -#### Filter Collection (`__FILTER_FROM_FILE__`) -**Version:** 1.1.0 +Takes collection `input` and produces `output`, a flat `list` of all leaf datasets. `join_identifier` is `_` by default, with `:` and `-` also accepted. Identifiers at each nesting level are joined in traversal order. -**Purpose:** Filter elements using identifier list from a file. +For `list:paired`, sample `i1` becomes `i1_forward` and `i1_reverse` with the default separator. A flat input remains a list with its existing identifiers. -**Inputs:** -- `input`: Any collection -- `how/how_filter`: Filter mode - - `remove_if_absent`: Keep only elements in file - - `remove_if_present`: Remove elements in file -- `filter_source`: Text file with identifiers (one per line) +Flattening discards the original hierarchy. Choose identifiers and separator so distinct paths produce distinct labels. The implementation stores results by joined identifier, so a collision can overwrite a preceding leaf. -**Outputs:** -- `output_filtered`: Elements passing filter -- `output_discarded`: Elements removed by filter +### Nest collection (`__NEST__`, 1.0.0) -**Use Case:** Subset collections based on sample lists, exclusion lists. +Declares collection `input` with types `list` or `paired`, and `output` with type `list:list`. Each top-level element is wrapped in a one-element inner list. The outer and inner labels are the original element identifier. Thus `[A, B]` becomes `[A → [A], B → [B]]`. ---- +This lets a tool that consumes a whole list run once per singleton list when Galaxy maps over the outer level. It does not group multiple samples by a key. Deeper inputs involve mapping, so account for the consumed inner type and remaining outer structure instead of treating the declared output as the final workflow type. The wrapper includes a nested-pair test, whose output collection assertion does not explicitly pin the full effective type. -### 4. Structure Transformation Tools +### Zip collections (`__ZIP_COLLECTION__`, 1.0.0) -#### Flatten Collection (`__FLATTEN__`) -**Version:** 1.0.0 +Takes dataset parameters `input_forward` and `input_reverse`. Produces `output`, a `paired` collection with member identifiers `forward` and `reverse`. -**Purpose:** Convert nested collection to flat list by merging identifiers. +With individual datasets, it creates one pair. With compatible mapped collections, Galaxy creates pairs per mapped position and assembles the outer collection. It does not perform a keyed join or verify that the two files are biologically matched. -**Inputs:** -- `input`: Any nested collection -- `join_identifier`: Separator for merging identifiers (`_`, `:`, `-`) +### Unzip collection (`__UNZIP_COLLECTION__`, 1.0.0) -**Output:** `list` collection +Consumes `input`, a `paired` collection, and produces datasets `forward` and `reverse`. Mapping over `list:paired` produces corresponding output lists. The tool separates associations without changing read contents. -**Example:** `list:paired` with element `i1` containing `forward`/`reverse` → flat list with `i1_forward`, `i1_reverse` +### Split paired and unpaired (`__SPLIT_PAIRED_AND_UNPAIRED__`, 1.0.0) ---- +Consumes `input` of type `list`, `list:paired`, or `list:paired_or_unpaired`. Produces: -#### Nest Collection (`__NEST__`) -**Version:** 1.0.0 +- `output_unpaired`, a `list` of singleton datasets. +- `output_paired`, a `list:paired` of pairs. -**Purpose:** Add nesting level to a collection. +A plain list sends all elements to the unpaired output. A list of pairs sends all elements to the paired output. For a mixed list, singleton subcollections are unwrapped and two-member subcollections are normalized to `paired`. Top-level identifiers are retained, and either output can be empty. This partitions an existing pairing structure. It does not infer pairs from filenames. -**Inputs:** -- `input`: Collection (`list`, `paired`) +## Combining and synchronizing collections -**Output:** `list:list` collection +### Merge collections (`__MERGE_COLLECTION__`, 1.0.0) -**Behavior:** Each element becomes a single-element inner list. +Takes repeat `inputs` with at least two entries, each containing collection parameter `input`. Produces `output`, deriving its collection type from the first input. Inputs need compatible structure. The tool appends their top-level elements, leaving inner collections intact. -**Use Case:** Enable mapping over elements that are themselves lists (Galaxy maps over outer level). +Conditional `advanced/conflict` controls overlapping identifiers: ---- +| `duplicate_options` | Behavior | +|---|---| +| `keep_first` | Default. Keeps the first element with each identifier. | +| `keep_last` | Keeps the last value for each identifier. Its position follows the identifier's first insertion. | +| `suffix_conflict` | Adds a suffix to every occurrence of an identifier appearing in multiple inputs. | +| `suffix_conflict_rest` | Adds a suffix to conflicting occurrences after the first input containing that identifier. | +| `suffix_every` | Adds a suffix to every element. | +| `fail` | Fails if an identifier occurs more than once. | -#### Zip Collections (`__ZIP_COLLECTION__`) -**Version:** 1.0.0 +For suffix modes, `advanced/conflict/suffix_pattern` defaults to `_#`. Every `#` is replaced by the one-based **input collection number**, not the occurrence number of that identifier. Check that generated suffixes cannot collide with existing labels. -**Purpose:** Create paired collection from two inputs. +The default can silently discard an overlapping sample. Use an explicit conflict policy when both occurrences matter. Merge is an append operation, not element-wise pairing. Compatible sample-sheet column definitions are retained when all inputs agree. -**Inputs:** -- `input_forward`: Dataset or collection for forward -- `input_reverse`: Dataset or collection for reverse +### Harmonize two collections (`__HARMONIZELISTS__`, 1.1.0) -**Output:** `paired` collection +Takes `input1`, the reference order, and optional `input2`. Both declare `list` or `list:paired`. Outputs `output1` and `output2` preserve their corresponding input types when both are supplied. -**Use Case:** Combine separate forward/reverse read files/collections into paired structure. +With two inputs, both outputs contain only top-level identifiers present in both collections, ordered as in `input1`. No intersection produces two empty collections. This is identifier synchronization, not sorting independently or joining on file contents. Discarded elements are not separate outputs. ---- +With `input2` omitted, `output1` copies `input1` and `output2` mirrors its structure using `expression.json` null datasets. For nested pairs, both members are null markers. This mode creates placeholder files, so the general shared-file behavior has an exception here. -#### Unzip Collection (`__UNZIP_COLLECTION__`) -**Version:** 1.0.0 +Wrapper 1.0.0 requires both inputs and has no omitted-input mode. -**Purpose:** Split paired collection into separate datasets. +## Cross products -**Inputs:** -- `input`: `paired` collection +### Flat Cross Product (`__CROSS_PRODUCT_FLAT__`, 1.0.0) -**Outputs:** -- `forward`: Forward dataset -- `reverse`: Reverse dataset +Consumes `input_a` and `input_b`, both `list`, and produces lists `output_a` and `output_b`. For input sizes n and m, each output has n × m elements. A is the outer loop, B the inner loop. Corresponding output elements refer to the A and B datasets for that combination. -**Use Case:** Separate paired reads for tools requiring individual files. +`join_identifier` accepts `_` by default, `:`, or `-`. Labels join the A and B identifiers. As with Flatten, ambiguous joined labels can collide. ---- +Normal element-wise mapping over both outputs then runs the downstream tool for every combination. The cross-product tool itself performs no comparison. -#### Split Paired and Unpaired (`__SPLIT_PAIRED_AND_UNPAIRED__`) -**Version:** 1.0.0 +### Nested Cross Product (`__CROSS_PRODUCT_NESTED__`, 1.0.0) -**Purpose:** Separate mixed paired/unpaired collection into two typed collections. +Consumes lists `input_a` and `input_b`, producing `output_a` and `output_b` as `list:list`. Both outputs use A identifiers at the outer level and B identifiers at the inner level. Within each A group, `output_a` repeats that A dataset and `output_b` contains the B datasets. -**Inputs:** -- `input`: Collection (`list:paired`, `list`, `list:paired_or_unpaired`) +Each output contains n × m leaf associations, organized in n groups of size m. Select the downstream input type and mapping depth to retain that grouping in its results. Large inputs expand job counts as well as collection metadata. [[nextflow-operators-to-galaxy-collection-recipes]] identifies unkeyed Cartesian expansion as a translation decision needing review. -**Outputs:** -- `output_unpaired`: `list` of unpaired datasets -- `output_paired`: `list:paired` collection +## Identifier and tag operations -**Use Case:** Handle mixed sequencing outputs, separate for different downstream tools. +### Relabel identifiers (`__RELABEL_FROM_FILE__`, 1.1.0) ---- - -### 5. Collection Combination Tools - -#### Merge Collections (`__MERGE_COLLECTION__`) -**Version:** 1.0.0 - -**Purpose:** Combine two or more collections into one. +Consumes any collection `input` and produces `output` with the same type. It changes top-level identifiers, without sorting elements or changing file contents. Conditional `how` holds these parameters: -**Inputs:** -- `inputs` (repeat, min 2): Collections to merge -- `advanced/conflict/duplicate_options`: Conflict handling - - `keep_first` (default): Keep first occurrence - - `keep_last`: Keep last occurrence - - `suffix_conflict`: Add suffix to all conflicting identifiers - - `suffix_conflict_rest`: Add suffix to conflicts after first - - `suffix_every`: Add suffix to every identifier - - `fail`: Error on conflict -- `suffix_pattern`: Pattern for suffixes (default `_#`) +| Parameter | Contract | +|---|---| +| `how/how_select` | `txt` by default, `tabular`, or `tabular_extended`. | +| `how/labels` | Text or tabular dataset supplying new names. | +| `how/strict` | Boolean, off by default. Requires matching row count and, for tables, coverage of every original identifier. | +| `how/from`, `how/to` | One-based column numbers for `tabular_extended`, defaults 1 and 2. | -**Output:** Merged collection (same type as inputs) +In `txt` mode, line N renames element N. Too few lines fail even with strict mode off. Extra lines are ignored unless strict mode is on. ---- +In `tabular` mode, each row must have exactly two tab-separated columns: old identifier and new identifier. Extended mode selects any two available columns. With strict mode off, identifiers missing from the table retain their original names. Extra source identifiers have no effect. Repeated source identifiers overwrite earlier table rows, so strict mode is not a complete duplicate-row validator. -#### Harmonize Two Collections (`__HARMONIZELISTS__`) -**Version:** 1.0.0 +New labels are stripped and must match Python regex `^[\w\- \.,]+$`: Unicode word characters, hyphen, space, dot, and comma. Empty labels, unsupported characters, and duplicate resulting labels fail. This differs from Extract Dataset's narrower identifier sanitizer. -**Purpose:** Make two collections have same identifiers in same order. +### Sort collection (`__SORTLIST__`, 1.0.0) -**Inputs:** -- `input1`: Reference collection with desired order -- `input2`: Collection to reorder +Consumes `input`, declared as `list` or `list:paired`, and produces `output` with the same type. Parameters are in conditional `sort_type`: -**Outputs:** -- `output1`: Filtered/ordered version of input1 -- `output2`: Filtered/ordered version of input2 +| `sort_type/sort_type` | Behavior | +|---|---| +| `alpha` | Default. Sorts identifiers lexicographically, with case significant. | +| `numeric` | Removes every character except ASCII digits, converts the remaining string to an integer, and sorts by that value. | +| `file` | Uses the line order from text dataset `sort_type/sort_file`. | -**Behavior:** Keeps only identifiers present in BOTH collections, orders by input1. +Numeric mode does not interpret signs or decimal points. `sample-1.2` has key 12. An identifier with no digits fails integer conversion. Equal keys retain their input order. -**Use Case:** Prepare two collections for element-wise processing. +For file mode, the file's reported data-line count must equal the collection element count and each stripped line must name an existing element. The wrapper asks for every identifier exactly once, but the implementation does not explicitly reject repeated names. Supply a permutation, since repetitions can collapse entries in the output dictionary. ---- +With valid inputs, sorting changes order while preserving sample membership and identity. It cannot establish correct pairing when identifiers disagree. -### 6. Cross Product Tools +### Tag elements (`__TAG_FROM_FILE__`, 1.0.0) -#### Flat Cross Product (`__CROSS_PRODUCT_FLAT__`) -**Version:** 1.0.0 +Consumes collection `input` and tabular dataset `tags`. Column 1 identifies the element, and later columns contain tags. Produces `output` with the same type, modifying tags on copied dataset associations. -**Purpose:** Create flat lists enabling all-vs-all comparison between two collections. +Parameter `how` accepts `add` by default, `set`, or `remove`. They add tags while retaining existing tags, replace tags, or remove named tags, respectively. Ordinary tags are labels. Name tags use `#` or `name:` and participate in Galaxy's name-tag inheritance. `group:` tags can supply grouping metadata to tools that interpret them. -**Inputs:** -- `input_a`: List collection A (length n) -- `input_b`: List collection B (length m) -- `join_identifier`: Separator for combined identifiers +For nested elements, the implementation looks up leaf element identifiers while traversing the copied subcollection. A row naming an outer sample does not necessarily tag all leaves under that sample. Repeated file identifiers keep the last row. Missing rows and empty tag lists should not be treated as a reliable way to clear all tags. The flat-dataset path only updates tags when a nonempty mapping is found. -**Outputs:** -- `output_a`: Flat list (n*m elements) - A elements repeated -- `output_b`: Flat list (n*m elements) - B elements repeated +## Apply Rules (`__APPLY_RULES__`, 1.1.0) -**Use Case:** Run comparison tool on every combination of elements. Element-wise processing of outputs produces all pairwise comparisons. +Consumes collection `input` and rule parameter `rules`. Produces `output` with its collection type determined by the rule mapping. The interactive rule builder previews metadata transformations, and workflows can store a fixed rule set. ---- +Rules operate on rows describing leaf datasets and columns holding metadata. They can filter and sort rows, derive columns from identifiers or tags, transform strings using regular expressions, and map columns into identifier levels, paired members, or tags. They can therefore reshape a hierarchy or split identifier components without modifying dataset contents. -#### Nested Cross Product (`__CROSS_PRODUCT_NESTED__`) -**Version:** 1.0.0 +This is a defined rule language, not arbitrary computation. Required mapping columns, pairing values, collisions, and resulting nesting determine whether a rule set is valid. [[galaxy-apply-rules-dsl]] documents all supported rule and mapping types, plus regex and column-index pitfalls. -**Purpose:** Create nested lists for all-vs-all comparison with hierarchical output. +## Tool selection guide -**Inputs:** -- `input_a`: List collection A -- `input_b`: List collection B +| Goal | Tool | Qualification | +|---|---|---| +| Assemble datasets into a list | Build list | Dataset inputs map when collections are selected. | +| Repeat one dataset | Duplicate file to collection | Repeats associations, not independent observations. | +| Select one dataset per collection | Extract dataset | Nested inputs may map over inner collections. | +| Remove empty content | Filter empty datasets | Requires successful inputs. | +| Continue after unsuccessful or paused samples | Keep success | Pending inputs still wait. | +| Remove unsuccessful completed elements | Filter failed datasets | Uses `is_ok`, not a red-only predicate. | +| Remove skipped/null markers | Filter null elements | Recognizes `expression.json` nulls. | +| Subset by top-level identifiers | Filter collection | Preserves input order and exposes discarded elements. | +| Collapse nesting | Flatten collection | Joined labels must remain unique. | +| Wrap elements in singleton lists | Nest collection | Account for mapped outer levels. | +| Form or separate pairs | Zip / Unzip | Does not infer biological pairing. | +| Separate existing mixed pairs and singletons | Split paired and unpaired | Either output can be empty. | +| Append collections | Merge collections | Default keeps only the first overlapping identifier. | +| Synchronize identifiers and order | Harmonize two collections | Two-input mode keeps only the intersection. | +| Materialize every A/B combination | Flat / Nested Cross Product | Expands to n × m combinations. | +| Change identifiers | Relabel identifiers | Does not synchronize independently ordered inputs. | +| Change order | Sort collection | Numeric mode concatenates digits. | +| Annotate datasets | Tag elements | Nested matching uses leaf identifiers. | +| Reshape using derived metadata | Apply Rules | Requires valid rule and output mappings. | -**Outputs:** -- `output_a`: `list:list` - nested structure of A elements -- `output_b`: `list:list` - nested structure of B elements +## Evidence and scope -**Use Case:** All-vs-all comparison preserving hierarchical structure in results (outer list = A elements, inner list = B comparisons). +The wrapper XML supplies parameter names, declared collection types, versions, output names, and examples. `DatabaseOperationTool` and its subclasses in `lib/galaxy/tools/__init__.py` supply readiness checks and operation behavior. The shared macros supply separator choices and quota guidance. Where wrapper help simplifies or contradicts the implementation, the qualifications above follow the implementation. ---- - -### 7. Metadata Manipulation Tools - -#### Relabel Identifiers (`__RELABEL_FROM_FILE__`) -**Version:** 1.1.0 - -**Purpose:** Change element identifiers using mapping file. - -**Inputs:** -- `input`: Any collection -- `how/how_select`: Mapping method - - `txt`: Simple text file (line N = new name for element N) - - `tabular`: Two-column mapping (old → new) - - `tabular_extended`: Any two columns from tabular file -- `labels`: Mapping file -- `strict`: Require exact match count - -**Output:** Collection with new identifiers - -**Valid Characters:** a-z, A-Z, 0-9, dash, underscore, dot, space, comma - ---- - -#### Sort Collection (`__SORTLIST__`) -**Version:** 1.0.0 - -**Purpose:** Reorder collection elements. - -**Inputs:** -- `input`: Collection (`list`, `list:paired`) -- `sort_type`: Method - - `alpha`: Alphabetical - - `numeric`: Numeric (ignores non-numeric chars) - - `file`: Custom order from text file - -**Output:** Sorted collection (same type) - ---- - -#### Tag Elements (`__TAG_FROM_FILE__`) -**Version:** 1.0.0 - -**Purpose:** Add/modify Galaxy tags on collection elements. - -**Inputs:** -- `input`: Any collection -- `tags`: Tabular file (col1: identifier, col2+: tags) -- `how`: Update mode - - `add`: Add new tags, keep existing - - `set`: Replace all tags - - `remove`: Remove specified tags - -**Output:** Tagged collection - -**Tag Types:** -- Simple tags: Plain labels -- Name tags: Prefix with `#` or `name:` - inherited by derived datasets -- Group tags: Prefix with `group:` - for grouping (e.g., treatment vs control) - ---- - -### 8. Apply Rules Tool (`__APPLY_RULES__`) -**Version:** 1.1.0 - -**Purpose:** Most powerful/flexible collection manipulation tool. Process collection metadata as tabular data using rule-based transformations. - -**Inputs:** -- `input`: Any collection -- `rules`: Rule definition (JSON-based transformation rules) - -**Output:** Transformed collection (type determined by rules) - -**Capabilities:** -- Filter elements (rows) -- Add/remove/reorder identifier levels (columns) -- Regular expression transformations -- Complex nesting/flattening -- Arbitrary reorganization - -**Use Cases:** -- Restructure collections between hierarchies -- Filter based on identifier patterns -- Split/merge identifier components -- Any operation not covered by simpler tools - -**Note:** Interactive form provides live preview. Can be used in workflows with static rules. - ---- - -## Tool Selection Guide - -| Goal | Recommended Tool | -|------|------------------| -| Build collection from datasets | Build List | -| Extract single element | Extract Dataset | -| Remove empty/failed/null elements | Filter Empty/Failed/Null | -| Subset by identifier list | Filter Collection | -| Convert nested → flat | Flatten Collection | -| Add nesting level | Nest Collection | -| Combine forward/reverse | Zip Collections | -| Separate paired collection | Unzip Collection | -| Concatenate collections | Merge Collections | -| Match two collections | Harmonize Two Collections | -| All-vs-all comparison | Cross Product (Flat or Nested) | -| Rename elements | Relabel Identifiers | -| Reorder elements | Sort Collection | -| Add metadata tags | Tag Elements | -| Complex restructuring | Apply Rules | -| Duplicate single dataset | Duplicate File to Collection | - ---- +The wrapper tests include concrete assertions for extraction, filtering, relabeling, sorting, nesting, merging, and cross-product output elements. This reference was checked against source and those test declarations. A live Galaxy server was not used to verify tool availability, mapped workflow output types, or deployment-specific readiness behavior. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-tool-job-failure-reference.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-tool-job-failure-reference.md index 0630071..9934a99 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-tool-job-failure-reference.md +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-tool-job-failure-reference.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-workflow-invocation-failure-reference]]" - "[[planemo-workflow-test-architecture]]" @@ -15,123 +15,130 @@ related_molds: - "[[implement-galaxy-tool-step]]" - "[[debug-galaxy-workflow-output]]" sources: - - "~/projects/repositories/galaxy/lib/galaxy/tools/__init__.py" - - "~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/xml.py" - - "~/projects/repositories/galaxy/lib/galaxy/tool_util/output_checker.py" - - "~/projects/repositories/galaxy/lib/galaxy/jobs" - - "~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/xml.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/stdio.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/output_checker.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/command_factory.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/jobs/runners/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/model/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/schema.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/api/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/services/jobs.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/jobs.py" summary: "Reference for Galaxy tool stdio rules, job failure detection, job states, and job API failure surfaces." --- -# Galaxy Tool And Job Failure Reference +# Galaxy tool and job failure reference -This is reference material, not a debug recipe. Use it to understand what Galaxy can know about a failed tool job and which API surfaces preserve that evidence. +A failed Galaxy job preserves execution evidence separately from workflow scheduling and output assertions. Start with its state, exit code, structured messages, and separate tool and runner streams. A red downstream dataset or stderr text alone does not identify the original failure. -## Model +[[galaxy-workflow-invocation-failure-reference]] covers scheduling and dependency messages. [[planemo-workflow-test-architecture]] locates the structured Planemo artifacts, and [[debug-galaxy-workflow-output]] uses this evidence to classify a failure before proposing repairs. -Galaxy tool failure handling is layered: +## Job evidence -- The tool wrapper defines expected failure semantics through `detect_errors`, ``, exit-code checks, regex checks, and command strictness. -- The job runner executes the command and captures exit code plus tool/job stdout and stderr streams. -- Galaxy evaluates configured failure rules and records structured `job_messages`. -- The job reaches a terminal state, output datasets may become `error`, and dependent jobs may pause or fail later. -- Workflow invocation APIs summarize those jobs, but job APIs preserve the most detailed tool-level evidence. +Full job detail includes execution identity, state, exit code, structured messages, dependencies, and separate tool and runner streams. The combined stream fields provide compatibility views of those streams. -## Tool Wrapper Failure Controls +| Field | Evidence | +|---|---| +| `id`, `tool_id`, `tool_version`, `state` | Identify the execution and the wrapper version whose failure rules apply. | +| `exit_code` | Process exit code recorded by the runner. A missing value is not evidence of exit code zero. | +| `tool_stdout`, `tool_stderr` | Captured streams from the tool command. | +| `job_stdout`, `job_stderr` | Captured streams from the job wrapper or runner context. | +| `stdout`, `stderr` | Combined compatibility views of tool and job streams. They lose the stream provenance. | +| `job_messages` | Structured records from stdio checks or output discovery. Empty messages do not prove success. | +| `dependencies` | Job dependencies included with full detail. | -Important wrapper controls: +Prefer separate streams for diagnosis. A runner failure can occur before the tool executes, and output processing can fail after the command exits successfully. -| Control | Meaning | -|---|---| -| `detect_errors="default"` | For non-legacy XML tools without explicit ``, Galaxy defaults to non-zero exit-code failure detection. | -| `detect_errors="exit_code"` | Adds fatal checks for non-zero exit codes, plus optional OOM exit-code handling. | -| `detect_errors="aggressive"` | Adds non-zero exit-code checks plus broad stdout/stderr regexes for OOM and generic error text. | -| `` | Adds explicit exit-code ranges or values with levels such as `fatal`, `warning`, or `fatal_oom`. | -| `` | Adds case-insensitive stdout/stderr regex checks. | -| `` | Controls shell strictness; newer tool profiles default to `set -e` behavior. | +## Job and output dataset states -Current Galaxy source parses these rules while loading the tool, then stores exit-code and regex rules on the tool object. XML parser behavior lives under `~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/xml.py`; concrete stdio presets live under `~/projects/repositories/galaxy/lib/galaxy/tool_util/parser/stdio.py`. +| Job state | Interpretation | +|---|---| +| `new`, `waiting`, `upload`, `queued`, `running`, `resubmitted` | Execution is pending or in progress. A timeout while polling these states needs timing and runner evidence. | +| `ok` | Galaxy accepted the execution. Check the expected outputs and assertions separately. | +| `error`, `failed` | Failure states. Inspect full detail and the producing job's outputs. | +| `paused` | Execution is blocked or paused. Inspect upstream inputs and pause information before blaming the tool command. | +| `skipped` | Execution was skipped, for example by a workflow condition. It is not itself a tool failure. | +| `stop`, `stopped`, `deleting`, `deleted` | Stopping or deletion states. Determine whether cancellation or deletion explains the missing result. | -## Rule Ordering And Levels +The API value for a stop in progress is `stop`, not `stopping`. Consumers use different terminal-state sets. Do not infer tool success from a workflow completion record, or infer permanent failure merely because polling stopped. -Galaxy evaluates explicit and preset rules in a defined order: +Output datasets have their own states, including `ok`, `error`, `paused`, `failed_metadata`, `deferred`, and `discarded`. `failed_metadata` identifies a metadata problem, while `deferred` denotes data awaiting materialization. Dataset deletion and purging are separate flags. For a red downstream output, trace its producing job and upstream datasets instead of treating job and dataset states as interchangeable. For mapped jobs, retain the failed member's job ID and collection element identifier. See [[galaxy-collection-semantics]]. -- Exit-code rules are evaluated before regex rules. -- Regexes search stdout and/or stderr case-insensitively. -- A fatal rule stops later checks. -- Warnings, log messages, and QC messages can produce `job_messages` without failing the job. +## Wrapper failure rules -Useful levels: +Galaxy loads an XML wrapper's failure rules from `` and ``, then checks the captured tool exit code and streams. Inspect the installed wrapper and its `profile` before interpreting a non-zero exit or stderr text. -| Level | Effect | +| Control | Behavior | |---|---| -| `log` | Records informational message; does not fail the job. | -| `qc` | Records QC message; does not fail the job. | -| `warning` | Records warning; does not fail the job. | -| `fatal` | Fails the job as a generic tool error. | -| `fatal_oom` | Fails as out-of-memory; runner behavior can use this for OOM handling/resubmission. | +| Omitted `detect_errors` or `detect_errors="default"` | Adds non-zero exit-code checks for profiles other than legacy `16.01` only when there is no `` element. An explicit `` disables this implicit preset, even if it contains no rules. | +| `detect_errors="exit_code"` | Adds fatal checks for negative and positive non-zero exit codes. Optional `oom_exit_code` adds an OOM-specific check before those ranges. | +| `detect_errors="aggressive"` | Adds non-zero exit checks and case-insensitive checks on both streams for OOM text, `exception:`, and `error:`. | +| `` | Defines explicit values or inclusive ranges and their severity. Open-ended ranges such as `1:` are supported. | +| `` | Searches selected tool streams case-insensitively. With no source specified, both stdout and stderr are searched. | +| `` | Requests `set -e` in the generated tool script. The default is true for profiles `20.09` and newer, false for older profiles. | -Low-confidence caveat: OOM resubmission behavior is runner/destination configuration dependent. The wrapper and output checker can classify OOM, but retry policy is not solely a wrapper property. +Explicit stdio rules are prepended to rules from an explicitly selected `exit_code` or `aggressive` preset. Adding a warning for an exit code does not override a later fatal preset rule matching that code. -## Job And Dataset States +When **no exit-code or regex rules exist**, the output checker falls back to treating any nonempty tool stderr as an error. In this fallback, a non-zero exit code with empty stderr does not itself trigger failure. Conversely, with configured rules, ordinary stderr text is not automatically fatal. -Job states relevant to workflow tests include: +Shell strictness and stdio checks serve different purposes. `set -e` can stop a multi-command script before later commands hide an earlier error, but follows the shell's exceptions and does not add `pipefail`. It does not guarantee that every component of a pipeline succeeded. Galaxy's command factory also bypasses script creation when the configured shell is `none`. -- In progress or queued: `new`, `queued`, `running`, `waiting`, `upload`, `resubmitted`. -- Success: `ok`. -- Failure or terminal problem: `error`, `failed`, `paused`, `stopped`, `deleted`, `skipped` depending context. +## Rule ordering and structured messages -Dataset states relevant to downstream failures include `ok`, `error`, `paused`, `failed_metadata`, `deferred`, and `discarded`. +Exit-code checks run before regex checks. Within each group, explicit rules precede presets. Regex rules check stderr before stdout when both are selected. -For workflow debugging, do not collapse job state and dataset state. A job can fail, its outputs can become error datasets, and a downstream workflow step can later fail because it consumes those datasets. +| XML level | Numeric `error_level` | Effect | +|---|---:|---| +| `log` | 1 | Records an informational message. | +| `qc` | 1.1 | Records a quality-control message. | +| `warning` | 2 | Records a warning. | +| `fatal` | 3 | Classifies a generic tool error. | +| `fatal_oom` | 4 | Classifies an out-of-memory error and stops further checking. | -## Stream And Message Fields +Log, QC, and warning matches do not fail the job by themselves. A generic `fatal` match does **not** stop all later checks. A later OOM match can upgrade the classification. The checker tracks the maximum severity, and level 4 ends checking. OOM classification can inform runner resubmission, but retry policy depends on the runner and destination configuration. -Galaxy distinguishes tool streams from job/runner streams: +Structured `job_messages` retain the matching rule evidence: -| Field | Meaning | +| Message `type` | Additional fields | |---|---| -| `tool_stdout` | stdout from the executed tool command. | -| `tool_stderr` | stderr from the executed tool command. | -| `job_stdout` | stdout from job wrapper/runner context. | -| `job_stderr` | stderr from job wrapper/runner context. | -| `stdout` | Combined stdout compatibility view. | -| `stderr` | Combined stderr compatibility view. | -| `job_messages` | Structured failure/warning messages produced by stdio and output checking. | -| `exit_code` | Process exit code observed by the runner. | +| `exit_code` | `exit_code`, `code_desc`, `desc`, `error_level`. | +| `regex` | `stream`, `match`, `code_desc`, `desc`, `error_level`. | +| `max_discovered_files` | `desc`, `error_level` for output-discovery failures, including too many files or an output name that is too long. | -Prefer `job_messages` plus separate streams for reference-quality failure interpretation. Combined `stdout` and `stderr` are useful for humans but lose provenance. +Regex `match` text is truncated after 256 characters when longer. `code_desc` is the rule's description, while `desc` includes the detected severity and explanation. Preserve the raw message rather than reducing it to “stderr contained an error.” Fallback stderr failure and runner errors need not produce stdio messages. -## API Surfaces +## Job API reference -Useful job APIs from Galaxy source under `~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/jobs.py`: +Paths are relative to the server URL and use encoded API IDs. -| API | Use | +| Method and path | Result | |---|---| -| `GET /api/jobs` | Filter jobs by state, tool id, workflow id, invocation id, history id, etc. | -| `GET /api/jobs/{job_id}` | Basic job detail. | -| `GET /api/jobs/{job_id}?full=true` | Full job detail, including streams and `job_messages`. | +| `GET /api/jobs` | Lists accessible jobs with filters including state, tool ID, history ID, workflow ID, and invocation ID. | +| `GET /api/jobs/{job_id}` | Basic job detail, including state, tool identity, and exit code. | +| `GET /api/jobs/{job_id}?full=true` | Adds separate and combined streams, `job_messages`, and dependencies. | | `GET /api/jobs/{job_id}/stdout` | Combined stdout as plain text. | | `GET /api/jobs/{job_id}/stderr` | Combined stderr as plain text. | -| `GET /api/jobs/{job_id}/console_output` | Live or stored tool stdout/stderr, useful while a job is running. | -| `GET /api/jobs/{job_id}/inputs` | Input datasets for the job. | -| `GET /api/jobs/{job_id}/outputs` | Output datasets and output collections. | -| `GET /api/jobs/{job_id}/metrics` | Job metrics, if available. | -| `GET /api/jobs/{job_id}/common_problems` | Known simple problems such as empty or duplicate inputs. | +| `GET /api/jobs/{job_id}/console_output` | Tool console output, subject to the requirements below. | +| `GET /api/jobs/{job_id}/inputs` | Job input datasets. | +| `GET /api/jobs/{job_id}/outputs` | Job output datasets and collections. | +| `GET /api/jobs/{job_id}/metrics` | Job metrics subject to access policy and availability. | +| `GET /api/jobs/{job_id}/common_problems` | Simple checks such as empty or duplicate inputs. These are clues, not a complete diagnosis. | -Admin-only or admin-enriched fields can include command line, traceback, destination, runner, handler, external id, and destination parameters. Do not assume a normal workflow-testing API key can retrieve everything. +`console_output` requires four query parameters: `stdout_position`, `stdout_length`, `stderr_position`, and `stderr_length`. They define the starting offsets and lengths of the requested stream ranges. -## Durable Reference Use +The destination must enable `live_tool_output_reporting`, including when retrieving a finished job through this endpoint. While running, Galaxy reads the requested ranges from tool stream files. Otherwise it returns stored tool streams. Full job detail exposes stored failure evidence independently of live reporting. -This note should inform generated skills when they need to preserve or inspect tool-level failure evidence: +Full detail does not grant administrator access. System details such as traceback, runner, destination, handler, and external job ID can require an administrator. Command-line exposure also depends on server configuration, and full-detail `job_metrics` are added for administrators. A normal workflow-testing key may therefore lack some server or runner evidence. -- A concrete workflow step should preserve tool id, version, input labels, output labels, datatype and collection shape so a later job failure can be traced back to the authoring decision. -- A runtime failure should not be classified from stderr alone if `job_messages`, exit code, and wrapper stdio rules are available. -- A workflow invocation failure may be caused by an upstream job, but invocation APIs are not a substitute for full job detail. +## Diagnostic pitfalls and verification limits -## Low-Confidence Or Version-Sensitive Points +- **Stderr is evidence, not a verdict.** Warnings can be harmless under configured stdio rules. An aggressive regex can also classify text as fatal despite exit code zero. Compare streams with the installed wrapper's rules. +- **Exit code zero is not scientific success.** Output collection, metadata, and assertions can still fail. Preserve output states and Planemo's `output_problems` alongside job records. +- **A paused consumer may hide an upstream failure.** Follow dependencies and producing jobs before changing the consumer's parameters. +- **No messages is not a clean bill of health.** A runner, fallback stderr check, or output-processing failure can leave no matching stdio record. Keep all available streams and state evidence. +- **Resubmission changes the context.** Record runner and destination details when available and distinguish an OOM classification from a configured retry decision. -- Tool profile defaults changed over time. Prefer current Galaxy source and wrapper profile over old generic rules. -- Some docs describe failure messages as stream text; current code preserves structured `job_messages` separately. -- OOM handling depends on job runner and destination configuration. -- Planemo may print or summarize job detail differently from raw Galaxy APIs; see [[planemo-workflow-test-architecture]]. +The parser, checker, state definitions, and API contracts were inspected in the pinned Galaxy source. No live run verified every runner's stream capture, OOM resubmission, or access configuration. Check the deployed server and installed wrapper before applying version-sensitive conclusions. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-user-tool-workflow-binding.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-user-tool-workflow-binding.md new file mode 100644 index 0000000..4a5bc55 --- /dev/null +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-user-tool-workflow-binding.md @@ -0,0 +1,91 @@ +--- +type: research +title: "Binding a workflow step to a user-defined tool" +tags: + - target/galaxy +status: draft +created: 2026-09-24 +revised: 2026-10-23 +revision: 4 +related_notes: + - "[[galaxy-user-tool-authoring-help]]" + - "[[galaxy-user-tool-authoring]]" + - "[[galaxy-workflow-draft-format]]" + - "[[gxformat2-schema]]" +related_molds: + - "[[author-galaxy-tool-wrapper]]" + - "[[implement-galaxy-tool-step]]" + - "[[advance-galaxy-draft-step]]" +sources: + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/client/src/components/Tool/authoringHelp.yml" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy/managers/tools.py" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py" + - "https://github.com/galaxyproject/galaxy/blob/5af1d03c929748006dd53ad451e5cb1fd140399a/lib/galaxy_test/workflow/inline_user_defined_tool.gxwf.yml" +summary: "How a workflow step uses an authored GalaxyUserTool: embedded under run:, never by tool_uuid, tool_id or content_id; registering via /api/unprivileged_tools." +--- + +A gxformat2 step that uses an authored `GalaxyUserTool` carries the whole tool definition under its `run:` key. That is the only way the Foundry binds a step to a user-defined tool. + +A step never names a user-defined tool by `tool_uuid`, `tool_id`, or `content_id` alone. The tool is not in a server's toolbox, so a `tool_id` does not resolve to it, even when the string matches the tool's `id`. A `uuid` identifies one registered copy of the tool, for its owner, on one server. Galaxy's workflow editor and its exports write any such references themselves, and Galaxy rejects a `tool_uuid` step without an embedded definition whose `tool_id` names a different tool. + +The rule is derived from the `workflows` section ("Using a tool in a workflow") of Galaxy's authoring help, vendored in [[galaxy-user-tool-authoring-help]]. The Galaxy-side facts below are pinned to the same commit. The gxwf behavior was checked with `@galaxy-tool-util/cli` 1.14.0, the version the Foundry pins. + +## Embedding the definition in a gxformat2 step + +The embedded form makes the workflow self-contained. When a user imports it, Galaxy registers a private copy of the tool for that user. That requires a server with `enable_beta_tool_formats` set and a user holding the `Custom Tool Execution` role, the same conditions as registering the tool directly. Converting the workflow to native `.ga` with gxformat2 carries the definition as `tool_representation`, with no `tool_id`, `tool_version`, or `tool_uuid`, and Galaxy imports that form the same way. Galaxy's own `.ga` export also includes the definition as `tool_representation`. Nothing has to be registered first, and nothing has to be injected into the converted file afterwards. + +To bind a step: + +- Put the complete `GalaxyUserTool` document under `run:`, unchanged. Its `id` and `version` identify the tool. +- Leave out `tool_id`, `tool_version`, and `tool_shed_repository`. They describe toolbox tools. Remove any `TODO` sentinel the template left in those fields rather than filling it. +- Key `in:` by the names of the definition's `data` and `collection` inputs. +- List every output a downstream step or a workflow output uses under `out:`, by the definition's output `name`. +- Set values for non-data inputs in `state`, keyed by input `name`. + +```yaml +steps: + head: + run: + class: GalaxyUserTool + id: head_lines + version: 0.1.0 + name: Head lines + container: quay.io/biocontainers/python:3.13 + shell_command: head -n '$(inputs.n_lines)' '$(inputs.data_input.path)' > out.txt + inputs: + - name: n_lines + type: integer + value: 10 + - name: data_input + type: data + format: [txt] + outputs: + - name: out + type: data + format: txt + from_work_dir: out.txt + in: + data_input: text_file + out: + - id: out + state: + n_lines: 5 +``` + +gxformat2's converter turns that step into a native tool step whose `tool_state` contains `n_lines: 5`, with the definition in `tool_representation`. Galaxy's own test workflow `inline_user_defined_tool.gxwf.yml` exercises the same form end to end, without `out:` or `state`. + +## Validating a draft with an embedded tool + +gxwf treats a `GalaxyUserTool` under `run:` as a tool step. [[draft-validate]] with and without `--concrete`, [[draft-next-step]], [[draft-extract]], and `gxwf validate` on the extracted workflow all run as they do for toolbox steps, and `draft-extract` keeps workflow outputs sourced from the embedded step even without an `out:` list. `gxwf validate-tool-source ` validates the definition itself when it is saved as a separate file. + +## Registering a tool on a server + +A step that embeds its tool does not need it registered. Registering is for running the tool by itself, outside a workflow, or for binding a native `.ga` step by `tool_uuid`. + +- Register with `POST /api/unprivileged_tools`, body `{"representation": }`. It needs the same `enable_beta_tool_formats` setting and `Custom Tool Execution` role as importing an embedded step, and returns the registered copy's `uuid`. The `api` section of [[galaxy-user-tool-authoring-help]] lists the other endpoints. +- Do not use `POST /api/dynamic_tools`. It is admin-only and returns 403 for an ordinary user. +- A returned `uuid` means Galaxy accepted the definition, not that the tool runs. Run it on test data before reporting it as working. + +## Open questions + +- Whether a Planemo-managed Galaxy runs a workflow with an embedded tool has not been tested. It would need `enable_beta_tool_formats` on that server and the `Custom Tool Execution` role for the test user. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/gxformat2-schema.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/gxformat2-schema.md new file mode 100644 index 0000000..af5053a --- /dev/null +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/gxformat2-schema.md @@ -0,0 +1,74 @@ +--- +type: research +title: "gxformat2 structural JSON Schema" +tags: + - target/galaxy +status: draft +created: 2026-05-05 +revised: 2026-11-06 +revision: 5 +license: MIT +license_file: LICENSES/galaxy-tool-util-ts.LICENSE +related_notes: + - "[[galaxy-collection-semantics]]" + - "[[galaxy-datatypes-conf]]" + - "[[galaxy-workflow-testability-design]]" + - "[[gxformat2-workflow-inputs]]" + - "[[galaxy-workflow-draft]]" +sources: + - "https://github.com/jmchilton/galaxy-tool-util-ts/blob/7ae4ecd0ba8d492225f58a6d455c4cc5317298f0/packages/schema/src/galaxy-workflow.ts" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py" +companions: + - "gxformat2.schema.json" +summary: "Vendored gxformat2 structural schema, its input, output, and step vocabulary, and the limits of schema-only validation." +--- + +# gxformat2 structural schema + +The companion `gxformat2.schema.json` is a vendored Draft-07 JSON Schema generated from the `@galaxy-tool-util/schema` Effect-TS model at `7ae4ecd`. Casting carries it with this note. Use it to inspect the structural vocabulary of a gxformat2 workflow, then check connections, tool references, and runtime behavior with gxwf and Galaxy validation. The schema does not establish that an `outputSource` exists or that a tool state is valid for its wrapper. + +The schema is regenerated by `pnpm sync:vendored --update`, which builds the upstream package and runs `gxwf structural-schema --format format2` before copying it here. Do not edit the companion by hand. + +## Root and input fields + +The root requires `class: GalaxyWorkflow`, `inputs`, `outputs`, and `steps`. Those sections accept arrays or keyed maps. The root also permits `id`, `label`, `doc`, `uuid`, `report`, `tags`, `comments`, `creator`, `license`, and `release`. + +Input entries describe the public workflow interface. The array form exposes these fields: + +| Field | Structural schema | Meaning and use | +|---|---|---| +| `id`, `label`, `doc` | Optional | Identify or describe the input. Check the effective imported name before using it as a test `job:` key. | +| `type` | Optional string, string array, or `null` | Enum members are `null`, `boolean`, `int`, `long`, `float`, `double`, `string`, `integer`, `text`, `File`, `data`, and `collection`. This is accepted vocabulary, not a list of preferred authoring spellings. | +| `format` | Optional string array or `null` | Galaxy datatype extension filter. See [[galaxy-datatypes-conf]]. | +| `collection_type` | Optional string or `null` | Collection shape such as `list:paired`. The schema does not enumerate valid shapes. See [[galaxy-collection-semantics]]. | +| `optional`, `default` | Optional boolean or `null`, and opaque value, respectively | Control whether an input can be omitted and what value is supplied when absent. See [[gxformat2-workflow-inputs]] for runtime behavior. | +| `position` | Optional `{top, left}` numeric coordinates or `null` | Workflow editor placement. | + +The generated schema does not include all fields accepted by current gxformat2 input handling, including `restrictions`, `suggestions`, and `restrictOnConnections`. [[gxformat2-workflow-inputs]] distinguishes that implementation behavior from what this vendored schema can check. + +## Output names and connections + +An output entry can carry `id`, `label`, `doc`, `outputSource`, and `type`. `outputSource` names the producing step output, written `step/output`. The output's **public name** is not always its `label`: normalized gxformat2 export places a native Galaxy workflow-output label in `id`, and the pinned IWC [SRA manifest workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs.ga) exports `id: paired_output`. Current gxformat2 import chooses `label` when present and otherwise `id`. Check the effective name after import or conversion before writing the sibling test's `outputs:` key. See [[galaxy-workflow-testability-design]]. + +The schema makes `outputSource` optional, even though a usable produced output needs a valid source. Its string type cannot prove that the referenced step and port exist. gxformat2 lint checks the step reference. Workflow validation must also check that the producing port exists. + +The step part of a `source` or `outputSource` depends on how the workflow lists its steps. In list form it is the step's `label` when one is present, otherwise its `id`. In map form, for a step with no `label`, it is the map key. + +A step's `in:` key addresses a parameter nested in a conditional or section with a `|`-joined path, and a repeat instance with `_N`. For example, `input_type|i` wires an input inside a conditional in `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences`, and `style_cond|type_cond|pick_from_0|value` wires the first instance of a repeat inside two conditionals in [[conditional-route-between-alternative-outputs]]. The key names the path, and the conditional's selector value stays in `state`. Each `_N` connection also needs its own entry in the repeat in `state`. The repeat creates the input terminal, so without the entry the connection does nothing and the job can still succeed. + +Step `out:` entries serve a different purpose. They can be output names or records with `id` and actions such as `rename`, `change_datatype`, `add_tags`, `remove_tags`, `hide`, `delete_intermediate_datasets`, and `set_columns`. These actions affect produced datasets or history presentation. They do not supply a top-level output's public name. + +A `rename` value is not a tool-XML label template. Galaxy's [`RenameDatasetAction`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py#L152-L256) fills in two forms. `#{input}` becomes the name of the dataset bound to that tool input. Use the tool's input name, with `.` for a nested one (e.g. `queries_0.input2`), and optional `| basename`, `| upper` or `| lower`. `${label}` becomes the value of the workflow parameter input with that label. Write `#{gtf_to_extend} BREW3R extended` to name an output after its input. A `${name}` with no matching parameter input is an informal, untyped parameter ([`get_informal_replacement_parameters`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py#L3094-L3103)). The run form asks for a free-text value, and an API or Planemo run that supplies no `replacement_params` leaves `${name}` literal in the dataset name. The tool-XML variables `${on_string}` and `${tool.name}` work only in a wrapper's ``; in a rename they fall into this case. This is what the workflow linters report as an "untyped parameter"; declare a parameter input instead. + +## Step fields and state + +The structural step `type` enum is `tool`, `subworkflow`, `pause`, and `pick_value`, with `null` also admitted. gxformat2's source schema defaults an omitted type to `tool`. A step can also carry `id`, `label`, `doc`, `position`, `tool_id`, `tool_shed_repository`, `tool_version`, `in`, `out`, `state`, `tool_state`, `run`, `runtime_inputs`, `when`, `errors`, and `uuid`. The structural schema does not require subtype-specific fields such as `tool_id` on a tool step, so validation must continue beyond JSON Schema. + +For hand-authored tool steps, use `state` for typed nested YAML, including conditionals and repeats. A converted workflow may contain `tool_state`, as the pinned IWC SRA manifest example does. Treat that as the representation received, not an authoring recommendation. Current import accepts `tool_state` as a map or JSON string, but gives `state` precedence if both fields are present. The structural schema permits both, so use only one on a step. Preserve an exported representation when reading or carrying it forward, and use `state` when authoring a new step. + +## What schema validation leaves open + +The root and array-form entries constrain their listed properties, but generated map-form `inputs` and `outputs` each include a permissive object branch. Passing this schema therefore does not prove that every map entry has only documented fields. In map form, the `type` enum and `format` shape can also escape the documented constraints. Every form still needs semantic checks for connections and tool configuration. + +The generated JSON also omits field descriptions and repeats `$id: "/schemas/unknown"` on opaque values. Some strict Ajv configurations reject those duplicate identifiers. If that occurs, compile with the project's supported non-strict configuration or remove duplicate markers in a validation copy. Keep the vendored companion unchanged. The upstream [gxformat2 source schema](https://github.com/galaxyproject/gxformat2/blob/main/schema/v19_09/workflow.yml) supplies field documentation that this generated artifact lacks. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/gxformat2.schema.json b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/gxformat2.schema.json new file mode 100644 index 0000000..526c8d2 --- /dev/null +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/gxformat2.schema.json @@ -0,0 +1,3926 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$defs": { + "WorkflowStepSchema": { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "tool_id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "tool_shed_repository": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "name", + "changeset_revision", + "owner", + "tool_shed" + ], + "properties": { + "name": { + "type": "string" + }, + "changeset_revision": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "tool_shed": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "tool_version": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "errors": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "in": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "source": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "source": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "null" + } + ] + }, + "out": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "add_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "change_datatype": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "delete_intermediate_datasets": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "hide": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "remove_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "rename": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "set_columns": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "add_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "change_datatype": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "delete_intermediate_datasets": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "hide": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "remove_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "rename": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "set_columns": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "null" + } + ] + }, + "state": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "tool_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "tool", + "subworkflow", + "pause", + "pick_value" + ] + } + ] + }, + "run": { + "anyOf": [ + { + "type": "object", + "required": [ + "inputs", + "outputs", + "class", + "steps" + ], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "inputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "outputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "GalaxyWorkflow" + ] + }, + "steps": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/WorkflowStepSchema" + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$ref": "#/$defs/WorkflowStepSchema" + } + } + ] + }, + "report": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "markdown" + ], + "properties": { + "markdown": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "comments": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "creator": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Person" + ] + }, + "givenName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "familyName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificPrefix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificSuffix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "jobTitle": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Organization" + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "license": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "release": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "runtime_inputs": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "when": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + } + }, + "type": "object", + "required": [ + "inputs", + "outputs", + "class", + "steps" + ], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "inputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "outputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "GalaxyWorkflow" + ] + }, + "steps": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/WorkflowStepSchema" + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$ref": "#/$defs/WorkflowStepSchema" + } + } + ] + }, + "report": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "markdown" + ], + "properties": { + "markdown": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "comments": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "creator": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Person" + ] + }, + "givenName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "familyName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificPrefix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificSuffix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "jobTitle": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Organization" + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "license": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "release": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false +} diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/nextflow-to-galaxy-channel-shape-mapping.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/nextflow-to-galaxy-channel-shape-mapping.md index f8219d8..caf83b8 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/nextflow-to-galaxy-channel-shape-mapping.md +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/nextflow-to-galaxy-channel-shape-mapping.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-06 -revision: 2 +revised: 2026-11-10 +revision: 3 related_notes: - "[[nextflow-workflow-io-semantics]]" - "[[nextflow-params-to-galaxy-inputs]]" @@ -25,6 +25,10 @@ related_notes: - "[[relabel-via-rules-and-find-replace]]" - "[[reshape-relabel-remap-by-collection-axis]]" - "[[sync-collections-by-identifier]]" + - "[[harmonize-by-sortlist-from-identifiers]]" + - "[[derive-parameter-from-file]]" + - "[[conditional-route-between-alternative-outputs]]" + - "[[open-requirements-ledger]]" - "[[tabular-compute-new-column]]" - "[[tabular-concatenate-collection-to-table]]" - "[[tabular-cut-and-reorder-columns]]" @@ -48,19 +52,20 @@ related_molds: - "[[implement-galaxy-tool-step]]" sources: - "https://github.com/galaxyproject/foundry/issues/52" + - "https://github.com/galaxyproject/foundry/issues/757" summary: "Maps common Nextflow channel, tuple, and path shapes to Galaxy dataset and collection shapes." --- # Nextflow-to-Galaxy Channel Shape Mapping -This note maps source-level Nextflow channel shapes onto Galaxy data and dataset-collection shapes. Evidence quality is uneven: Galaxy collection semantics and IWC collection operations are well grounded; several Nextflow shapes are observed in the pinned fixtures; `fromFilePairs` and arbitrary deep tuple mapping remain low-confidence without direct fixture evidence. +This note maps source-level Nextflow channel shapes onto Galaxy data and dataset-collection shapes. The Confidence column marks how well each row is grounded. ## Shape Mapping | Nextflow shape | Galaxy shape | Confidence | Notes | |---|---|---|---| | `path(x)` or a single `Channel.fromPath(...).first()` | `File` | High | One dataset input or output. | -| `val(meta)` alone | No dataset shape | High | Treat as identifiers, labels, tags, sample-sheet metadata, or parameters. It is not a Galaxy dataset by itself. | +| `val(meta)` alone | No dataset shape | High | Treat as identifiers, labels, tags, sample-sheet metadata, or parameters. It is not a Galaxy dataset by itself. For a value computed per sample and read by a later process as a tool argument, see "Per-element parameters from `meta`" below. | | Repeated `tuple val(meta), path(file)` | `list` | High | One dataset per sample/key. Galaxy maps tools over list collections. | | One `tuple(meta, [R1, R2])` | `paired` | High | Use only when the workflow input is one paired sample or one tool consumes one pair. | | Repeated `tuple(meta, [R1, R2])` | `list:paired` | High | Common paired-end workflow input shape. | @@ -69,12 +74,12 @@ This note maps source-level Nextflow channel shapes onto Galaxy data and dataset | `tuple(meta, path(a), path(b))` | Parallel lists or per-step `File` inputs | Medium | Not automatically a `paired` collection. It is usually a keyed record of multiple tool inputs. | | Global file plus per-sample files | Broadcast `File` plus mapped `list` | Medium | If one input is global, connect it once and map the collection input. | | `collect()` or `toList()` | Collection reduction | High | Usually one downstream invocation over a collection or multiple-input value. | -| `collectFile(...)` | `File` | High | Nextflow creates one new file; Galaxy should model a tool output, not a collection operation. | +| `collectFile(...)` | `File` | High | Nextflow creates one new file. Galaxy should model a tool output, not a collection operation. | | `groupTuple()` by key | `list`, `list:paired`, or `list:list` | Medium | Depends on grouped payload and whether the grouping axis matters downstream. | | `transpose()` after grouping | Explicit reshape or subcollection mapping | Medium | May need flattening, Apply Rules, or identifier-preserving reshaping. | | `combine(..., by: key)` or `join(...)` | Multi-input collection map if identifiers match | Medium | Otherwise use explicit synchronization by identifiers/order. | | `branch { ... }` | Branch wiring or explicit filters | Medium | Shape is whatever each branch returns; per-element routing needs review. | -| `mix(...)` | Merge compatible streams/collections | Medium | Use direct wiring for reports/versions; use `__MERGE_COLLECTION__` or `__BUILD_LIST__` when materialized. | +| `mix(...)` | Merge compatible streams/collections | Medium | Use direct wiring for reports/versions, and `__MERGE_COLLECTION__` or `__BUILD_LIST__` when materialized. | | `multiMap { ... }` | Synchronized split into Galaxy inputs | Medium | Usually channel-only fan-out, but downstream rejoin depends on identifier discipline. | | `fromFilePairs` | `paired` or `list:paired` | Low | Conceptually clean, but not directly observed in the pinned fixtures. | | Arbitrary deep tuples | Parallel lists, `list:list`, or manual modeling | Low | Galaxy collection types are not arbitrary tuple records. Require per-tool review. | @@ -95,6 +100,38 @@ Use explicit Galaxy collection-operation or tabular steps when the translation c | Unbox a singleton collection | `__EXTRACT_DATASET__` | | Convert a collection of tabular outputs to one table | `collapse_dataset` or `collection_column_join` | +## Per-element parameters from `meta` + +nf-core pipelines often compute a value per sample, write it into `meta`, and read it as a tool argument in that sample's later processes. In nf-core/rnaseq's `strandedness: auto` path, `subworkflows/nf-core/fastq_qc_trim_filter_setstrandedness/main.nf` returns `meta + [strandedness: ...]`, and later modules read `meta.strandedness`. The collection operations above reshape data. This route keeps the data as it is and varies a tool parameter per element. + +Carry the value as a one-value dataset per element, in a collection with the same structure and order as the data collection. Map `param_value_from_file` over it and choose `param_type` by the target: `integer` or `float` for a number, `boolean` for a boolean, `text` for text. Connect the matching output (`integer_param`, `float_param`, `boolean_param` or `text_param`) to the target parameter of the step that maps over the data collection: + +```text +per-element values (list) ──> param_value_from_file (mapped) ──> integer_param ──┐ + ▼ +reads (list) ──────────────────────────────────────────> downstream tool (mapped, target parameter) +``` + +Galaxy runs one job per element and pairs the two collections by position, not by element identifier. A same-length value collection in another order runs without error and gives each element the value at its position, not its own. Produce the values with a step mapped over the data collection so they inherit its order. When they arrive as a separate collection, reorder them first: run `collection_element_identifiers` on the data collection, then `__SORTLIST__` with `sort_type: file` on the value collection, connecting the identifier file to `sort_type|sort_file` ([[harmonize-by-sortlist-from-identifiers]]). + +| Target | Evidence | Status | +|---|---|---| +| Integer tool parameter | `verification/workflows/per-element-parameter-from-file/`: values 1 and 3 on two elements, and positional pairing in its second case | Verified | +| Text tool parameter | Galaxy's `lib/galaxy_test/workflow/map_over_expression.gxwf.yml` and its `-tests.yml`: per-element values `A` and `B` reach a text input | Verified | +| Per-element `when` (boolean) | Galaxy's `lib/galaxy_test/api/test_workflows.py::test_run_workflow_conditional_step_map_over_expression_tool`: one of two elements is skipped | Verified | +| Parameter inside a tool conditional's case | IWC `epigenetics/consensus-peaks/consensus-peaks-atac-cutandrun`: mapped "convert min nb of reads to parameter" feeds `samtools_view`'s `mode\|subsample_config\|subsampling_mode\|target`, with values identical across elements | Wiring only | +| `select` tool parameter | The same fixture's `per-element-select-boolean` workflow: text values `C` and `D` choose `Cut1`'s `delimiter` per element | Verified | +| Boolean tool parameter | The same workflow: `true` and `false` set `wc_gnu`'s `include_header` per element | Verified | +| Float tool parameter | None | Not tested | +| A tool conditional's test parameter | [#661](https://github.com/galaxyproject/foundry/issues/661) reports a Galaxy 500 when one is connected. [#667](https://github.com/galaxyproject/foundry/issues/667) reports a workflow connecting one that runs on usegalaxy.org | Open | + +Use this route for every target except the Open row, and cover a Wiring only or Not tested target with a workflow test whose elements carry different values. rnaseq's strandedness reaches both kinds of target. featureCounts' `strand_specificity` is a `select` and takes this route. A `select` takes an option value, not the source's string, so translate first: featureCounts' options are `0`, `1` and `2`, where rnaseq writes `unstranded`, `forward` and `reverse`. [[map-workflow-enum-to-tool-parameter]] does this with `map_param_value`, which hasn't been tested mapped over elements. Salmon's `--libType` is `strandedness`, the test parameter of its `libtype` conditional, and takes the fallback. + +For a conditional's test parameter, or when a workflow test of this route fails, fall back in this order: + +1. For a small fixed set of values, run one branch per value. For each value, write a per-element boolean file with a mapped comparison step, read it with mapped `param_value_from_file` (`param_type: boolean`), gate that value's branch with `when`, and give the branch's step that value as a fixed parameter. Rejoin the branches with Galaxy's built-in `pick_value` (`pick_style: first_or_error`) mapped over the branch outputs, never `__MERGE_COLLECTION__`. A gated mapped step still emits a full-size collection with its skipped elements as null, so a merge returns one element per branch per sample. Galaxy's `test_run_workflow_conditional_step_map_over_expression_tool_pick_value` covers this mapped rejoin, and [[conditional-route-between-alternative-outputs]] covers `pick_value`'s settings. +2. Otherwise, expose one workflow-level parameter for all elements. Append a `kind: dropped` entry to the [[open-requirements-ledger]] whose `units` names the per-element parameter and whose `because` cites the Open row or the failing test. Expose the computing step's output as a workflow output so a user can run once, read the computed values, and re-run with the value they choose. + ## Evidence Corpus-observed Galaxy semantics: @@ -104,13 +141,13 @@ Corpus-observed Galaxy semantics: - [[galaxy-apply-rules-dsl]] explains identifier-derived collection reshaping. - [[iwc-transformations-survey]] records which collection transformations appear in IWC workflows. -Pinned fixture examples used by the research pass: +Nextflow fixture examples: - `workflow-fixtures/pipelines/nf-core__demo/subworkflows/local/utils_nfcore_demo_pipeline/main.nf` for `tuple(meta, reads)` paired-read handling. - `workflow-fixtures/pipelines/nf-core__rnaseq/main.nf` for single-end versus paired-end branching and reads normalization. - `workflow-fixtures/pipelines/nf-core__taxprofiler/main.nf` and `workflow-fixtures/pipelines/nf-core__taxprofiler/subworkflows/local/*/main.nf` for `groupTuple`, `transpose`, `mix`, `combine`, and `multiMap` patterns. - `workflow-fixtures/pipelines/nf-core__fetchngs/workflows/sra/main.nf` for `collectFile` and accession-driven data fetching. -- `$IWC_FORMAT2/virology/influenza-isolates-consensus-and-subtyping/influenza-consensus-and-subtyping.gxwf.yml` as a Galaxy `list:paired` exemplar. +- IWC `virology/influenza-isolates-consensus-and-subtyping/influenza-consensus-and-subtyping` as a Galaxy `list:paired` exemplar. ## Low-Confidence TODOs diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/schemas/galaxy-workflow-draft.schema.json index 3a39f43..dd3a94a 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/schemas/galaxy-workflow-draft.schema.json +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/schemas/galaxy-workflow-draft.schema.json @@ -75,7 +75,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_version": { "anyOf": [ @@ -108,7 +108,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "out": { "anyOf": [ @@ -309,7 +309,7 @@ ] }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "release": { "anyOf": [ @@ -412,22 +412,28 @@ "items": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" } ] }, @@ -437,22 +443,28 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" }, { "type": "string" @@ -927,7 +939,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "label": { "anyOf": [ @@ -1040,7 +1052,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_state": { "anyOf": [ @@ -1856,7 +1868,9 @@ "text", "File", "data", - "collection" + "collection", + "color", + "directory_uri" ], "type": "string" } @@ -1952,10 +1966,10 @@ ] }, "restrictions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "suggestions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "type": { "enum": [ @@ -1963,6 +1977,9 @@ "string" ], "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" } }, "required": [ @@ -1971,6 +1988,96 @@ "type": "object" }, "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { "additionalProperties": false, "properties": { "default": { @@ -2016,7 +2123,7 @@ "required": [], "type": "object" }, - "Shared10": { + "Shared11": { "additionalProperties": false, "properties": { "default": { @@ -2124,7 +2231,7 @@ ], "type": "object" }, - "Shared11": { + "Shared12": { "additionalProperties": false, "properties": { "default": { @@ -2230,7 +2337,7 @@ "required": [], "type": "object" }, - "Shared12": { + "Shared13": { "additionalProperties": false, "properties": { "default": { @@ -2337,7 +2444,7 @@ ], "type": "object" }, - "Shared13": { + "Shared14": { "additionalProperties": false, "properties": { "default": { @@ -2424,7 +2531,143 @@ ], "type": "object" }, - "Shared14": { + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { "anyOf": [ { "items": { @@ -2457,7 +2700,7 @@ } ] }, - "Shared15": { + "Shared18": { "anyOf": [ { "type": "null" @@ -2488,11 +2731,11 @@ } ] }, - "Shared16": { + "Shared19": { "anyOf": [ { "items": { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, "type": "array" }, @@ -2500,7 +2743,7 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, { "type": "string" @@ -2522,7 +2765,7 @@ } ] }, - "Shared17": { + "Shared20": { "anyOf": [ { "type": "null" @@ -2610,7 +2853,7 @@ "$ref": "#/$defs/Shared1" }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "uuid": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/schemas/parsed-tool.schema.json b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/schemas/parsed-tool.schema.json new file mode 100644 index 0000000..52bda10 --- /dev/null +++ b/plugins/foundry-skills/skills/implement-galaxy-tool-step/references/schemas/parsed-tool.schema.json @@ -0,0 +1,930 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "id", + "version", + "name", + "description", + "inputs", + "outputs", + "citations", + "license", + "profile", + "edam_operations", + "edam_topics", + "xrefs" + ], + "properties": { + "id": { + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "name": { + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "inputs": { + "type": "array", + "items": { + "type": "object" + } + }, + "outputs": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type", + "format", + "format_source", + "metadata_source", + "discover_datasets", + "from_work_dir", + "precreate_directory" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "data" + ] + }, + "format": { + "type": "string" + }, + "format_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "metadata_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "discover_datasets": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path", + "sort_key", + "sort_comp", + "sort_reverse", + "pattern" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "pattern" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + }, + "sort_key": { + "type": "string", + "enum": [ + "filename", + "name", + "designation", + "dbkey" + ] + }, + "sort_comp": { + "type": "string", + "enum": [ + "lexical", + "numeric" + ] + }, + "sort_reverse": { + "type": "boolean" + }, + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "tool_provided_metadata" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "from_work_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "precreate_directory": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "collection" + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_from_rules": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "structured_like": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "discover_datasets": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path", + "sort_key", + "sort_comp", + "sort_reverse", + "pattern" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "pattern" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + }, + "sort_key": { + "type": "string", + "enum": [ + "filename", + "name", + "designation", + "dbkey" + ] + }, + "sort_comp": { + "type": "string", + "enum": [ + "lexical", + "numeric" + ] + }, + "sort_reverse": { + "type": "boolean" + }, + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "tool_provided_metadata" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "structure": { + "type": "object", + "required": [ + "collection_type", + "collection_type_source", + "collection_type_from_rules", + "structured_like", + "discover_datasets" + ], + "properties": { + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_from_rules": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "structured_like": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "discover_datasets": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path", + "sort_key", + "sort_comp", + "sort_reverse", + "pattern" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "pattern" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + }, + "sort_key": { + "type": "string", + "enum": [ + "filename", + "name", + "designation", + "dbkey" + ] + }, + "sort_comp": { + "type": "string", + "enum": [ + "lexical", + "numeric" + ] + }, + "sort_reverse": { + "type": "boolean" + }, + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "tool_provided_metadata" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "integer" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "float" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "name", + "label", + "hidden", + "type" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "hidden": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "boolean" + ] + } + }, + "additionalProperties": false + } + ] + } + }, + "citations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "type", + "content" + ], + "properties": { + "type": { + "type": "string" + }, + "content": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "license": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "edam_operations": { + "type": "array", + "items": { + "type": "string" + } + }, + "edam_topics": { + "type": "array", + "items": { + "type": "string" + } + }, + "xrefs": { + "type": "array", + "items": { + "type": "object", + "required": [ + "value", + "type" + ], + "properties": { + "value": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "help": { + "anyOf": [ + { + "type": "object", + "required": [ + "format", + "content" + ], + "properties": { + "format": { + "type": "string" + }, + "content": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false +} diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/SKILL.md b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/SKILL.md index 6ffff86..9c41541 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/SKILL.md +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/SKILL.md @@ -13,9 +13,9 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs -- Read artifact `galaxy-test-plan`. Schema: galaxy-workflow-test-plan. Produced by `changeset-to-galaxy-test-plan`, `cwl-test-to-galaxy-test-plan`, `freeform-summary-to-galaxy-test-plan`, `nextflow-test-to-galaxy-test-plan`. Schema-valid Galaxy test plan (galaxy-workflow-test-plan) from a *-test-to-galaxy-test-plan Mold; carries job inputs, expected outputs, assertion intent, fixture provenance, label assumptions, unresolved mappings, and omissions. +- Read artifact `galaxy-test-plan`. Schema: galaxy-workflow-test-plan. Produced by `changeset-to-galaxy-test-plan`, `cwl-test-to-galaxy-test-plan`, `freeform-summary-to-galaxy-test-plan`, `nextflow-test-to-galaxy-test-plan`, `workflow-brief-to-galaxy-test-plan`. Schema-valid Galaxy test plan (galaxy-workflow-test-plan) from a *-test-to-galaxy-test-plan Mold; carries job inputs, expected outputs, assertion intent, fixture provenance, label assumptions, unresolved mappings, and omissions. - Read artifact `galaxy-workflow`. Produced by `advance-galaxy-draft-step`, `mature-galaxy-workflow-for-iwc`. Concrete gxformat2 workflow being tested — the loop-endstate `galaxy-workflow.gxwf.yml` from advance-galaxy-draft-step (`class: GalaxyWorkflow`); provides the real input/output labels, outputs, and collection shapes the test must assert against. -- Read artifact `test-data-refs`. Produced by `cwl-to-test-data`, `find-test-data`, `nextflow-to-test-data`, `paper-to-test-data`. Resolved test data references (URLs, paths, expected shapes) from paper-to-test-data or find-test-data. +- Read artifact `test-data-refs`. Schema: test-data-refs. Produced by `cwl-to-test-data`, `find-test-data`, `nextflow-to-test-data`, `paper-to-test-data`. Resolved test data references (URLs, paths, expected shapes) from paper-to-test-data or find-test-data. ## Outputs @@ -23,12 +23,12 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ Bundled reference: `references/cli/planemo.md`. @@ -71,7 +71,7 @@ The workflow is the contract: input and output labels in the test file must addr - Never emit a `path:` (or `location:`) the test references but no source produces — an un-materialized path passes the static cross-check and fails only at upload, far from here. If a ref is `resolved: false`, surface the gap rather than authoring a path to a file that does not exist. Inputs must match the workflow's collection shapes and datatypes. -3. **Author assertions.** Materialize the plan's assertion intent into concrete output assertions. Choose assertion families and tolerances per planemo-asserts-idioms; check each shortcut against iwc-shortcuts-anti-patterns so an existence-only or size-only assertion is a deliberate choice, not an evasion. Honor the plan's `omissions[]` and treat low-`confidence` synthesized intent as a starting point to tighten against the real invocation. +3. **Author assertions.** Materialize the plan's assertion intent into concrete output assertions. Choose assertion families and tolerances per planemo-asserts-idioms; check each shortcut against iwc-shortcuts-anti-patterns so an existence-only or size-only assertion is a deliberate choice, not an evasion. Honor the plan's `omissions[]` and treat low-`confidence` synthesized intent as a starting point to tighten against the real invocation. A `family` naming an output-attribute field (`element_count`, `count`, `ftype`, `checksum`, `metadata`) or a compare operator goes beside `asserts:`, not inside it; a non-null `element_identifier` goes under `element_tests`, with `/` separating nested ranks. 4. **Validate static.** Run validate-tests for the schema gate, then the workflow-label cross-check (`checkTestsAgainstWorkflow`): zero missing input labels, zero missing output labels, no collection/datatype mismatches. Fix before spending a Planemo run. 5. **Run green.** Drive planemo `test` with the staged data. "Managed Galaxy" here means **Planemo-managed**: `planemo test` bootstraps its own Galaxy and installs the workflow's tools from the Tool Shed/conda — it does **not** require a pre-provisioned external server, so absence of a running Galaxy is not a reason to skip this gate (cost/runtime of heavy tool or reference-DB installs may be, but that is a deliberate deferral, not an impossibility). On green, hand off the test file plus enough invocation/job/assertion context for run-workflow-test and debug-galaxy-workflow-output to use if a later run fails. diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_provenance.json b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_provenance.json index d7b4a63..06ff539 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_provenance.json +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "implement-galaxy-workflow-test", "path": "content/molds/implement-galaxy-workflow-test/index.md", - "revision": 8, - "content_hash": "966c486ccda2eb1e0f06afa674c5d6b85a7e7bfcc9b1545309e3ef8b1016f931", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 11, + "content_hash": "34855691581889d36265e00178088b8a52a51ee72119329261bca186f7cb480e", + "commit": "4c747d65e84f0100611458b5e869e9641bc50a21" }, - "cast_at": "2026-09-13T20:58:28.168Z", + "cast_at": "2026-11-09T16:55:02.589Z", "cast_date": "2026-07-20", "cast_revision": 1, "cast_history": [ @@ -44,8 +44,8 @@ "load": "upfront", "evidence": "corpus-observed", "purpose": "Runtime for workflow_test_init / workflow_test_on_invocation; install before authoring tests against a live invocation.", - "src_hash": "12e8cac3ffb4ab7f007ced7a8d4f01e6b3785d06a55926496bbacc0a8f3fc5c5", - "dst_hash": "12e8cac3ffb4ab7f007ced7a8d4f01e6b3785d06a55926496bbacc0a8f3fc5c5", + "src_hash": "7a149ff6f1cc396dd72bbdd7d0b93acecb12e1b86663c6119d4c0815b411fe94", + "dst_hash": "7a149ff6f1cc396dd72bbdd7d0b93acecb12e1b86663c6119d4c0815b411fe94", "source": "deterministic" }, { @@ -59,8 +59,8 @@ "evidence": "corpus-observed", "purpose": "Revise workflow inputs, outputs, labels, checkpoints, and collection identifiers so meaningful tests can be authored.", "trigger": "When test authoring reveals missing labels, omitted workflow-level outputs, unstable collection identifiers, weakly assertable final outputs, or fixture-shape pressure on workflow inputs.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -74,8 +74,8 @@ "evidence": "corpus-observed", "purpose": "Flag assertion shortcuts that are acceptable in IWC versus shortcuts that should be avoided.", "trigger": "When considering existence-only, size-only, image-only, checksum, output-label, or negative-test patterns.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -89,8 +89,8 @@ "evidence": "corpus-observed", "purpose": "Assemble job input fixtures, remote URLs, hashes, collection shapes, and test-data layout in IWC style.", "trigger": "When writing or revising the job/input side of a Galaxy workflow test file.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -104,8 +104,8 @@ "evidence": "corpus-observed", "purpose": "Choose assertion families, tolerance magnitudes, and the static/Planemo validation loop.", "trigger": "When writing or revising output assertions for a Galaxy workflow test file.", - "src_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", - "dst_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", "source": "deterministic" }, { @@ -119,8 +119,8 @@ "evidence": "corpus-observed", "purpose": "Write tests with stable labels and artifacts that Planemo can connect back to Galaxy invocations, jobs, and outputs.", "trigger": "When adding or revising workflow tests that will be iterated with Planemo or generated from existing invocations.", - "src_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", - "dst_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", + "src_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "dst_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", "source": "deterministic" }, { @@ -135,8 +135,8 @@ "purpose": "Input contract: read the schema-valid Galaxy test plan (job inputs, expected outputs, assertion intent, tolerances, label assumptions, unresolved mappings, omissions) and convert it into tests-format output, reconciling assumed labels and fixtures against the real draft.", "verification": "Consume a cast test plan from a *-test-to-galaxy-test-plan Mold and confirm tests-format output can be authored from it without re-reading the source summary.", "license": "MIT", - "src_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", - "dst_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", + "src_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "dst_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "source": "deterministic" }, { @@ -151,8 +151,8 @@ "purpose": "JSON Schema contract for the Galaxy workflow test format. Output of this Mold must validate against it.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", - "dst_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", + "src_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "dst_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } @@ -175,7 +175,8 @@ "changeset-to-galaxy-test-plan", "cwl-test-to-galaxy-test-plan", "freeform-summary-to-galaxy-test-plan", - "nextflow-test-to-galaxy-test-plan" + "nextflow-test-to-galaxy-test-plan", + "workflow-brief-to-galaxy-test-plan" ] }, { @@ -189,6 +190,7 @@ { "id": "test-data-refs", "description": "Resolved test data references (URLs, paths, expected shapes) from paper-to-test-data or find-test-data.", + "inherited_schema": "[[test-data-refs]]", "producers": [ "cwl-to-test-data", "find-test-data", diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_required_tools.json b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_required_tools.json index d93d84d..de85976 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_required_tools.json +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_required_tools.json @@ -3,9 +3,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", @@ -17,9 +17,9 @@ "tool": "planemo", "origin": "pypi", "package": "planemo", - "package_version": "0.75.47", + "package_version": "0.75.48", "invoke": "planemo", - "invoke_fallback": "uvx --from planemo==0.75.47 planemo", + "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "reference": "references/cli/planemo.md", diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_verify.json b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_verify.json index b2d4253..3369065 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_verify.json +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/_verify.json @@ -10,6 +10,20 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-workflow-test-plan", + "--schema-sha256", + "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "{artifact_path}" + ] + }, + { + "artifact_id": "test-data-refs", + "direction": "input", + "kind": "json", + "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", + "validator_bin": "foundry", + "args": [ + "validate-test-data-refs", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/cli/planemo.md b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/cli/planemo.md index 10eeb14..d5fa285 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/cli/planemo.md +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/cli/planemo.md @@ -3,17 +3,17 @@ type: cli-tool tool: planemo origin: pypi package: planemo -package_version: "0.75.47" +package_version: "0.75.48" invoke: planemo -invoke_fallback: "uvx --from planemo==0.75.47 planemo" +invoke_fallback: "uvx --from planemo==0.75.48 planemo" availability_check: "planemo --version" docs_url: "https://planemo.readthedocs.io/" tags: - cli/planemo status: draft created: 2026-05-10 -revised: 2026-09-13 -revision: 6 +revised: 2026-11-09 +revision: 7 summary: "Galaxy tool/workflow runtime testing CLI; used by run-workflow-test and friends." --- @@ -23,7 +23,7 @@ Galaxy's runtime testing and authoring CLI. Foundry Molds invoke `planemo test`, ## Pin -`package_version` pins to the released `planemo==0.75.47` from PyPI — base upstream planemo, no fork. It is the single source of truth: the sync scripts carry no pin of their own and stamp whichever planemo they invoked, and `make check-planemo-pin` fails if any recorded copy of the version disagrees with this field. +`package_version` pins to the released `planemo==0.75.48` from PyPI — base upstream planemo, no fork. It is the single source of truth: the sync scripts carry no pin of their own and stamp whichever planemo they invoked, and `make check-planemo-pin` fails if any recorded copy of the version disagrees with this field. [galaxyproject/planemo#1636](https://github.com/galaxyproject/planemo/pull/1636) merged 2026-05-14 and first shipped in released **0.75.42**, so `planemo cli_metadata` and `planemo output_schema` are available in every release at or above this pin. Every Foundry consumer runs off this base pin: the workflow-test phases ([[run-workflow-test]], [[implement-galaxy-workflow-test]]) which need only `planemo test --test_output_json`, the convergence loop in [[convert-nfcore-module-to-galaxy-tool]], and the vendored-artifact regeneration story (`packages/planemo-cli-meta/`, `packages/planemo-test-report-schema/`). No fork pin is required. @@ -32,13 +32,13 @@ The floor is now 0.75.46 for a packaging reason, not a feature one. `galaxy-tool ## Install ```sh -uvx --from planemo==0.75.47 planemo --version +uvx --from planemo==0.75.48 planemo --version ``` For a persistent install: ```sh -uv tool install planemo==0.75.47 +uv tool install planemo==0.75.48 ``` Contributor laptops only need planemo when **regenerating** vendored artifacts (the schema JSON in `packages/planemo-test-report-schema/`, the planemo CLI manual pages under `content/cli/planemo/`). Normal Foundry development — `npm run validate`, `npm run test`, `npm run packages-test` — builds against the vendored JSON and does not invoke planemo. diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-asserts-idioms.md index 6d0fef7..bd1160e 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-asserts-idioms.md +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-asserts-idioms.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-11 -revision: 6 +revised: 2026-10-21 +revision: 10 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,231 +16,109 @@ related_notes: - "[[validate-tests]]" - "[[iwc-tabular-operations-survey]]" - "[[galaxy-discover-datasets]]" -summary: "Decision and idiom guide for picking planemo workflow-test assertions: which family per output type, how to size tolerances, when to validate." +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." --- -# Planemo asserts: idiom and decision guide +# Choosing Planemo workflow-test assertions -Companion to [[iwc-test-data-conventions]] (input shapes), [[galaxy-workflow-testability-design]] (workflow structure before test YAML exists), and [[iwc-shortcuts-anti-patterns]] (what's accepted vs smell). This note is forward-looking: when authoring a new `-tests.yml`, which assertion family fits which output, and what the recommended tolerances and operators are. +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. -The **vocabulary itself is not restated here** — every assertion's parameter list, types, defaults, required fields, and Python docstring is rendered from the test-format JSON Schema at [[tests-format]]. Assertion names below deep-link into that page (e.g. [[tests-format#has_text_model|has_text]] jumps straight to that `$def`). +## 1. Choose by the property, then the output type -## 1. Choose by output type +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | -The single most useful decision table. Pick the row that matches the file format the workflow emits; default to the recommended assertion family. +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. -| Output type | Default assertion family | Why | Fallback | -|---|---|---|---| -| **Plain text reports / logs** (FastQC summary, MultiQC text section) | [[tests-format#has_text_model|has_text]] (substring on a known stable token) + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Substring is robust across versions; line count catches truncation | [[tests-format#has_text_matching_model|has_text_matching]] (regex) when token text changes per run | -| **HTML reports** (MultiQC HTML, custom dashboards) | [[tests-format#has_text_model|has_text]] against stable section names | HTML embeds timestamps and asset hashes; byte-diff is hopeless | [[tests-format#has_size_model|has_size]] + large `delta` as last resort | -| **Tabular** (TSV, CSV, BED-like) | [[tests-format#has_n_columns_model|has_n_columns]] + [[tests-format#has_text_model|has_text]] for headers + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Schema-shape + content probes; tolerant to row reordering | [[tests-format#has_line_matching_model|has_line_matching]] for a known canonical row | -| **VCF** | `compare: diff` with `lines_diff: 6` | The `lines_diff: 6` constant matches the typical VCF header preamble that embeds `##fileDate=` and `##source=` | [[tests-format#has_text_matching_model|has_text_matching]] for known variants | -| **BAM** | [[tests-format#has_size_model|has_size]] + [[tests-format#has_archive_member_model|has_archive_member]] (BAM is a gzipped block format) | Headers contain `@PG` lines with command lines and Galaxy job IDs; never byte-diff | Convert to SAM in a downstream step and assert on text | -| **FASTA** (deterministic — assemblies, consensus) | `file:` exact comparison or [[tests-format#has_text_model|has_text]] for known sequence | Output is byte-stable when the upstream tool is deterministic | [[tests-format#has_n_lines_model|has_n_lines]] if line wrapping varies | -| **FASTA** (non-deterministic — RepeatModeler libraries) | `compare: sim_size` with large `delta:` | Family content varies run-to-run | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | -| **FASTQ** (rare as workflow output) | [[tests-format#has_n_lines_model|has_n_lines]] (must be multiple of 4) | Quality scores are read-id-dependent | `compare: sim_size` | -| **JSON** (deterministic — config dumps, params) | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] / [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] | Surgical: assert on the property you care about, ignore the rest | `compare: diff` if the JSON is fully canonical | -| **JSON** (stochastic — HyPhy stats, MCMC results) | `has_text: text: "{"` (existence-only) | Embedded floats break any structural assertion; see [[iwc-shortcuts-anti-patterns]] §1 | [[tests-format#has_h5_keys_model|has_h5_keys]]-equivalent for top-level structural keys | -| **HDF5 / AnnData** | [[tests-format#has_h5_keys_model|has_h5_keys]] + [[tests-format#has_h5_attribute_model|has_h5_attribute]] for known structure | Asserts shape, not values — values are float-noisy | [[tests-format#has_size_model|has_size]] with `delta:` | -| **XML** | [[tests-format#is_valid_xml_model|is_valid_xml]] + [[tests-format#has_element_with_path_model|has_element_with_path]] + `element_text_is`/`element_text_matches` | Surgical structural probes; byte-diff fails on whitespace/attribute-order | [[tests-format#has_n_elements_with_path_model|has_n_elements_with_path]] for cardinality | -| **PNG / image plots** | [[tests-format#has_image_width_model|has_image_width]] + [[tests-format#has_image_height_model|has_image_height]] + [[tests-format#has_size_model|has_size]] with `delta:` (5–10%) | Catches "rendered something with the right dimensions"; corpus norm | [[tests-format#has_image_center_of_mass_model|has_image_center_of_mass]] if mask/segmentation outputs are deterministic | -| **TIFF / multipage images** | [[tests-format#has_image_frames_model|has_image_frames]] + [[tests-format#has_image_channels_model|has_image_channels]] + [[tests-format#has_size_model|has_size]] with `delta:` | Same logic, channel/frame structure matters | — | -| **Archives** (zip, tar.gz) | `has_archive_member: path: "regex"` with nested `asserts:` | Asserts on a specific member; archive timestamps never byte-stable | [[tests-format#has_size_model|has_size]] with `delta:` | -| **GFF / GTF** | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` + [[tests-format#has_text_model|has_text]] for known feature | Tools reorder freely; hard to byte-diff | `has_n_columns: 9` (GFF spec) | -| **Cool / HiC matrices** | `compare: sim_size` with multi-MB `delta:` | Binary, run-to-run variance | [[tests-format#has_archive_member_model|has_archive_member]] for HDF5 component | +## 2. Compare operators and expected artifacts -When in doubt: start with [[tests-format#has_size_model|has_size]] + `delta_frac: 0.1`. It catches the catastrophic failure mode (empty / 10x bigger output). Then add a content probe. +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. -### 1a. If the assertion is too weak, revisit the workflow output +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. -Assertion choice sometimes reveals a workflow-design problem. If the only available output can support only a size check or image-dimension smoke test, check whether the workflow should expose a stronger checkpoint before settling for the weak assertion. +## 3. Set tolerances from the failure boundary -- Scanpy's plot outputs use size and image-dimension assertions, but the same workflow also exposes AnnData HDF5 checkpoints and a cluster-count table (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end uses coarse size bands for coverage/mapped-read outputs, but expression/count tables get stronger regex and exact-line checks (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. -Rule: if a translated workflow exposes only weakly assertable final reports, consult [[galaxy-workflow-testability-design]] and consider promoting a table/text/HDF5 checkpoint before writing the final assertions. +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. -## 2. The `compare:` operators +## 4. Text assertions and repeated checks -Top-level on a `file:` output assertion. Vocabulary, in decreasing strictness: - -- **`diff`** (default). Byte-for-byte equality with optional `lines_diff:` tolerance for a fixed number of header lines. Use only when the upstream tool is deterministic on fixed inputs *and* the output has no embedded timestamps, command lines, version banners, or hash-ordered Python-dict-style keys. -- **`re_match` / `re_match_multiline`**. Each line of the expected fixture is a regex that must match the corresponding output line. Useful when a few fields per row are timestamped but the rest is canonical. Rare in the corpus. -- **`contains`**. The expected fixture is a substring of the output. Cheap; weak. Prefer `asserts: has_text` for new code unless you genuinely have a multi-line block to assert as a whole. -- **`sim_size`**. Output file size matches the fixture's size within `delta:` (bytes) or `delta_frac:` (fraction). Use when the output is necessarily non-deterministic but its rough size is reproducible (RepeatModeler libraries, HiC matrices, Bayesian sampler outputs). - -Picking `lines_diff:`: count the mutable header lines in the output format. VCF: ~6 (`##fileformat`, `##fileDate`, `##source`, `##reference`, contig/info lines vary). SAM/text headers: count `@HD`/`@PG`/`@CO` lines. Set `lines_diff:` to the count exactly — looser values mask real diffs. - -## 3. Tolerance picking - -**`delta:`** is bytes (for [[tests-format#has_size_model|has_size]] and `compare: sim_size`) or absolute count (for [[tests-format#has_n_lines_model|has_n_lines]], [[tests-format#has_n_columns_model|has_n_columns]], [[tests-format#has_image_width_model|has_image_width]], etc.). Suffix multipliers documented in the schema — `1K`, `1M`, `1G` work. - -**`delta_frac:`** is a fraction (0.1 = 10%). Use when expected size scales with input volume. Three IWC tests use it (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`); the rest use absolute `delta:`. - -**Picking magnitudes** (from corpus survey in [[iwc-shortcuts-anti-patterns]] §2): - -- Image dimensions: `delta: 25–30` pixels (5% of typical matplotlib defaults). -- Image file size: `delta: 5K–60K` (5–10% of file size). -- Small text reports: `delta: 1K–10K`. -- HTML reports: `delta: 25K–100K`. -- BAM files: `delta: 1M–10M`. -- RepeatModeler / Bayesian sampler outputs: `delta: 30K–90M` (extreme, but justified by the underlying nondeterminism). - -**Heuristic for new outputs:** `delta_frac: 0.1` is a defensible default. Tighten if the output proves more deterministic than expected. - -## 4. Text family — [[tests-format#has_text_model|has_text]] vs [[tests-format#has_text_matching_model|has_text_matching]] vs [[tests-format#has_line_model|has_line]] vs [[tests-format#has_line_matching_model|has_line_matching]] vs [[tests-format#has_n_lines_model|has_n_lines]] - -All five are common; choose by what you're verifying. - -- **[[tests-format#has_text_model|has_text]]** — output contains the substring `text:`. Anywhere in the output, any number of times. Add `n:` / `min:` / `max:` to constrain occurrence count. Add `delta:` to allow slack on the count. -- **[[tests-format#has_text_matching_model|has_text_matching]]** — output matches the regex `expression:`. Use sparingly; prefer literal [[tests-format#has_text_model|has_text]] when you can. -- **[[tests-format#has_line_model|has_line]]** — output has at least one *line* matching `line:` exactly. Use when line boundaries matter (e.g. asserting on a specific row in a table). -- **[[tests-format#has_line_matching_model|has_line_matching]]** — same but with regex. -- **[[tests-format#has_n_lines_model|has_n_lines]]** — assert the line count is `n:` ± `delta:`. - -A common combination in IWC: `has_n_lines: n: 100, delta: 5` + `has_text: text: "expected_token"` — line-count sanity-check plus a content marker. This catches both truncation and content drift in one assertion pair. - -`negate: true` is supported on every assertion. Used for the "this output should NOT contain X" case. - -## 5. Collection output assertions (`element_tests:`) - -For Galaxy collection outputs, the test format keys element assertions by element identifier: +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. ```yaml -my_collection_output: - element_tests: - sample_1: +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: asserts: - has_text: - text: "expected" - sample_2: - file: test-data/expected_sample_2.txt -``` - -Optional `attributes:` at the collection level can assert on the produced collection shape: - -```yaml -my_collection_output: - attributes: {collection_type: list:list} - element_tests: - ... + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 ``` -Nested collections: outer `element_tests:` keyed by outer identifier; inner uses `elements:` (note plural, no `_tests` suffix on the inner). See [[iwc-test-data-conventions]] §2f for the live example. - -For a list-of-files where every element should pass the same minimal check, the existence-probe pattern (`has_text: "{"` for JSON; `has_size: min: 100` for any non-empty binary) is widely used and accepted in IWC. - -## 6. The validate-against-workflow inner loop - -A `-tests.yml` file can be **structurally invalid** in two distinct ways: - -1. **Schema-invalid** — wrong field names, wrong nesting, wrong types. Caught by the test-format JSON Schema. -2. **Workflow-incoherent** — schema-valid YAML, but the input/output labels don't match the actual workflow. Renaming an output in the `.ga` and forgetting to update its sibling `-tests.yml` produces this case. Planemo will surface it as an "output not found" error at test-runtime, but only after a full workflow run. - -The `@galaxy-tool-util/schema` npm package ships **two** validators that catch both cases statically — no Galaxy or Planemo invocation needed: +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. -- **`validateTestsFile(yaml)`** — runs the file against `tests.schema.json` (AJV). Reports schema violations with paths. -- **`checkTestsAgainstWorkflow(workflow, tests)`** — cross-checks a `.ga` / format2 workflow against a tests file: missing input labels, missing output labels, type incompatibilities (e.g. test supplies a `File` for a parameter typed `int`). +## 5. Collection and structured-output examples -Both are pure-JS, take milliseconds, and have no Galaxy dependency. Wire them into the inner authoring loop: +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" ``` -edit -tests.yml - → validateTestsFile() # schema gate - → checkTestsAgainstWorkflow(.ga, tests) # coherence gate - → planemo workflow_test_on_invocation # assertion gate (no full re-run) - → planemo test # full integration (slow) -``` - -The first two gates short-circuit cheap mistakes before a slow planemo run. They are the static-validation equivalent of `gxwf` for tests, and the `implement-galaxy-workflow-test` mold should reference them as its primary inner-loop tooling. Source: galaxy-tool-util-ts package, `src/test-format/index.ts` exports. - -## 7. Authoring loop — generation, then refinement - -Reviewer convention is to **generate** the initial `-tests.yml` rather than hand-write it. Two planemo subcommands cover this: - -- **`planemo workflow_test_init --from_invocation `** ([[planemo-workflow_test_init]]) — given a successful Galaxy invocation ID, emit a `-tests.yml` with a `job:` block that captures all inputs (with SHA-1 hashes) and an `outputs:` block with `file:` references to the actual outputs (downloaded into `test-data/`). Hand-tighten the assertions afterward. -- **`planemo workflow_test_on_invocation `** ([[planemo-workflow_test_on_invocation]]) — re-evaluate an edited `-tests.yml` against a saved invocation without re-running the workflow. The fast inner loop for assertion iteration; complements the static gates in §6. - -Together these cut the assertion-iteration cost dramatically. An agent should: - -1. Run the workflow once on usegalaxy.* (or local) to get a known-good invocation. -2. `--from_invocation` to bootstrap the test file. -3. Replace the autogenerated `file:` exact-comparison assertions with assertion-family-appropriate alternatives per §1. -4. [[planemo-workflow_test_on_invocation]] after each edit; full [[planemo-test]] at the end. - -## 8. What the schema gives you for free - -When the test-format schema lands as a Foundry-rendered note, the agent can consult any assertion's `$def` directly for: parameter types, defaults, required fields, the `that` discriminator constant, and the original Python docstring (carried through as `description`). This note does **not** restate that vocabulary — it complements it with the corpus-grounded *which-and-when*. - -What's still missing from the schema and worth keeping in research notes: - -- This decision table (§1) — output-type → assertion family. -- Tolerance magnitudes (§3) — corpus-derived defaults. -- The `validateTestsFile` / `checkTestsAgainstWorkflow` integration story (§6). -- Anti-pattern flags — see [[iwc-shortcuts-anti-patterns]]. -## 9. Common combinations (recipes) +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. -Six recipes worth memorizing. +## 6. Validate labels and schema before execution -**Stable text report (FastQC summary, simple stats).** -```yaml -my_report: - asserts: - has_n_lines: { n: 12, delta: 2 } - has_text: { text: "Total Sequences" } -``` - -**MultiQC HTML report.** -```yaml -multiqc_report: - asserts: - has_text: { text: "Filtered Reads" } - has_text: { text: "FastQC" } -``` +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: -**VCF (pinned tool, fixed reference).** -```yaml -called_variants: - file: test-data/expected.vcf - compare: diff - lines_diff: 6 +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga ``` -**Stochastic JSON (HyPhy-style).** -```yaml -hyphy_meme: - element_tests: - geneA: { asserts: { has_text: { text: "{" } } } - geneB: { asserts: { has_text: { text: "{" } } } -``` +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. -**Matplotlib plot.** -```yaml -umap_plot: - asserts: - has_size: { size: 68416, delta: 6000 } - has_image_width: { width: 601, delta: 30 } - has_image_height: { height: 429, delta: 25 } -``` +## 7. Create and refine the test -**AnnData (HDF5).** -```yaml -clustered_anndata: - asserts: - has_h5_keys: { keys: "obs/louvain" } - has_h5_keys: { keys: "var/highly_variable" } - has_h5_keys: { keys: "uns/rank_genes_groups" } - has_size: { size: 12000000, delta: 1500000 } -``` +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. -## 10. Cross-references +## 8. Source of the exact syntax -- [[galaxy-workflow-testability-design]] — decide which workflow outputs and checkpoints to expose before choosing assertions. -- [[iwc-test-data-conventions]] — input-side conventions (job inputs, collection shapes, `hashes:`, CVMFS). -- [[iwc-shortcuts-anti-patterns]] — accepted-vs-smell catalog and corpus prevalence; this note's mirror image. -- Test-format schema (`@galaxy-tool-util/schema` npm package) — authoritative vocabulary; will be vendored into a Foundry-rendered schema note. See `content/meta/casting.md` for the casting story. -- Planemo test-format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- [[galaxy-xsd]] — Galaxy XSD assertion source of truth, vendored from upstream. -- Tightening of the schema and Pydantic source: [galaxyproject/galaxy#22566](https://github.com/galaxyproject/galaxy/pull/22566). -- TS schema sync into npm: [jmchilton/galaxy-tool-util-ts#75](https://github.com/jmchilton/galaxy-tool-util-ts/pull/75). +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-workflow-test-architecture.md b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-workflow-test-architecture.md index 59b3c25..72583a5 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-workflow-test-architecture.md +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/notes/planemo-workflow-test-architecture.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-11 -revision: 3 +revised: 2026-10-26 +revision: 4 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[galaxy-tool-job-failure-reference]]" @@ -23,6 +23,8 @@ sources: - "~/projects/repositories/planemo/planemo/galaxy/activity.py" - "~/projects/repositories/planemo/planemo/galaxy/invocations" - "~/projects/repositories/planemo/planemo/galaxy/config.py" + - "~/projects/repositories/planemo/planemo/galaxy/serve.py" + - "~/projects/repositories/planemo/planemo/engine/galaxy.py" summary: "Reference for Planemo workflow test/run architecture, Galaxy modes, API polling, and noisy failure boundaries." --- @@ -94,6 +96,20 @@ Important operations: For workflows, Planemo invokes Galaxy using input labels/names. Stable generated labels are therefore important for both test execution and debugging. +### Workflow Import Failures + +Planemo imports the workflow while it brings Galaxy up for the run (`config.install_workflows()`, called from `_serve()` in `planemo/galaxy/serve.py` for managed Galaxy and from `ExternalGalaxyEngine.ensure_runnables_served` in `planemo/engine/galaxy.py` for external Galaxy, as of planemo 0.75.47). For gxformat2 it converts the workflow to a native dict client-side first, so Galaxy's own gxformat2 conversion never runs. That is outside the per-test `execute()` wrapper that turns exceptions into an `ErrorRunResponse`. When Galaxy rejects the import request, BioBlend raises `ConnectionError`, the test command exits non-zero, and Planemo writes neither the `--test_output_json` report nor the HTML report. No workflow, history, or invocation id exists. + +The evidence then lives in three places: + +| Evidence | What it holds | +|---|---| +| Upload response | HTTP status and body. A 400 usually names the failing field or validation rule. A 500 body rarely explains the cause. | +| Galaxy server log | The traceback for the failing request, the only place a 500's cause appears. Managed Galaxy streams its log into the Planemo terminal output and keeps it on disk. External Galaxy keeps it on the server. | +| Planemo terminal log | The BioBlend exception and the point in the run where it was raised. | + +Re-importing the same converted dict as the same user on the same server reproduces the failure without the rest of the Planemo run, which makes it the cheap loop for isolating a trigger. + Workflow testability design guidance lives in [[galaxy-workflow-testability-design]]. This architecture note only records why Planemo makes labels and workflow-level outputs operationally important. ## Structured Artifacts @@ -121,6 +137,7 @@ Planemo transforms and summarizes Galaxy failure information. These are likely i - Polling may summarize “at least one job is in error” while detailed job evidence is printed elsewhere or stored separately. - Missing outputs become assertion/output problems, which can conflate label mismatch, workflow output omission, optional output absence, failed invocation, or output download issue. - Output download failures may be logged but not always propagated as the most specific failure. +- A rejected workflow import aborts the run before any structured report is written, so terminal and server logs are the only record. - External Galaxy without an admin key may defer missing-tool problems until runtime. - `allow_tool_state_corrections=True` can let Galaxy adjust tool state during invocation, which is useful but can mask definition drift. - `--no_wait` is not suitable for debug conclusions because invocation creation can succeed before jobs fail. @@ -135,7 +152,7 @@ For [[run-workflow-test]]: For [[debug-galaxy-workflow-output]]: -- Start from structured Planemo output. +- Start from structured Planemo output. When there is none because the workflow import failed, start from the upload response and the Galaxy server log. - For missing-output failures, check label drift and omitted workflow-level outputs before assuming tool failure. - If the failure is job-level, inspect Galaxy job APIs described in [[galaxy-tool-job-failure-reference]]. - If the failure is invocation-level, inspect Galaxy invocation APIs described in [[galaxy-workflow-invocation-failure-reference]]. diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/galaxy-workflow-test-plan.schema.json b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/galaxy-workflow-test-plan.schema.json index 1539281..82e4529 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/galaxy-workflow-test-plan.schema.json +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/galaxy-workflow-test-plan.schema.json @@ -358,7 +358,7 @@ "properties": { "family": { "type": "string", - "description": "tests-format assertion family or compare operator this intent maps to, by name, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys, file, compare:diff, compare:sim_size. Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." + "description": "tests-format assertion family, compare operator, or output-attribute field this intent maps to, by name. Assertion families go inside the output's `asserts:`, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys. Compare operators and output-attribute fields sit beside `asserts:`: file, compare:diff, compare:sim_size, element_count (collection outputs), count (nested sub-collections), ftype, checksum, metadata. Do not use collection_type (state it in ExpectedOutput.collection_shape) or element_tests (a non-null element_identifier implies it). Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." }, "intent": { "type": "string", @@ -390,15 +390,16 @@ "string", "null" ], - "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions." + "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions. Address an element of a nested collection by joining identifiers outermost first with `/`, e.g. WT_REP1/forward in a list:paired." }, "evidence": { "type": "string", "enum": [ "test-evidence", + "fixture-derived", "intent" ], - "description": "Whether this assertion was translated from upstream test evidence or synthesized from intent." + "description": "Basis of the expected value: test-evidence means translated from an upstream assertion, checksum, or expected file; fixture-derived means measured from upstream test fixture bytes where no upstream assertion exists (e.g. snapshot-only nf-tests); intent means synthesized from workflow intent, wrapper XML, or step state with no upstream anchor." }, "confidence": { "type": "string", diff --git a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/tests-format.schema.json b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/tests-format.schema.json index 8de90e7..99d551f 100644 --- a/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/tests-format.schema.json +++ b/plugins/foundry-skills/skills/implement-galaxy-workflow-test/references/schemas/tests-format.schema.json @@ -1110,6 +1110,18 @@ "default": "Collection", "title": "Class" }, + "count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Count" + }, "element_tests": { "anyOf": [ { @@ -1171,6 +1183,30 @@ ], "default": null, "title": "Elements" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Maximum Count" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum Count" } }, "title": "TestCollectionCollectionElementAssertions", @@ -1366,6 +1402,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestCollectionDatasetElementAssertions", @@ -1679,6 +1728,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestDataOutputAssertions", diff --git a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/SKILL.md b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/SKILL.md index c6c25cf..995e6ec 100644 --- a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/SKILL.md +++ b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/SKILL.md @@ -14,7 +14,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs - Read artifact `summary-galaxy-workflow`. Schema: summary-galaxy-workflow. Produced by `summarize-galaxy-workflow`. Structured summary of the existing workflow from summarize-galaxy-workflow; the anchor set the change-set addresses (step labels, input labels, output names, tool_state keys). -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -48,7 +48,7 @@ The harness owns the interaction style (interactive session, saved transcript, o Emit Markdown: an ordered list of edit intents. Each entry carries: -- **kind** — one of: `add-step`, `replace-tool`, `remove-step`, `change-parameter`, `add-input`, `remove-input`, `add-output` / `expose-output`, `rewire`, `relabel` / `annotate`. +- **kind** — one of: `add-step`, `replace-tool`, `remove-step`, `change-parameter`, `add-input`, `remove-input`, `add-output` / `expose-output`, `remove-output`, `rewire`, `relabel` / `annotate`. - **anchor** — the existing step label, input label, output name, or `tool_state` key the edit targets, quoted from the summary; or `new` for an added element. - **intent** — what the user wants, in their words plus the concrete target value where they gave one (e.g. a parameter's new value). - **acceptance** — how a reviewer or a test would confirm the edit landed (a new output is present, a parameter reads the new value, a step feeds its intended consumer). diff --git a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_provenance.json b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_provenance.json index 8e919f2..9ecb969 100644 --- a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_provenance.json +++ b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "interview-to-galaxy-workflow-changeset", "path": "content/molds/interview-to-galaxy-workflow-changeset/index.md", - "revision": 3, - "content_hash": "9e2b8567ee121020d44ac03ff4e119393e799c6cb35a0f21b9d2ce1d4a21dce1", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 4, + "content_hash": "4da72dca9397dd3e2590a6c83245febc39f6b0137a7aa5c51676c908f26a16c7", + "commit": "94ce4ea20107680b467d03e8889b63689774e325" }, - "cast_at": "2026-09-13T20:58:28.260Z", + "cast_at": "2026-11-10T10:05:12.638Z", "refs": [ { "kind": "research", @@ -21,8 +21,8 @@ "evidence": "hypothesis", "purpose": "Record a requested change the workflow cannot support as described as an open entry, so an unsupportable request survives as a tracked obligation rather than a fabricated anchor or a silent drop.", "verification": "Promote after a worked run shows entries this Mold appends are consumed by apply-galaxy-workflow-changeset or a reviewer without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -85,7 +85,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_verify.json b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_verify.json index 8c0e832..c089863 100644 --- a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_verify.json +++ b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-galaxy-workflow", + "--schema-sha256", + "55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/interview-to-galaxy-workflow-changeset/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/SKILL.md b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/SKILL.md index f8fdd52..e23d979 100644 --- a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/SKILL.md +++ b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/SKILL.md @@ -16,7 +16,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `starting-galaxy-workflow`. Produced by `summarize-galaxy-workflow`. The normalized concrete gxformat2 workflow to mature, normally emitted by summarize-galaxy-workflow; this is the only required input. - Read artifact `summary-galaxy-workflow`. Optional; absence is allowed and must be reported honestly. Schema: summary-galaxy-workflow. Produced by `summarize-galaxy-workflow`. Optional structured summary from summarize-galaxy-workflow; derive the needed facts from the workflow when this artifact is absent. - Read artifact `galaxy-workflow-test`. Optional; absence is allowed and must be reported honestly. Produced by `implement-galaxy-workflow-test`, `mature-galaxy-workflow-for-iwc`. Optional existing workflow test; preserve its assertions and update only keys that must follow supported workflow-label changes. -- Read artifact `open-requirements-ledger`. Optional; absence is allowed and must be reported honestly. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Optional carried obligations ledger open-requirements-ledger; initialize an empty ledger when it is absent and append unresolved maturation decisions. +- Read artifact `open-requirements-ledger`. Optional; absence is allowed and must be reported honestly. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Optional carried obligations ledger open-requirements-ledger; initialize an empty ledger when it is absent and append unresolved maturation decisions. - Read artifact `iwc-publication-context`. Optional; absence is allowed and must be reported honestly. Optional user or harness context for intended use, repository slug, creators, ORCIDs, citations, license, release, and durable test-data URLs. - Read artifact `iwc-workflow-readme`. Optional; absence is allowed and must be reported honestly. Produced by `mature-galaxy-workflow-for-iwc`. Optional existing workflow README to update while preserving accurate material that remains relevant after maturation. - Read artifact `iwc-workflow-changelog`. Optional; absence is allowed and must be reported honestly. Produced by `mature-galaxy-workflow-for-iwc`. Optional existing workflow changelog to extend without rewriting or discarding prior release history. @@ -29,7 +29,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Write artifact `iwc-workflow-readme` as `README.md`. Format: `markdown`. Proposed IWC workflow README describing evidenced purpose, valid inputs, expected outputs, and relevant comparisons or resources. - Write artifact `iwc-workflow-changelog` as `CHANGELOG.md`. Format: `markdown`. Proposed IWC changelog preserving prior history and recording only the release and maturation changes supported by available evidence. - Write artifact `iwc-dockstore-metadata` as `.dockstore.yml`. Format: `yaml`. Proposed Dockstore metadata whose workflow and test paths and evidenced creators agree with the emitted submission files. -- Write artifact `iwc-maturation-report` as `iwc-maturation-report.md`. Format: `markdown`. Per-checklist status report with evidence, before-and-after changes, unresolved decisions, source pin, and explicit downstream validation disclaimer. +- Write artifact `iwc-maturation-report` as `iwc-maturation-report.md`. Format: `markdown`. Per-checklist status report with evidence, before-and-after changes, unresolved decisions, source pin, run evidence kept out of the README, and explicit downstream validation disclaimer. - Write artifact `open-requirements-ledger` as `open-requirements.ledger.yml`. Format: `yaml`. Carried or initialized obligations ledger with unresolved IWC maturation decisions appended and all prior entries preserved. ## Required Tools @@ -95,13 +95,14 @@ When no test exists, emit no `galaxy-workflow-test` artifact. Record `needs-user #### 5. Prepare ordinary IWC companions -- Create or update a README from evidenced workflow purpose, valid inputs, and expected outputs. Preserve useful supplied material. State unknown comparisons or tutorial links as unresolved; do not invent them. -- Create or update the changelog, preserving existing history and adding only an evidence-backed entry. If the release cannot be chosen safely, make that a ledger item rather than fabricating a version history. +- Create or update a README from evidenced workflow purpose, valid inputs, and expected outputs, written for a user of the workflow as submitted. Preserve useful supplied material other than run evidence (below). State unknown comparisons or tutorial links as unresolved; do not invent them. Add fixed settings, current limitations, or a short description of the test data when they help that user. When the workflow evidently departs from its source, state the choice and its reason in a sentence or two. +- Keep run evidence out of the README: invocation ids, test or validation outcomes, benchmark tables, failed or abandoned tools and versions, environment-only failures, validator caveats, reviewer notes, and the story of how the run reached its result. One sentence stating an evidenced accuracy claim may stay. Move that material from a supplied README to the maturation report, keeping in the README any current limitation it contains. +- Create or update the changelog, preserving existing history and adding only an evidence-backed entry. An entry says what the release contains, not how it was reached. If the release cannot be chosen safely, make that a ledger item rather than fabricating a version history. - Create or update `.dockstore.yml` and align its paths with the emitted filenames. Preserve existing metadata and copy creator information only from workflow metadata or supplied context. Missing authorship is unresolved, not a license to guess. #### 6. Report and hand off -Write `iwc-maturation-report.md` with the pinned IWC source revisions from the bundled references, supplied-input inventory, per-item status and evidence, concise before/after descriptions, unresolved ledger ids, and the exact emitted files. End with an explicit statement that this skill did not run terminal validation or workflow tests and did not create or update a GitHub pull request. +Write `iwc-maturation-report.md` with the pinned IWC source revisions from the bundled references, supplied-input inventory, per-item status and evidence, concise before/after descriptions, unresolved ledger ids, the exact emitted files, and the run evidence kept out of the README, ready to copy into a pull request description. End with an explicit statement that this skill did not run terminal validation or workflow tests and did not create or update a GitHub pull request. The downstream path owns validate-galaxy-workflow, run-workflow-test, repair/retest, and publication automation. diff --git a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_provenance.json b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_provenance.json index b2801b7..f770b25 100644 --- a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_provenance.json +++ b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "mature-galaxy-workflow-for-iwc", "path": "content/molds/mature-galaxy-workflow-for-iwc/index.md", - "revision": 1, - "content_hash": "38d9fed5e83bcb02cdc044ac1a0858d2b3f5ce42af5905bea2b920451bacca6b", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 2, + "content_hash": "f7dd7ae7e1f5fa2227344cbf16efedb5c2b63ea2455231687641cf6d8ce70d55", + "commit": "a4988f41d403f6134dc7b944b8b05f3ade4b9e00" }, - "cast_at": "2026-09-13T20:58:28.307Z", + "cast_at": "2026-11-11T17:11:19.085Z", "refs": [ { "kind": "prompt", @@ -38,8 +38,8 @@ "evidence": "corpus-observed", "purpose": "Keep workflow interface and test addressing aligned when a safe genericity or human-readable-label correction changes the workflow interface.", "trigger": "When maturation changes an input, promoted output, collection identifier, or workflow-output label referenced by a supplied test.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -53,8 +53,8 @@ "evidence": "corpus-observed", "purpose": "Apply current IWC conventions for test labels, durable remote fixtures, creator identifiers, release changes, and companion-file naming.", "trigger": "When a supplied test, test-data reference, creator record, or release decision must be checked or updated.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -68,8 +68,8 @@ "evidence": "hypothesis", "purpose": "Initialize, preserve, and append precise unresolved maturation obligations rather than guessing authorship, licensing, or scientific intent.", "verification": "Exercise the ambiguous-reference fixture and confirm the emitted ledger carries a focused unresolved entry without a speculative workflow edit.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -125,7 +125,7 @@ "id": "iwc-maturation-report", "kind": "markdown", "default_filename": "iwc-maturation-report.md", - "description": "Per-checklist status report with evidence, before-and-after changes, unresolved decisions, source pin, and explicit downstream validation disclaimer." + "description": "Per-checklist status report with evidence, before-and-after changes, unresolved decisions, source pin, run evidence kept out of the README, and explicit downstream validation disclaimer." }, { "id": "open-requirements-ledger", @@ -181,7 +181,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] }, { diff --git a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_verify.json b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_verify.json index 8c0e832..c089863 100644 --- a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_verify.json +++ b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-galaxy-workflow", + "--schema-sha256", + "55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/mature-galaxy-workflow-for-iwc/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_provenance.json b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_provenance.json index 9f5cb9e..ff1f651 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/nextflow-summary-to-cwl-data-flow/index.md", "revision": 1, "content_hash": "68a75d5f934610fd9bf95434c94c3965dc2c591ea33160e911a3662f1f290aaa", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "commit": "871b5e6deadfd56aeb5e2ba92eb8f6f50c8449ee" }, - "cast_at": "2026-09-15T23:16:23.499Z", + "cast_at": "2026-11-10T14:49:42.167Z", "refs": [ { "kind": "schema", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Read process, channel, operator, and fixture structure while drafting CWL-facing abstract data flow.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_verify.json b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_verify.json index 667ae3b..d508d96 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-data-flow/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_provenance.json b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_provenance.json index b81481b..0649f75 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/nextflow-summary-to-cwl-interface/index.md", "revision": 1, "content_hash": "3fc1ccd78b3828e77382b921fc547a4ea2f56d7123cd92cf0e1cfefd4561984f", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "commit": "871b5e6deadfd56aeb5e2ba92eb8f6f50c8449ee" }, - "cast_at": "2026-09-15T23:16:23.556Z", + "cast_at": "2026-11-10T14:49:42.194Z", "refs": [ { "kind": "schema", @@ -21,8 +21,8 @@ "evidence": "corpus-observed", "purpose": "Read source-level channel, parameter, process, and test-fixture evidence before choosing CWL Workflow inputs and outputs.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_verify.json b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_verify.json index 667ae3b..d508d96 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-cwl-interface/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/SKILL.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/SKILL.md index bc0e15a..cd43bd7 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/SKILL.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/SKILL.md @@ -16,7 +16,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `summary-nextflow`. Schema: summary-nextflow. Produced by `summarize-nextflow`. Structured Nextflow pipeline summary emitted by summarize-nextflow; the JSON the data-flow translation reads. - Read artifact `nextflow-galaxy-reference-data`. Produced by `nextflow-summary-to-galaxy-reference-data`. Reference-data shape brief from nextflow-summary-to-galaxy-reference-data that pins per-asset reference inputs and rebuild-on-absence behavior. - Read artifact `nextflow-galaxy-interface`. Produced by `nextflow-summary-to-galaxy-interface`. Preceding Galaxy interface brief from nextflow-summary-to-galaxy-interface that pins inputs, outputs, and labels. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -57,6 +57,8 @@ Read a Nextflow summary plus the preceding Galaxy interface brief and emit a rev The output is not gxformat2 and should not resolve exact Tool Shed tools. Nor does it rule them out. Recording that a step's tool does not exist, will not be found, or must be authored is the same adjudication made negative, and it belongs to advance-galaxy-draft-step, which owns routing. Say what each step needs a tool to *do*; let discovery report what exists. Justify step granularity from the source's own structure — subworkflow boundaries, the ratio of plumbing to scientific work. A guess about tool availability is not a justification. +When the brief names collection element identifiers, take them from the source's own output names and check them against the source test's rows. A per-row column such as `sample` can repeat, as when one sample has several runs, so key each element on something unique per dataset and say what makes it unique. A positional suffix (`_1`, `_2`) is a mate only when every row it numbers is paired. + ## Feedback Mode - Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_provenance.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_provenance.json index 7ba2542..80726f0 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "nextflow-summary-to-galaxy-data-flow", "path": "content/molds/nextflow-summary-to-galaxy-data-flow/index.md", - "revision": 7, - "content_hash": "1f98fd5cf9a54fd81e48db957f7451760f5e97629748f3d39fd66dc59b2ac765", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "revision": 8, + "content_hash": "f71cc2b4ceb115d7d93c3dd2e622cbfafb8b9f338f1dd450ee5773f3c0a0e247", + "commit": "61da115975d155a36face0e83d5c28f638da7fca" }, - "cast_at": "2026-09-15T23:16:23.786Z", + "cast_at": "2026-11-10T18:57:18.688Z", "cast_date": "2026-05-07", "cast_revision": 3, "cast_history": [ @@ -40,8 +40,8 @@ "evidence": "corpus-observed", "purpose": "Ground collection-shape choices in curated, corpus-observed operation and recipe patterns.", "trigger": "When selecting collection cleanup, reshape, identifier, or collection-tabular bridge patterns.", - "src_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", - "dst_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", "source": "deterministic" }, { @@ -55,8 +55,8 @@ "evidence": "corpus-observed", "purpose": "Ground conditional-branch and optional-step choices in curated, corpus-observed Galaxy when/pick_value patterns.", "trigger": "When data-flow translation needs optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", - "src_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", - "dst_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", "source": "deterministic" }, { @@ -70,8 +70,8 @@ "evidence": "corpus-observed", "purpose": "Ground genomic-interval operation choices in curated, corpus-observed Galaxy interval recipes.", "trigger": "When the workflow operates on genomic intervals (BED/GFF/VCF coordinate features) and data-flow translation needs overlap, merge, coverage, windowing, masking, or set-algebra steps.", - "src_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", - "dst_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", "source": "deterministic" }, { @@ -85,8 +85,8 @@ "evidence": "corpus-observed", "purpose": "Ground tabular bridge and table-operation choices in curated, corpus-observed operation patterns.", "trigger": "When data-flow translation needs filtering, joining, aggregation, pivoting, or tabular-collection bridges.", - "src_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", - "dst_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", "source": "deterministic" }, { @@ -100,8 +100,8 @@ "evidence": "hypothesis", "purpose": "Keep the data-flow brief separate from gxformat2 templating and concrete step implementation.", "verification": "Promote after two worked Galaxy translations preserve this Mold boundary without moving fields.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -130,8 +130,8 @@ "evidence": "corpus-observed", "purpose": "Decide between subworkflow `when:` and inline tool-step `when:` for each source conditional, and pick the right output fan-in primitive (`pick_value` vs twin-cascade) so the data-flow brief carries a coherent conditional disposition forward.", "trigger": "When the Nextflow summary's `workflow.conditionals[]` is non-empty, or when subworkflow boundaries in the source align with parameter-driven branches (step, aligner, wes, tools, skip_*, use_*).", - "src_hash": "e341284cdf3a2cdb7381c37aa50e409f99fde7c5dbcdefe64cf2ca15dd28f85e", - "dst_hash": "e341284cdf3a2cdb7381c37aa50e409f99fde7c5dbcdefe64cf2ca15dd28f85e", + "src_hash": "f38341565d0927321248b67a9d99ce9533cc89ef4e543cbf80378260e7025f7e", + "dst_hash": "f38341565d0927321248b67a9d99ce9533cc89ef4e543cbf80378260e7025f7e", "source": "deterministic" }, { @@ -188,8 +188,8 @@ "load": "upfront", "evidence": "corpus-observed", "purpose": "Translate Nextflow channel, tuple, and path shapes into Galaxy dataset and collection shapes.", - "src_hash": "32efd59cc86563b145819cda38f0692bdd3144aac10ae831dc6d8accd9dcda8d", - "dst_hash": "32efd59cc86563b145819cda38f0692bdd3144aac10ae831dc6d8accd9dcda8d", + "src_hash": "e8d9515c9ae7da29188e8744410d8459e7ee9acd48af550216076fcf7a02b73b", + "dst_hash": "e8d9515c9ae7da29188e8744410d8459e7ee9acd48af550216076fcf7a02b73b", "source": "deterministic" }, { @@ -218,8 +218,8 @@ "evidence": "hypothesis", "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's wiring and collection decisions settle, and append data-flow obligations the channel topology leaves open — an operator with no Galaxy collection recipe, a value channel with no Galaxy carrier, a join whose key Galaxy can't reconstruct.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -233,8 +233,8 @@ "evidence": "corpus-observed", "purpose": "Read process, channel, operator, and fixture structure while drafting Galaxy-facing abstract data flow.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], @@ -296,7 +296,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_verify.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_verify.json index 667ae3b..d508d96 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md index c7ce559..38a7704 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-08 -revised: 2026-05-08 -revision: 1 +revised: 2026-11-07 +revision: 2 related_notes: - "[[nextflow-to-galaxy-reference-data-mapping]]" - "[[nextflow-to-galaxy-channel-shape-mapping]]" @@ -19,7 +19,7 @@ related_molds: sources: - "https://github.com/galaxyproject/gxformat2" - "https://github.com/iwc-workflows" -summary: "Stub. Translate Nextflow conditionals into Galaxy `when:` (single-workflow v1). Subworkflow vs inline is an aesthetic call, not a rule." +summary: "Stub. Translate Nextflow conditionals into Galaxy `when:` (single workflow). Subworkflow vs inline is an aesthetic call, not a rule." --- # Nextflow conditional to Galaxy subworkflow / when @@ -28,9 +28,11 @@ Stub. Surfaced from sarek emulation (2026-05-08). Companion to [[nextflow-to-gal ## Posture -For v1 of the Nextflow-to-Galaxy translation Molds the output is a single Galaxy workflow per source pipeline, even when the source has substantial branching. IWC reviewers historically prefer sibling workflows for what looks like one pipeline with toggles, and we agree; but for the *translation step* a single artifact keeps the Mold pipeline deterministic, the harness simple, and the reviewer's mental model of "this draft maps 1:1 to the source" intact. Sibling-extraction is a polish pass a human or follow-up Mold runs *after* translation, not a decision the translation Mold makes. +The Nextflow-to-Galaxy translation Molds output a single Galaxy workflow per source pipeline, even when the source has substantial branching, and capture as much of the source's scope as conditionals allow. IWC workflows don't typically dispatch on different tools within a workflow, but that reflects the corpus, not a design constraint. When the source offers a tool choice, carry it in one workflow with conditionals, especially when porting a workflow that already dispatches this way. A second workflow just for a tool choice is not a goal. -The question this note addresses is: given that v1 is one Galaxy workflow, *how* should source conditionals translate? The two structural tools available are: +IWC reviewers have historically preferred sibling workflows for what looks like one pipeline with toggles. A human or follow-up Mold can still extract siblings after translation, but the translation Mold doesn't aim for that split. A single artifact also keeps the Mold pipeline deterministic, the harness simple, and the reviewer's mental model of "this draft maps 1:1 to the source" intact. + +The question this note addresses is: given one Galaxy workflow, *how* should source conditionals translate? The two structural tools available are: - **Inline `when:` on a tool step.** Step runs or doesn't, based on an ECMAScript 5.1 expression over workflow inputs. Lightweight; doesn't change the DAG shape. - **`when:` on a subworkflow step.** A bundle of steps runs or doesn't as a unit. Adds a subworkflow boundary and the wiring overhead that comes with it (typed inputs, typed outputs, explicit pass-through). When the parent's `when:` is false the inner steps inherit the skip via the engine's `when_values` propagation. @@ -63,7 +65,7 @@ These are the cases where the call is genuinely judgment, and an agent rendering - **Sarek `aligner` (bwa-mem | bwa-mem2 | dragmap | sentieon | parabricks).** Five-way. Source has `FASTQ_ALIGN_BWAMEM_MEM2_DRAGMAP_SENTIEON` as one subworkflow, plus a separate `FASTQ_PREPROCESS_PARABRICKS` because parabricks bundles align+sort+dedup. Mixed signal: the source leans subworkflow for four of the five, parabricks is its own beast. Probably means: subworkflow for the four-way alignment swap (matches source), a sibling `when:`-gated subworkflow step for parabricks (matches source). Inline `when:` would scatter five alignment tool choices through the main workflow body. - **Sarek `wes`.** Two-way boolean. Source scopes WES handling inside `PREPARE_INTERVALS` and a few variant-calling internals. Strong "subworkflow `when:`" signal — source already structured it that way, cascade is wide, interface is narrow (intervals BED in, intervals BED out). -- **Sarek `tools` selector (variant callers).** Source scopes each caller in its own subworkflow (`BAM_VARIANT_CALLING_HAPLOTYPECALLER`, `BAM_VARIANT_CALLING_FREEBAYES`, …). Each caller's outputs land in a downstream merge. Strong "subworkflow `when:` per caller" signal — but the wiring at the merge is delicate (each branch contributes a different VCF format), so the merge step itself becomes the awkward part. Maybe the right answer here is sibling Galaxy workflows after all, even at v1 — flag for the human reviewer. +- **Sarek `tools` selector (variant callers).** Source scopes each caller in its own subworkflow (`BAM_VARIANT_CALLING_HAPLOTYPECALLER`, `BAM_VARIANT_CALLING_FREEBAYES`, …). Each caller's outputs land in a downstream merge. Strong "subworkflow `when:` per caller" signal — but the wiring at the merge is delicate (each branch contributes a different VCF format), so the merge step itself becomes the awkward part. Flag that wiring for the human reviewer rather than splitting the workflow. ## Output fan-in: use `pick_value`, not a generic merge @@ -86,7 +88,7 @@ IWC uses the `when:` + `pick_value` pattern in production. Two confirmed example - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-artic-ivar-analysis/pe-wgs-ivar-analysis.gxwf.yml` — two parallel tool steps both gated by `when: $(inputs.when)` from a shared boolean, merged downstream with a `pick_value` step. - `comparative_genomics/hyphy/capheine-core-and-compare.gxwf.yml` — multiple HyPhy analyses guarded by optional foreground/background booleans, merged downstream. -So `when:` + `pick_value` for branch selection is corpus-observed and IWC-acceptable. **What is *not* observed in IWC**: dynamic tool selection via `when:` (e.g. "run bwa-mem or parabricks based on a `tools` selector input"). Every IWC workflow commits to a fixed tool stack at authoring time. A Nextflow-to-Galaxy translation that surfaces aligner choice as a runtime `when:`-gated swap would be a novel pattern in IWC. The mechanism is well-supported by the engine; the precedent is missing. Worth flagging in any draft brief that proposes it: "this works mechanically, but IWC has no exemplar for tool-selection-via-when, and a reviewer may push back toward sibling workflows." +So `when:` + `pick_value` for branch selection is corpus-observed and IWC-acceptable. **What IWC doesn't typically do**: dynamic tool selection via `when:` (e.g. "run bwa-mem or parabricks based on a `tools` selector input"). IWC workflows typically commit to a fixed tool stack at authoring time, so a Nextflow-to-Galaxy translation that keeps aligner choice as a runtime `when:`-gated swap has little corpus precedent. The engine supports the mechanism, and the missing precedent is not a reason to cut the choice or split the workflow. A draft brief can record it as evidence for the reviewer. ## Confirmed engine behavior @@ -103,7 +105,7 @@ The engine's design here is more deliberate and more tolerant than the working m ## When this note's posture stops working -- **A pipeline whose conditionals don't cleanly bucket either way.** If the heuristics keep producing "mixed signals" across most conditionals, the source pipeline is probably one where sibling Galaxy workflows really are the right answer at v1, and the translation Mold should surface that as an Open Question on the brief rather than force one workflow. +- **A pipeline whose conditionals don't cleanly bucket either way.** If the heuristics keep producing "mixed signals" across most conditionals, the translation Mold should surface that as an Open Question on the brief. Consider sibling workflows only when one workflow can't carry the branches, not because of a tool choice alone. - **Galaxy invocation engine doesn't behave the way this note assumes.** Confirmed against source on 2026-05-08; the engine is robust here. Risk reopens only if a future Galaxy refactor changes `when:` semantics or the `pick_value` module is deprecated. - **A second Nextflow pipeline emulation surfaces a third pattern.** sarek covers tool-swap (`aligner`), boolean phase toggle (`wes`), and selector (`tools`). RNAseq might add quantifier choice (`salmon | star_salmon | hisat2`) — likely a subworkflow case but worth checking. Taxprofiler might add database-driven branching. New patterns sharpen the heuristics. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-to-galaxy-channel-shape-mapping.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-to-galaxy-channel-shape-mapping.md index f8219d8..caf83b8 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-to-galaxy-channel-shape-mapping.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/nextflow-to-galaxy-channel-shape-mapping.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-06 -revision: 2 +revised: 2026-11-10 +revision: 3 related_notes: - "[[nextflow-workflow-io-semantics]]" - "[[nextflow-params-to-galaxy-inputs]]" @@ -25,6 +25,10 @@ related_notes: - "[[relabel-via-rules-and-find-replace]]" - "[[reshape-relabel-remap-by-collection-axis]]" - "[[sync-collections-by-identifier]]" + - "[[harmonize-by-sortlist-from-identifiers]]" + - "[[derive-parameter-from-file]]" + - "[[conditional-route-between-alternative-outputs]]" + - "[[open-requirements-ledger]]" - "[[tabular-compute-new-column]]" - "[[tabular-concatenate-collection-to-table]]" - "[[tabular-cut-and-reorder-columns]]" @@ -48,19 +52,20 @@ related_molds: - "[[implement-galaxy-tool-step]]" sources: - "https://github.com/galaxyproject/foundry/issues/52" + - "https://github.com/galaxyproject/foundry/issues/757" summary: "Maps common Nextflow channel, tuple, and path shapes to Galaxy dataset and collection shapes." --- # Nextflow-to-Galaxy Channel Shape Mapping -This note maps source-level Nextflow channel shapes onto Galaxy data and dataset-collection shapes. Evidence quality is uneven: Galaxy collection semantics and IWC collection operations are well grounded; several Nextflow shapes are observed in the pinned fixtures; `fromFilePairs` and arbitrary deep tuple mapping remain low-confidence without direct fixture evidence. +This note maps source-level Nextflow channel shapes onto Galaxy data and dataset-collection shapes. The Confidence column marks how well each row is grounded. ## Shape Mapping | Nextflow shape | Galaxy shape | Confidence | Notes | |---|---|---|---| | `path(x)` or a single `Channel.fromPath(...).first()` | `File` | High | One dataset input or output. | -| `val(meta)` alone | No dataset shape | High | Treat as identifiers, labels, tags, sample-sheet metadata, or parameters. It is not a Galaxy dataset by itself. | +| `val(meta)` alone | No dataset shape | High | Treat as identifiers, labels, tags, sample-sheet metadata, or parameters. It is not a Galaxy dataset by itself. For a value computed per sample and read by a later process as a tool argument, see "Per-element parameters from `meta`" below. | | Repeated `tuple val(meta), path(file)` | `list` | High | One dataset per sample/key. Galaxy maps tools over list collections. | | One `tuple(meta, [R1, R2])` | `paired` | High | Use only when the workflow input is one paired sample or one tool consumes one pair. | | Repeated `tuple(meta, [R1, R2])` | `list:paired` | High | Common paired-end workflow input shape. | @@ -69,12 +74,12 @@ This note maps source-level Nextflow channel shapes onto Galaxy data and dataset | `tuple(meta, path(a), path(b))` | Parallel lists or per-step `File` inputs | Medium | Not automatically a `paired` collection. It is usually a keyed record of multiple tool inputs. | | Global file plus per-sample files | Broadcast `File` plus mapped `list` | Medium | If one input is global, connect it once and map the collection input. | | `collect()` or `toList()` | Collection reduction | High | Usually one downstream invocation over a collection or multiple-input value. | -| `collectFile(...)` | `File` | High | Nextflow creates one new file; Galaxy should model a tool output, not a collection operation. | +| `collectFile(...)` | `File` | High | Nextflow creates one new file. Galaxy should model a tool output, not a collection operation. | | `groupTuple()` by key | `list`, `list:paired`, or `list:list` | Medium | Depends on grouped payload and whether the grouping axis matters downstream. | | `transpose()` after grouping | Explicit reshape or subcollection mapping | Medium | May need flattening, Apply Rules, or identifier-preserving reshaping. | | `combine(..., by: key)` or `join(...)` | Multi-input collection map if identifiers match | Medium | Otherwise use explicit synchronization by identifiers/order. | | `branch { ... }` | Branch wiring or explicit filters | Medium | Shape is whatever each branch returns; per-element routing needs review. | -| `mix(...)` | Merge compatible streams/collections | Medium | Use direct wiring for reports/versions; use `__MERGE_COLLECTION__` or `__BUILD_LIST__` when materialized. | +| `mix(...)` | Merge compatible streams/collections | Medium | Use direct wiring for reports/versions, and `__MERGE_COLLECTION__` or `__BUILD_LIST__` when materialized. | | `multiMap { ... }` | Synchronized split into Galaxy inputs | Medium | Usually channel-only fan-out, but downstream rejoin depends on identifier discipline. | | `fromFilePairs` | `paired` or `list:paired` | Low | Conceptually clean, but not directly observed in the pinned fixtures. | | Arbitrary deep tuples | Parallel lists, `list:list`, or manual modeling | Low | Galaxy collection types are not arbitrary tuple records. Require per-tool review. | @@ -95,6 +100,38 @@ Use explicit Galaxy collection-operation or tabular steps when the translation c | Unbox a singleton collection | `__EXTRACT_DATASET__` | | Convert a collection of tabular outputs to one table | `collapse_dataset` or `collection_column_join` | +## Per-element parameters from `meta` + +nf-core pipelines often compute a value per sample, write it into `meta`, and read it as a tool argument in that sample's later processes. In nf-core/rnaseq's `strandedness: auto` path, `subworkflows/nf-core/fastq_qc_trim_filter_setstrandedness/main.nf` returns `meta + [strandedness: ...]`, and later modules read `meta.strandedness`. The collection operations above reshape data. This route keeps the data as it is and varies a tool parameter per element. + +Carry the value as a one-value dataset per element, in a collection with the same structure and order as the data collection. Map `param_value_from_file` over it and choose `param_type` by the target: `integer` or `float` for a number, `boolean` for a boolean, `text` for text. Connect the matching output (`integer_param`, `float_param`, `boolean_param` or `text_param`) to the target parameter of the step that maps over the data collection: + +```text +per-element values (list) ──> param_value_from_file (mapped) ──> integer_param ──┐ + ▼ +reads (list) ──────────────────────────────────────────> downstream tool (mapped, target parameter) +``` + +Galaxy runs one job per element and pairs the two collections by position, not by element identifier. A same-length value collection in another order runs without error and gives each element the value at its position, not its own. Produce the values with a step mapped over the data collection so they inherit its order. When they arrive as a separate collection, reorder them first: run `collection_element_identifiers` on the data collection, then `__SORTLIST__` with `sort_type: file` on the value collection, connecting the identifier file to `sort_type|sort_file` ([[harmonize-by-sortlist-from-identifiers]]). + +| Target | Evidence | Status | +|---|---|---| +| Integer tool parameter | `verification/workflows/per-element-parameter-from-file/`: values 1 and 3 on two elements, and positional pairing in its second case | Verified | +| Text tool parameter | Galaxy's `lib/galaxy_test/workflow/map_over_expression.gxwf.yml` and its `-tests.yml`: per-element values `A` and `B` reach a text input | Verified | +| Per-element `when` (boolean) | Galaxy's `lib/galaxy_test/api/test_workflows.py::test_run_workflow_conditional_step_map_over_expression_tool`: one of two elements is skipped | Verified | +| Parameter inside a tool conditional's case | IWC `epigenetics/consensus-peaks/consensus-peaks-atac-cutandrun`: mapped "convert min nb of reads to parameter" feeds `samtools_view`'s `mode\|subsample_config\|subsampling_mode\|target`, with values identical across elements | Wiring only | +| `select` tool parameter | The same fixture's `per-element-select-boolean` workflow: text values `C` and `D` choose `Cut1`'s `delimiter` per element | Verified | +| Boolean tool parameter | The same workflow: `true` and `false` set `wc_gnu`'s `include_header` per element | Verified | +| Float tool parameter | None | Not tested | +| A tool conditional's test parameter | [#661](https://github.com/galaxyproject/foundry/issues/661) reports a Galaxy 500 when one is connected. [#667](https://github.com/galaxyproject/foundry/issues/667) reports a workflow connecting one that runs on usegalaxy.org | Open | + +Use this route for every target except the Open row, and cover a Wiring only or Not tested target with a workflow test whose elements carry different values. rnaseq's strandedness reaches both kinds of target. featureCounts' `strand_specificity` is a `select` and takes this route. A `select` takes an option value, not the source's string, so translate first: featureCounts' options are `0`, `1` and `2`, where rnaseq writes `unstranded`, `forward` and `reverse`. [[map-workflow-enum-to-tool-parameter]] does this with `map_param_value`, which hasn't been tested mapped over elements. Salmon's `--libType` is `strandedness`, the test parameter of its `libtype` conditional, and takes the fallback. + +For a conditional's test parameter, or when a workflow test of this route fails, fall back in this order: + +1. For a small fixed set of values, run one branch per value. For each value, write a per-element boolean file with a mapped comparison step, read it with mapped `param_value_from_file` (`param_type: boolean`), gate that value's branch with `when`, and give the branch's step that value as a fixed parameter. Rejoin the branches with Galaxy's built-in `pick_value` (`pick_style: first_or_error`) mapped over the branch outputs, never `__MERGE_COLLECTION__`. A gated mapped step still emits a full-size collection with its skipped elements as null, so a merge returns one element per branch per sample. Galaxy's `test_run_workflow_conditional_step_map_over_expression_tool_pick_value` covers this mapped rejoin, and [[conditional-route-between-alternative-outputs]] covers `pick_value`'s settings. +2. Otherwise, expose one workflow-level parameter for all elements. Append a `kind: dropped` entry to the [[open-requirements-ledger]] whose `units` names the per-element parameter and whose `because` cites the Open row or the failing test. Expose the computing step's output as a workflow output so a user can run once, read the computed values, and re-run with the value they choose. + ## Evidence Corpus-observed Galaxy semantics: @@ -104,13 +141,13 @@ Corpus-observed Galaxy semantics: - [[galaxy-apply-rules-dsl]] explains identifier-derived collection reshaping. - [[iwc-transformations-survey]] records which collection transformations appear in IWC workflows. -Pinned fixture examples used by the research pass: +Nextflow fixture examples: - `workflow-fixtures/pipelines/nf-core__demo/subworkflows/local/utils_nfcore_demo_pipeline/main.nf` for `tuple(meta, reads)` paired-read handling. - `workflow-fixtures/pipelines/nf-core__rnaseq/main.nf` for single-end versus paired-end branching and reads normalization. - `workflow-fixtures/pipelines/nf-core__taxprofiler/main.nf` and `workflow-fixtures/pipelines/nf-core__taxprofiler/subworkflows/local/*/main.nf` for `groupTuple`, `transpose`, `mix`, `combine`, and `multiMap` patterns. - `workflow-fixtures/pipelines/nf-core__fetchngs/workflows/sra/main.nf` for `collectFile` and accession-driven data fetching. -- `$IWC_FORMAT2/virology/influenza-isolates-consensus-and-subtyping/influenza-consensus-and-subtyping.gxwf.yml` as a Galaxy `list:paired` exemplar. +- IWC `virology/influenza-isolates-consensus-and-subtyping/influenza-consensus-and-subtyping` as a Galaxy `list:paired` exemplar. ## Low-Confidence TODOs diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md index bd4ddc7..b00b1d5 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md @@ -13,9 +13,9 @@ tags: - topic/collection-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy collection transformation patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." related_notes: - "[[iwc-transformations-survey]]" - "[[iwc-conditionals-survey]]" @@ -25,10 +25,13 @@ related_patterns: - "[[reshape-relabel-remap-by-collection-axis]]" - "[[fan-in-bundle-consume-and-flatten]]" - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" - "[[sync-collections-by-identifier]]" - "[[harmonize-by-sortlist-from-identifiers]]" - "[[regex-relabel-via-tabular]]" - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" - "[[collection-swap-nesting-with-apply-rules]]" - "[[collection-split-identifier-via-rules]]" - "[[collection-build-list-paired-with-apply-rules]]" @@ -42,45 +45,46 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: collection patterns -This is the runtime-facing map for Galaxy collection transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. -## Cleanup +## Enter or leave a collection -- [[collection-cleanup-after-mapover-failure]] — use `__FILTER_EMPTY_DATASETS__` or `__FILTER_FAILED_DATASETS__` after map-over when empty or errored elements would break downstream steps. -- [[collection-unbox-singleton]] — use `__EXTRACT_DATASET__` with `which: first` when a known one-element collection must become a dataset. +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. -## Map-over Lifecycle Recipes +## Keep mapped siblings aligned -- [[manifest-to-mapped-collection-lifecycle]] — build collection elements from manifest/table rows, map a tool over them, then relabel or reshape outputs. -- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped outputs, sync siblings from surviving identifiers, then gate final reports. -- [[reshape-relabel-remap-by-collection-axis]] — correct the mapped axis after domain fan-out using Apply Rules and deterministic relabeling. -- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection consumer, then flatten or aggregate results. +First identify what changed. Membership, order, and labels are separate decisions. -## Identifiers +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. -- [[sync-collections-by-identifier]] — membership sync: extract identifiers from one collection and filter or relabel a sibling collection. -- [[harmonize-by-sortlist-from-identifiers]] — order sync: sort one sibling collection by another collection's identifier order. -- [[regex-relabel-via-tabular]] — label rewrite: derive new element identifiers in tabular form and apply them with `__RELABEL_FROM_FILE__`. -- [[relabel-via-rules-and-find-replace]] — relabel inside a structural reshape, currently grounded in the influenza fan-out pattern. +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. -## Structural Reshape +## Change the collection axes -- [[collection-flatten-after-fanout]] — collapse nested collection output to a flat list when the outer axis no longer matters. -- [[collection-build-named-bundle]] — assemble individual outputs into a named collection bundle for publishing or downstream fan-in. -- [[collection-swap-nesting-with-apply-rules]] — use Apply Rules to swap `list:list` axes. -- [[collection-split-identifier-via-rules]] — use Apply Rules regex columns to derive nested list identifiers from one identifier string. -- [[collection-build-list-paired-with-apply-rules]] — use Apply Rules to promote identifier columns into `list:paired` structure. +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. -## Bridges +## Follow a map-over lifecycle -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. ## See also diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md index 23fae29..2dfe890 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md @@ -12,9 +12,9 @@ tags: - topic/galaxy-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy when and pick_value conditional patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." related_notes: - "[[iwc-conditionals-survey]]" related_patterns: @@ -30,29 +30,33 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: conditionals patterns -This is the runtime-facing map for Galaxy conditional workflow choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. -## Direct Gates +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. -- [[conditional-run-optional-step]] — expose or derive a boolean, connect it as `inputs.when`, and use `when: $(inputs.when)` to skip optional steps. -- [[conditional-gate-on-nonempty-result]] — compute a boolean from empty/non-empty dataset or collection state before gating downstream reporting/export. The MGnify recipe is corpus-backed but clunky pending verified-pattern workflow work. +## Skip work without replacing its output -## Routes and Fallbacks +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. -- [[conditional-route-between-alternative-outputs]] — run one of several `when`-gated alternatives, then merge possible outputs with `pick_value`. -- [[conditional-transform-or-pass-through]] — optionally transform a value, then use `pick_value` to choose transformed output or original fallback. +## Continue with one value -## Not Conditionals +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. -- [[collection-cleanup-after-mapover-failure]] — use collection filters when empty or failed mapped elements should be dropped or replaced. This is collection-state cleanup, not a `when` gate. -- `__FILTER_NULL__` — catalog capability for null outputs after conditional steps, but no IWC-backed pattern candidate in the conditionals survey. Zero uptake is not itself an anti-pattern call. +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. ## See Also - [[iwc-conditionals-survey]] — conditionals survey and evidence trail. -- [[galaxy-collection-patterns]] — companion MOC for collection transforms and cleanup. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md index 8123493..de9ec07 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md @@ -13,8 +13,8 @@ tags: - topic/interval-transform status: draft created: 2026-06-10 -revised: 2026-08-03 -revision: 2 +revised: 2026-09-23 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." related_notes: - "[[iwc-interval-operations-survey]]" @@ -34,6 +34,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md index 445ed6c..591b3f5 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md @@ -13,8 +13,8 @@ tags: - topic/tabular-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-10-15 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." related_notes: - "[[iwc-tabular-operations-survey]]" @@ -40,38 +40,39 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: tabular patterns -This is the runtime-facing map for Galaxy tabular transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation pages are the actionable references. +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. -## Row And Column Operations +## Rows and columns in one table -- [[tabular-filter-by-column-value]] — keep/drop rows by string column value with `Filter1`. -- [[tabular-filter-by-regex]] — keep/drop rows by regex or pattern matching. -- [[tabular-cut-and-reorder-columns]] — project and reorder columns with `Cut1`. -- [[tabular-compute-new-column]] — compute row-wise values into a new column. +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. -## Joins And Aggregation +## Combine or summarize tables -- [[tabular-join-on-key]] — join two tabular datasets by key columns. -- [[tabular-group-and-aggregate-with-datamash]] — group and aggregate rows with `datamash_ops`. -- [[tabular-sql-query]] — use SQL when filtering, joining, and projection are clearer as one query. +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. -## Text-Processing Recipes +## Text-processing recipes -- [[tabular-prepend-header]] — add a header row with an awk/text-processing step. -- [[tabular-synthesize-bed-from-3col]] — build BED from three-column tabular inputs. -- [[tabular-split-taxonomy-string]] — split taxonomy-like strings into useful columns. -- [[tabular-relabel-by-row-counter]] — synthesize row labels from row order. +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. -## Bridges +## Between tables and collections -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. ## See also diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-data-flow/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/SKILL.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/SKILL.md index 81348d1..c8cf2ca 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/SKILL.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/SKILL.md @@ -15,7 +15,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `summary-nextflow`. Schema: summary-nextflow. Produced by `summarize-nextflow`. Structured Nextflow pipeline summary emitted by summarize-nextflow; the source-of-truth JSON for interface choices. - Read artifact `nextflow-galaxy-reference-data`. Produced by `nextflow-summary-to-galaxy-reference-data`. Reference-data shape brief from nextflow-summary-to-galaxy-reference-data that pins per-asset reference inputs and rebuild-on-absence behavior. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_provenance.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_provenance.json index 77abee0..4c55505 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/nextflow-summary-to-galaxy-interface/index.md", "revision": 6, "content_hash": "43966773391578490688098c9333f6eb2eaa94574755f5b6554e7f9b72a605c0", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "commit": "61da115975d155a36face0e83d5c28f638da7fca" }, - "cast_at": "2026-09-15T23:16:24.375Z", + "cast_at": "2026-11-10T18:57:18.730Z", "cast_date": "2026-05-07", "cast_revision": 3, "cast_history": [ @@ -55,8 +55,8 @@ "evidence": "corpus-observed", "purpose": "Choose stable workflow input/output labels and promoted checkpoint outputs that future tests can address.", "trigger": "When deciding labels, public outputs, checkpoint outputs, or fixture-compatible collection inputs.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -112,8 +112,8 @@ "load": "upfront", "evidence": "corpus-observed", "purpose": "Choose Galaxy File/list/paired/list:paired/list:list interface shapes from Nextflow channel shapes.", - "src_hash": "32efd59cc86563b145819cda38f0692bdd3144aac10ae831dc6d8accd9dcda8d", - "dst_hash": "32efd59cc86563b145819cda38f0692bdd3144aac10ae831dc6d8accd9dcda8d", + "src_hash": "e8d9515c9ae7da29188e8744410d8459e7ee9acd48af550216076fcf7a02b73b", + "dst_hash": "e8d9515c9ae7da29188e8744410d8459e7ee9acd48af550216076fcf7a02b73b", "source": "deterministic" }, { @@ -142,8 +142,8 @@ "evidence": "hypothesis", "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's input, output, and label decisions settle, and append interface obligations the pipeline leaves open — a params entry with no Galaxy input, an emitted-but-unpublished output, a channel shape no collection maps cleanly onto.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -157,8 +157,8 @@ "evidence": "corpus-observed", "purpose": "Read source-level channel, parameter, process, and test-fixture evidence before choosing Galaxy workflow inputs and outputs.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], @@ -213,7 +213,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_verify.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_verify.json index 667ae3b..d508d96 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/nextflow-to-galaxy-channel-shape-mapping.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/nextflow-to-galaxy-channel-shape-mapping.md index f8219d8..caf83b8 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/nextflow-to-galaxy-channel-shape-mapping.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/nextflow-to-galaxy-channel-shape-mapping.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-06 -revision: 2 +revised: 2026-11-10 +revision: 3 related_notes: - "[[nextflow-workflow-io-semantics]]" - "[[nextflow-params-to-galaxy-inputs]]" @@ -25,6 +25,10 @@ related_notes: - "[[relabel-via-rules-and-find-replace]]" - "[[reshape-relabel-remap-by-collection-axis]]" - "[[sync-collections-by-identifier]]" + - "[[harmonize-by-sortlist-from-identifiers]]" + - "[[derive-parameter-from-file]]" + - "[[conditional-route-between-alternative-outputs]]" + - "[[open-requirements-ledger]]" - "[[tabular-compute-new-column]]" - "[[tabular-concatenate-collection-to-table]]" - "[[tabular-cut-and-reorder-columns]]" @@ -48,19 +52,20 @@ related_molds: - "[[implement-galaxy-tool-step]]" sources: - "https://github.com/galaxyproject/foundry/issues/52" + - "https://github.com/galaxyproject/foundry/issues/757" summary: "Maps common Nextflow channel, tuple, and path shapes to Galaxy dataset and collection shapes." --- # Nextflow-to-Galaxy Channel Shape Mapping -This note maps source-level Nextflow channel shapes onto Galaxy data and dataset-collection shapes. Evidence quality is uneven: Galaxy collection semantics and IWC collection operations are well grounded; several Nextflow shapes are observed in the pinned fixtures; `fromFilePairs` and arbitrary deep tuple mapping remain low-confidence without direct fixture evidence. +This note maps source-level Nextflow channel shapes onto Galaxy data and dataset-collection shapes. The Confidence column marks how well each row is grounded. ## Shape Mapping | Nextflow shape | Galaxy shape | Confidence | Notes | |---|---|---|---| | `path(x)` or a single `Channel.fromPath(...).first()` | `File` | High | One dataset input or output. | -| `val(meta)` alone | No dataset shape | High | Treat as identifiers, labels, tags, sample-sheet metadata, or parameters. It is not a Galaxy dataset by itself. | +| `val(meta)` alone | No dataset shape | High | Treat as identifiers, labels, tags, sample-sheet metadata, or parameters. It is not a Galaxy dataset by itself. For a value computed per sample and read by a later process as a tool argument, see "Per-element parameters from `meta`" below. | | Repeated `tuple val(meta), path(file)` | `list` | High | One dataset per sample/key. Galaxy maps tools over list collections. | | One `tuple(meta, [R1, R2])` | `paired` | High | Use only when the workflow input is one paired sample or one tool consumes one pair. | | Repeated `tuple(meta, [R1, R2])` | `list:paired` | High | Common paired-end workflow input shape. | @@ -69,12 +74,12 @@ This note maps source-level Nextflow channel shapes onto Galaxy data and dataset | `tuple(meta, path(a), path(b))` | Parallel lists or per-step `File` inputs | Medium | Not automatically a `paired` collection. It is usually a keyed record of multiple tool inputs. | | Global file plus per-sample files | Broadcast `File` plus mapped `list` | Medium | If one input is global, connect it once and map the collection input. | | `collect()` or `toList()` | Collection reduction | High | Usually one downstream invocation over a collection or multiple-input value. | -| `collectFile(...)` | `File` | High | Nextflow creates one new file; Galaxy should model a tool output, not a collection operation. | +| `collectFile(...)` | `File` | High | Nextflow creates one new file. Galaxy should model a tool output, not a collection operation. | | `groupTuple()` by key | `list`, `list:paired`, or `list:list` | Medium | Depends on grouped payload and whether the grouping axis matters downstream. | | `transpose()` after grouping | Explicit reshape or subcollection mapping | Medium | May need flattening, Apply Rules, or identifier-preserving reshaping. | | `combine(..., by: key)` or `join(...)` | Multi-input collection map if identifiers match | Medium | Otherwise use explicit synchronization by identifiers/order. | | `branch { ... }` | Branch wiring or explicit filters | Medium | Shape is whatever each branch returns; per-element routing needs review. | -| `mix(...)` | Merge compatible streams/collections | Medium | Use direct wiring for reports/versions; use `__MERGE_COLLECTION__` or `__BUILD_LIST__` when materialized. | +| `mix(...)` | Merge compatible streams/collections | Medium | Use direct wiring for reports/versions, and `__MERGE_COLLECTION__` or `__BUILD_LIST__` when materialized. | | `multiMap { ... }` | Synchronized split into Galaxy inputs | Medium | Usually channel-only fan-out, but downstream rejoin depends on identifier discipline. | | `fromFilePairs` | `paired` or `list:paired` | Low | Conceptually clean, but not directly observed in the pinned fixtures. | | Arbitrary deep tuples | Parallel lists, `list:list`, or manual modeling | Low | Galaxy collection types are not arbitrary tuple records. Require per-tool review. | @@ -95,6 +100,38 @@ Use explicit Galaxy collection-operation or tabular steps when the translation c | Unbox a singleton collection | `__EXTRACT_DATASET__` | | Convert a collection of tabular outputs to one table | `collapse_dataset` or `collection_column_join` | +## Per-element parameters from `meta` + +nf-core pipelines often compute a value per sample, write it into `meta`, and read it as a tool argument in that sample's later processes. In nf-core/rnaseq's `strandedness: auto` path, `subworkflows/nf-core/fastq_qc_trim_filter_setstrandedness/main.nf` returns `meta + [strandedness: ...]`, and later modules read `meta.strandedness`. The collection operations above reshape data. This route keeps the data as it is and varies a tool parameter per element. + +Carry the value as a one-value dataset per element, in a collection with the same structure and order as the data collection. Map `param_value_from_file` over it and choose `param_type` by the target: `integer` or `float` for a number, `boolean` for a boolean, `text` for text. Connect the matching output (`integer_param`, `float_param`, `boolean_param` or `text_param`) to the target parameter of the step that maps over the data collection: + +```text +per-element values (list) ──> param_value_from_file (mapped) ──> integer_param ──┐ + ▼ +reads (list) ──────────────────────────────────────────> downstream tool (mapped, target parameter) +``` + +Galaxy runs one job per element and pairs the two collections by position, not by element identifier. A same-length value collection in another order runs without error and gives each element the value at its position, not its own. Produce the values with a step mapped over the data collection so they inherit its order. When they arrive as a separate collection, reorder them first: run `collection_element_identifiers` on the data collection, then `__SORTLIST__` with `sort_type: file` on the value collection, connecting the identifier file to `sort_type|sort_file` ([[harmonize-by-sortlist-from-identifiers]]). + +| Target | Evidence | Status | +|---|---|---| +| Integer tool parameter | `verification/workflows/per-element-parameter-from-file/`: values 1 and 3 on two elements, and positional pairing in its second case | Verified | +| Text tool parameter | Galaxy's `lib/galaxy_test/workflow/map_over_expression.gxwf.yml` and its `-tests.yml`: per-element values `A` and `B` reach a text input | Verified | +| Per-element `when` (boolean) | Galaxy's `lib/galaxy_test/api/test_workflows.py::test_run_workflow_conditional_step_map_over_expression_tool`: one of two elements is skipped | Verified | +| Parameter inside a tool conditional's case | IWC `epigenetics/consensus-peaks/consensus-peaks-atac-cutandrun`: mapped "convert min nb of reads to parameter" feeds `samtools_view`'s `mode\|subsample_config\|subsampling_mode\|target`, with values identical across elements | Wiring only | +| `select` tool parameter | The same fixture's `per-element-select-boolean` workflow: text values `C` and `D` choose `Cut1`'s `delimiter` per element | Verified | +| Boolean tool parameter | The same workflow: `true` and `false` set `wc_gnu`'s `include_header` per element | Verified | +| Float tool parameter | None | Not tested | +| A tool conditional's test parameter | [#661](https://github.com/galaxyproject/foundry/issues/661) reports a Galaxy 500 when one is connected. [#667](https://github.com/galaxyproject/foundry/issues/667) reports a workflow connecting one that runs on usegalaxy.org | Open | + +Use this route for every target except the Open row, and cover a Wiring only or Not tested target with a workflow test whose elements carry different values. rnaseq's strandedness reaches both kinds of target. featureCounts' `strand_specificity` is a `select` and takes this route. A `select` takes an option value, not the source's string, so translate first: featureCounts' options are `0`, `1` and `2`, where rnaseq writes `unstranded`, `forward` and `reverse`. [[map-workflow-enum-to-tool-parameter]] does this with `map_param_value`, which hasn't been tested mapped over elements. Salmon's `--libType` is `strandedness`, the test parameter of its `libtype` conditional, and takes the fallback. + +For a conditional's test parameter, or when a workflow test of this route fails, fall back in this order: + +1. For a small fixed set of values, run one branch per value. For each value, write a per-element boolean file with a mapped comparison step, read it with mapped `param_value_from_file` (`param_type: boolean`), gate that value's branch with `when`, and give the branch's step that value as a fixed parameter. Rejoin the branches with Galaxy's built-in `pick_value` (`pick_style: first_or_error`) mapped over the branch outputs, never `__MERGE_COLLECTION__`. A gated mapped step still emits a full-size collection with its skipped elements as null, so a merge returns one element per branch per sample. Galaxy's `test_run_workflow_conditional_step_map_over_expression_tool_pick_value` covers this mapped rejoin, and [[conditional-route-between-alternative-outputs]] covers `pick_value`'s settings. +2. Otherwise, expose one workflow-level parameter for all elements. Append a `kind: dropped` entry to the [[open-requirements-ledger]] whose `units` names the per-element parameter and whose `because` cites the Open row or the failing test. Expose the computing step's output as a workflow output so a user can run once, read the computed values, and re-run with the value they choose. + ## Evidence Corpus-observed Galaxy semantics: @@ -104,13 +141,13 @@ Corpus-observed Galaxy semantics: - [[galaxy-apply-rules-dsl]] explains identifier-derived collection reshaping. - [[iwc-transformations-survey]] records which collection transformations appear in IWC workflows. -Pinned fixture examples used by the research pass: +Nextflow fixture examples: - `workflow-fixtures/pipelines/nf-core__demo/subworkflows/local/utils_nfcore_demo_pipeline/main.nf` for `tuple(meta, reads)` paired-read handling. - `workflow-fixtures/pipelines/nf-core__rnaseq/main.nf` for single-end versus paired-end branching and reads normalization. - `workflow-fixtures/pipelines/nf-core__taxprofiler/main.nf` and `workflow-fixtures/pipelines/nf-core__taxprofiler/subworkflows/local/*/main.nf` for `groupTuple`, `transpose`, `mix`, `combine`, and `multiMap` patterns. - `workflow-fixtures/pipelines/nf-core__fetchngs/workflows/sra/main.nf` for `collectFile` and accession-driven data fetching. -- `$IWC_FORMAT2/virology/influenza-isolates-consensus-and-subtyping/influenza-consensus-and-subtyping.gxwf.yml` as a Galaxy `list:paired` exemplar. +- IWC `virology/influenza-isolates-consensus-and-subtyping/influenza-consensus-and-subtyping` as a Galaxy `list:paired` exemplar. ## Low-Confidence TODOs diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-interface/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/SKILL.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/SKILL.md index bea0dd5..c6ccfec 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/SKILL.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/SKILL.md @@ -14,7 +14,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Inputs - Read artifact `summary-nextflow`. Schema: summary-nextflow. Produced by `summarize-nextflow`. Structured Nextflow pipeline summary emitted by summarize-nextflow; the source for reference-data param evidence. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_provenance.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_provenance.json index db91c1f..a16df93 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/nextflow-summary-to-galaxy-reference-data/index.md", "revision": 5, "content_hash": "b49bae31e3b8918b45f852b8f50e1101e71837380b97f8e8ec237a5dd2955f7f", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "commit": "77425ce07f69da2bdda1de21faea3abc4600ae00" }, - "cast_at": "2026-09-15T23:16:24.569Z", + "cast_at": "2026-11-10T15:33:28.743Z", "refs": [ { "kind": "research", @@ -64,8 +64,8 @@ "evidence": "hypothesis", "purpose": "Inherit open entries rather than re-deriving them, close the ones a settled reference-data shape discharges, and append reference obligations the pipeline leaves open — a genome key with no Galaxy data table, a reference the pipeline builds at runtime, a build the corpus doesn't carry.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -79,8 +79,8 @@ "evidence": "corpus-observed", "purpose": "Read `reference_assets[]` (curated path-typed inputs with `used_by` attribution to consuming subworkflows), `reference_rebuilds[]` (compute-if-missing rebuild rules with builder process and guard), and `params[]` provenance (`source_kind`, including `getGenomeAttribute` for nf-core key-expanded bundles) to determine which reference assets the source pipeline consumes and how it rebuilds them when absent.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], @@ -128,7 +128,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_verify.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_verify.json index 667ae3b..d508d96 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-reference-data/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/SKILL.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/SKILL.md index b13446d..20672cb 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/SKILL.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/SKILL.md @@ -19,7 +19,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `nextflow-galaxy-data-flow`. Produced by `nextflow-summary-to-galaxy-data-flow`. Galaxy data-flow brief from nextflow-summary-to-galaxy-data-flow that pins abstract operations and collection choices. - Read artifact `iwc-comparison-notes`. Produced by `compare-against-iwc-exemplar`. Structural diff guidance from compare-against-iwc-exemplar (run on the design briefs); steers the skeleton toward IWC-aligned structure before per-step authoring. Carries an inline gxformat2 excerpt of the nearest exemplar. - Read artifact `iwc-exemplar-gxformat2`. Produced by `compare-against-iwc-exemplar`. Cleaned gxformat2 view of the nearest IWC exemplar's relevant subgraph from compare-against-iwc-exemplar; pattern-match the draft's input/collection shapes, map-over wiring, output promotion, and post-job actions against this concrete idiom. Absent when no nearest exemplar was found. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs @@ -28,8 +28,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli @@ -45,6 +45,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load On Demand +- `references/cli/draft-next-step.json`: CLI command reference packaged as a sidecar. Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop. Use when: deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step. - `references/cli/draft-validate.json`: CLI command reference packaged as a sidecar. Validate the emitted draft against draft-contract rules (sentinel form, topology, _plan_* placement) before handing off. Use when: after writing or modifying the draft workflow file. - `references/patterns/galaxy-collection-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded collection pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges. - `references/patterns/galaxy-conditionals-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded conditional pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_provenance.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_provenance.json index 93d6425..1dca535 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "nextflow-summary-to-galaxy-template", "path": "content/molds/nextflow-summary-to-galaxy-template/index.md", - "revision": 9, - "content_hash": "10008cbd7536688ce772c1e70a26598be7b865d97efd9f9c0c2dff2a49492f96", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "revision": 10, + "content_hash": "77f885d640900cca7dac4a3bbb33308d4d8065e871c919c672487db61caf93ab", + "commit": "77425ce07f69da2bdda1de21faea3abc4600ae00" }, - "cast_at": "2026-09-15T23:16:25.083Z", + "cast_at": "2026-11-10T15:33:28.827Z", "cast_date": "2026-05-07", "cast_revision": 3, "cast_history": [ @@ -29,6 +29,22 @@ } ], "refs": [ + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[draft-next-step]]", + "src": "content/cli/gxwf/draft-next-step.md", + "dst": "references/cli/draft-next-step.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop.", + "trigger": "When deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step.", + "verification": "Cast the skill, emit a draft with a drafty step inside a subworkflow, confirm draft-next-step returns that step's full path.", + "src_hash": "d4fe7945945a57846fe8b168e7bacba04f739de1d86bc613ae6f0e5879ac2f55", + "dst_hash": "8638645f5974d8072dc3c63e522b7ab25d562f8d6217ea93b61f0a29f371d6f7", + "source": "deterministic" + }, { "kind": "cli-command", "mode": "sidecar", @@ -56,8 +72,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded collection pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges.", - "src_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", - "dst_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", "source": "deterministic" }, { @@ -71,8 +87,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded conditional pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", - "src_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", - "dst_hash": "cbeeb61caaa4f9e09cc7a209f76c3d84b7a9e6934610cb7eac1c2ec4fead70a1", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", "source": "deterministic" }, { @@ -86,8 +102,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded genomic-interval pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for interval overlap, merge, coverage, windowing, masking, or set-algebra on coordinate features.", - "src_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", - "dst_hash": "2f6e228baab452d7bd51fbd27fc4f584cb9387d496261cb264695824901d23bf", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", "source": "deterministic" }, { @@ -101,8 +117,8 @@ "evidence": "corpus-observed", "purpose": "Use corpus-grounded tabular pattern guidance for unresolved skeleton steps.", "trigger": "When adding TODO steps for tabular filtering, projection, joins, aggregation, text-processing recipes, or tabular-collection bridges.", - "src_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", - "dst_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", "source": "deterministic" }, { @@ -154,8 +170,8 @@ "purpose": "Respect the handoff from abstract data-flow draft to gxformat2 skeleton.", "trigger": "When translating abstract nodes, unresolved tool needs, and placeholder transformations into template TODOs.", "verification": "Promote after two worked Nextflow-to-Galaxy templates preserve the data-flow/template split without schema changes.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -169,8 +185,8 @@ "evidence": "corpus-observed", "purpose": "Group the settled step set into titled stage frames (the gxformat2 `comments:` array) so the skeleton carries the analysis-stage narrative IWC authors annotate by hand. Schema-legal and optional.", "trigger": "After topology is settled and the skeleton can be partitioned into named analysis stages (inputs, per-stage step clusters, parameter-derivation knots, visualization/outputs).", - "src_hash": "a206d5e08a8caaeaa42d9eadcd78eb0e2724c5756be681ca4b543cf0f7269176", - "dst_hash": "a206d5e08a8caaeaa42d9eadcd78eb0e2724c5756be681ca4b543cf0f7269176", + "src_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", + "dst_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", "source": "deterministic" }, { @@ -184,8 +200,8 @@ "evidence": "hypothesis", "purpose": "Emit the gxformat2 draft superset: TODO tool_id, optional state / tool_shed_repository, and per-step _plan_state / _plan_context planning fields.", "verification": "Promote after a downstream per-step implementation Mold consumes _plan_state and _plan_context without round-tripping back through the source summary.", - "src_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", - "dst_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", + "src_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", + "dst_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", "source": "deterministic" }, { @@ -199,8 +215,8 @@ "evidence": "corpus-observed", "purpose": "Choose stable workflow input/output labels, testable checkpoint outputs, and fixture-compatible workflow interfaces while drafting the skeleton.", "trigger": "When the template decides workflow inputs, workflow outputs, promoted checkpoints, or collection output identifiers that future tests will need to address.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -215,8 +231,8 @@ "purpose": "Use the gxformat2 structural vocabulary for workflow inputs, outputs, steps, and producer-side output actions while emitting the skeleton.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "4fe42382f99655115c53c1afde0da0d0556ad77984da7d0ffa8ab725ea260aa3", - "dst_hash": "4fe42382f99655115c53c1afde0da0d0556ad77984da7d0ffa8ab725ea260aa3", + "src_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", + "dst_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" }, @@ -249,8 +265,8 @@ "evidence": "corpus-observed", "purpose": "Emit `when:` clauses on the right step type (subworkflow vs tool) and the right output fan-in primitive (`pick_value`, modes `first_non_null` / `first_or_skip` / `the_only_non_null`) when the data-flow brief carries a conditional disposition through to the skeleton.", "trigger": "When the upstream data-flow brief assigns a source conditional to a subworkflow `when:` or inline `when:` shape, or when emitting the merge step for two or more `when:`-gated branches that produce the same logical output.", - "src_hash": "e341284cdf3a2cdb7381c37aa50e409f99fde7c5dbcdefe64cf2ca15dd28f85e", - "dst_hash": "e341284cdf3a2cdb7381c37aa50e409f99fde7c5dbcdefe64cf2ca15dd28f85e", + "src_hash": "f38341565d0927321248b67a9d99ce9533cc89ef4e543cbf80378260e7025f7e", + "dst_hash": "f38341565d0927321248b67a9d99ce9533cc89ef4e543cbf80378260e7025f7e", "source": "deterministic" }, { @@ -294,8 +310,8 @@ "evidence": "hypothesis", "purpose": "Close the open entries the settled topology discharges, and append a blocking entry for any settled step whose declared output no wired input can supply — the computability gap gxwf validation cannot see, raised here rather than left for the per-step loop to hit.", "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -310,8 +326,8 @@ "purpose": "Output contract: the emitted gxformat2 draft conforms to [[galaxy-workflow-draft]]. Cast bundles the JSON Schema so the skill carries its output shape alongside the [[draft-validate]] CLI checks.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", - "dst_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" }, @@ -326,8 +342,8 @@ "evidence": "corpus-observed", "purpose": "Read process, channel, operator, and fixture structure when emitting placeholder steps and TODO context.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], @@ -411,7 +427,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_required_tools.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_required_tools.json index 54fd1c9..1a8a44d 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_required_tools.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_required_tools.json @@ -3,13 +3,14 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": [ + "gxwf draft-next-step", "gxwf draft-validate" ] } diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_verify.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_verify.json index 667ae3b..d508d96 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/cli/draft-next-step.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/cli/draft-next-step.json new file mode 100644 index 0000000..296abe7 --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/cli/draft-next-step.json @@ -0,0 +1,42 @@ +{ + "type": "cli-command", + "tool": "gxwf", + "command": "draft-next-step", + "summary": "Pick the next drafty step a harness should work on, or report no remaining work; deterministic topological + alphabetical tiebreak.", + "source_path": "content/cli/gxwf/draft-next-step.md", + "source_revision": 1, + "package": "@galaxy-tool-util/cli", + "description": "Pick the next step a downstream agent should work on (or report no remaining work)", + "synopsis": "gxwf draft-next-step [options] ", + "args": [ + { + "raw": "file", + "name": "file", + "required": true, + "variadic": false, + "description": "Draft workflow file (.gxwf.yml)" + } + ], + "options": [ + { + "flags": "--format ", + "name": "format", + "description": "Force input format: format2 (default; native is rejected)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--output-format ", + "name": "outputFormat", + "description": "Output format: json (default) or markdown", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false, + "defaultValue": "json" + } + ], + "body": "# `gxwf draft-next-step`\n\nDeterministic next-step picker for the per-step authoring loop. Wraps `nextDraftStep` from `@galaxy-tool-util/schema`. Native (.ga) input is rejected — drafts are format2-only.\n\nThe harness loop reads as: `while (next = gxwf draft-next-step ).draft: invoke per-step skill on next.step`.\n\n## Output\n\nDefault output is pretty JSON — the agent-loop wire format. Same input → byte-identical output. `--output-format markdown` renders a human-glance checklist instead.\n\nJSON fields:\n\n- `draft` (bool) — `true` while a drafty step remains, `false` at the terminal cases below.\n- `step` (string array) — the picked step's **path**: a single id at top level (`[\"merge assembled transcripts\"]`), or the full path through containing subworkflow steps when the pick is nested. The leaf (last element) is the step to work on.\n- `work` (string array) — the chosen step's remaining `TODO[...]` sentinels followed by its `_plan_*` fields, in checklist order.\n\nAt the terminal cases the object is just `{ \"draft\": false }` — no `step` or `work` keys.\n\nTwo terminal cases both surface as `draft: false`:\n\n- The workflow's class is already `GalaxyWorkflow` (concretization complete, class promoted).\n- The class is still `GalaxyWorkflowDraft` but every step is fully resolved (no `TODO_*`, no `_plan_*`). The harness's next move is [[draft-extract]] (which calls `promoteFullyConcreteDrafts` to flip the class) followed by terminal [[validate]].\n\n## Examples\n\n```bash\ngxwf draft-next-step workflow.gxwf.yml\ngxwf draft-next-step workflow.gxwf.yml --output-format markdown\n```\n\nHarness loop sketch:\n\n```bash\nwhile [ \"$(gxwf draft-next-step workflow.gxwf.yml | jq -r .draft)\" = \"true\" ]; do\n step=$(gxwf draft-next-step workflow.gxwf.yml | jq -r '.step | last')\n # invoke per-step skill on step, which mutates workflow.gxwf.yml in place\ndone\n```\n\n## Gotchas\n\n- Pick order is **topological** (a step waits until its `in:` sources are concretized) with **alphabetical tiebreak** on step id at the same level. Same input across runs returns the same pick — safe to call repeatedly without state.\n- Descends into draft subworkflows; the returned `step` array is the full path through containing subworkflow steps (leaf last).\n- A `draft: false` result does **not** mean the workflow has passed [[validate]] yet. It means the per-step authoring loop is done; the harness still owns the promote + terminal-validate pass.\n- This command does not mutate the workflow file. It's a pure observer.\n- Insertion order of step dicts does not affect output — semantically equal workflows produce identical picks." +} diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md index efd7c87..9256dbf 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-comments.md @@ -4,12 +4,13 @@ tags: - target/galaxy status: draft created: 2026-06-12 -revised: 2026-06-12 -revision: 1 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-comments-survey]]" - "[[gxformat2-schema]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" summary: "How to annotate a gxformat2 workflow with editor comments: one titled frame per analysis stage, populate contains_steps, color decorative." diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md index 2a66f1a..9fc6e51 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-06 -revised: 2026-05-10 -revision: 2 +revised: 2026-11-10 +revision: 6 related_notes: - "[[gxformat2-schema]]" - "[[galaxy-data-flow-draft-contract]]" @@ -17,6 +17,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[implement-galaxy-tool-step]]" - "[[advance-galaxy-draft-step]]" @@ -117,14 +118,14 @@ steps: ## Topology repair escalation -The template Mold settles topology, but it can miss a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. +The template Mold settles topology, but a settled draft can still be unable to give a correct run. The common case is a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. When one surfaces, the per-step loop does not author topology to fix it. It **escalates**: -- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. -- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and re-wires the affected region: one producer step, a small sub-path, or honest narrowing of the output. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. +- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs, or that the chosen wrapper can't produce it. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. Earlier Molds and the loop raise blocking entries too. +- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and makes the smallest structural edits that close them. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. -This is **repair**, distinct from **settling**: it is narrow (one named region), reactive (only a detected computability gap triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each escalation must strictly reduce that count, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. +This is **repair**, distinct from **settling**: it is narrow (the entries bound it), reactive (an open blocking entry triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each repair must leave fewer than it found, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. ## Open work diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/gxformat2-schema.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/gxformat2-schema.md index 7682a33..af5053a 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/gxformat2-schema.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/gxformat2-schema.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-05 -revised: 2026-05-06 -revision: 2 +revised: 2026-11-06 +revision: 5 license: MIT license_file: LICENSES/galaxy-tool-util-ts.LICENSE related_notes: @@ -17,69 +17,58 @@ related_notes: - "[[galaxy-workflow-draft]]" sources: - "https://github.com/jmchilton/galaxy-tool-util-ts/blob/7ae4ecd0ba8d492225f58a6d455c4cc5317298f0/packages/schema/src/galaxy-workflow.ts" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py" companions: - "gxformat2.schema.json" -summary: "Vendored structural JSON Schema for gxformat2 workflows: vocabulary for inputs, outputs, steps, and step subtypes." +summary: "Vendored gxformat2 structural schema, its input, output, and step vocabulary, and the limits of schema-only validation." --- -> **Vendored from upstream**, pinned at SHA `7ae4ecd`. One file lives next to this note, declared in `companions:` so casting carries it into every bundle this note reaches: -> -> - `gxformat2.schema.json` — Draft-07 JSON Schema generated from the `@galaxy-tool-util/schema` Effect-TS schema via `gxwf structural-schema --format format2`. **Agents and casting should consume this** when validating gxformat2 workflows or reasoning about the closed vocabulary of input/step types. -> -> **Re-sync:** `pnpm sync:vendored --update`. Sync builds galaxy-tool-util-ts and re-runs the generator before copy. +# gxformat2 structural schema -## Top-level shape +The companion `gxformat2.schema.json` is a vendored Draft-07 JSON Schema generated from the `@galaxy-tool-util/schema` Effect-TS model at `7ae4ecd`. Casting carries it with this note. Use it to inspect the structural vocabulary of a gxformat2 workflow, then check connections, tool references, and runtime behavior with gxwf and Galaxy validation. The schema does not establish that an `outputSource` exists or that a tool state is valid for its wrapper. -A gxformat2 document is an object with required `inputs`, `outputs`, `class` (`enum: ["GalaxyWorkflow"]`), and `steps`. Optional: `id`, `label`, `doc`, `uuid`, `report`, `tags`, `comments`, `creator`, `license`, `release`. +The schema is regenerated by `pnpm sync:vendored --update`, which builds the upstream package and runs `gxwf structural-schema --format format2` before copying it here. Do not edit the companion by hand. -## Workflow input vocabulary +## Root and input fields -Each entry under `inputs` carries: +The root requires `class: GalaxyWorkflow`, `inputs`, `outputs`, and `steps`. Those sections accept arrays or keyed maps. The root also permits `id`, `label`, `doc`, `uuid`, `report`, `tags`, `comments`, `creator`, `license`, and `release`. -- `type` — closed enum: `null | boolean | int | long | float | double | string | integer | text | File | data | collection`. Drives whether the input is a parameter, a single dataset, or a collection. -- `format` — Galaxy datatype extension when `type ∈ {File, data}`. See [[galaxy-datatypes-conf]] for valid values. -- `collection_type` — free-form string (e.g. `list`, `paired`, `list:paired`, `list:list`) when `type = collection`. Closed value set lives in [[galaxy-collection-semantics]], not in this schema. -- `optional` — boolean. -- `default` — opaque (`/schemas/unknown`); type-dependent. -- `label`, `doc`, `id`, `position` — metadata. +Input entries describe the public workflow interface. The array form exposes these fields: -## Workflow output vocabulary +| Field | Structural schema | Meaning and use | +|---|---|---| +| `id`, `label`, `doc` | Optional | Identify or describe the input. Check the effective imported name before using it as a test `job:` key. | +| `type` | Optional string, string array, or `null` | Enum members are `null`, `boolean`, `int`, `long`, `float`, `double`, `string`, `integer`, `text`, `File`, `data`, and `collection`. This is accepted vocabulary, not a list of preferred authoring spellings. | +| `format` | Optional string array or `null` | Galaxy datatype extension filter. See [[galaxy-datatypes-conf]]. | +| `collection_type` | Optional string or `null` | Collection shape such as `list:paired`. The schema does not enumerate valid shapes. See [[galaxy-collection-semantics]]. | +| `optional`, `default` | Optional boolean or `null`, and opaque value, respectively | Control whether an input can be omitted and what value is supplied when absent. See [[gxformat2-workflow-inputs]] for runtime behavior. | +| `position` | Optional `{top, left}` numeric coordinates or `null` | Workflow editor placement. | -Top-level `outputs` can be an array or object map. Each output entry carries: +The generated schema does not include all fields accepted by current gxformat2 input handling, including `restrictions`, `suggestions`, and `restrictOnConnections`. [[gxformat2-workflow-inputs]] distinguishes that implementation behavior from what this vendored schema can check. -- `label` — public workflow-output name; tests and users should address this stable label. -- `outputSource` — producing step output reference. -- `doc` — optional prose context, string or string array. -- `id` — optional identifier. -- `type` — same permissive type vocabulary as inputs, when needed. +## Output names and connections -Producer-side output actions are separate from top-level `outputs`. Step `out:` entries can carry `change_datatype`, `rename`, `add_tags`, `remove_tags`, `hide`, `delete_intermediate_datasets`, and `set_columns`, or a plain string output name. See [[galaxy-workflow-testability-design]] for authoring posture: `label` is the public API; producer-side actions make the dataset useful. +An output entry can carry `id`, `label`, `doc`, `outputSource`, and `type`. `outputSource` names the producing step output, written `step/output`. The output's **public name** is not always its `label`: normalized gxformat2 export places a native Galaxy workflow-output label in `id`, and the pinned IWC [SRA manifest workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs.ga) exports `id: paired_output`. Current gxformat2 import chooses `label` when present and otherwise `id`. Check the effective name after import or conversion before writing the sibling test's `outputs:` key. See [[galaxy-workflow-testability-design]]. -## Workflow step vocabulary +The schema makes `outputSource` optional, even though a usable produced output needs a valid source. Its string type cannot prove that the referenced step and port exist. gxformat2 lint checks the step reference. Workflow validation must also check that the producing port exists. -Each entry under `steps` (`WorkflowStepSchema`) carries `type` ∈ `{tool, subworkflow, pause, pick_value}`, plus `tool_id`, `tool_shed_repository`, `tool_version`, `in`, `out`, `state`, `tool_state`, `run`, `runtime_inputs`, `when`, `errors`, `uuid`, `label`, `doc`, `position`, `id`. +The step part of a `source` or `outputSource` depends on how the workflow lists its steps. In list form it is the step's `label` when one is present, otherwise its `id`. In map form, for a step with no `label`, it is the map key. -### `state` vs `tool_state` +A step's `in:` key addresses a parameter nested in a conditional or section with a `|`-joined path, and a repeat instance with `_N`. For example, `input_type|i` wires an input inside a conditional in `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences`, and `style_cond|type_cond|pick_from_0|value` wires the first instance of a repeat inside two conditionals in [[conditional-route-between-alternative-outputs]]. The key names the path, and the conditional's selector value stays in `state`. Each `_N` connection also needs its own entry in the repeat in `state`. The repeat creates the input terminal, so without the entry the connection does nothing and the job can still succeed. -Both hold the same thing — a map from tool parameter names to values — and a step should -carry only one. Which one marks how the workflow was produced: +Step `out:` entries serve a different purpose. They can be output names or records with `id` and actions such as `rename`, `change_datatype`, `add_tags`, `remove_tags`, `hide`, `delete_intermediate_datasets`, and `set_columns`. These actions affect produced datasets or history presentation. They do not supply a top-level output's public name. -- **`state` is the authoring key.** A typed map, with conditionals and repeats written as - nested YAML. Molds that *author* a workflow or draft write `state`. Upstream agrees: of - the twelve gxformat2 format2 examples that carry step state, ten use `state`, and the - schema's own field docs say `tool_state` "may be present in workflows exported from Galaxy - but shouldn't be written by humans." -- **`tool_state` is the export key.** `gxwf convert --to format2` emits it for every tool - step, so any format2 view of a `.ga` workflow — the IWC corpus included — arrives carrying - `tool_state`. Molds that *read*, summarize, or preserve an existing workflow keep the key - they were given rather than rewriting it. +A `rename` value is not a tool-XML label template. Galaxy's [`RenameDatasetAction`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py#L152-L256) fills in two forms. `#{input}` becomes the name of the dataset bound to that tool input. Use the tool's input name, with `.` for a nested one (e.g. `queries_0.input2`), and optional `| basename`, `| upper` or `| lower`. `${label}` becomes the value of the workflow parameter input with that label. Write `#{gtf_to_extend} BREW3R extended` to name an output after its input. A `${name}` with no matching parameter input is an informal, untyped parameter ([`get_informal_replacement_parameters`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py#L3094-L3103)). The run form asks for a free-text value, and an API or Planemo run that supplies no `replacement_params` leaves `${name}` literal in the dataset name. The tool-XML variables `${on_string}` and `${tool.name}` work only in a wrapper's ``; in a rename they fall into this case. This is what the workflow linters report as an "untyped parameter"; declare a parameter input instead. -Nothing enforces this. Galaxy and `gxwf` both accept a nested map under either key, so a -hand-authored `tool_state` validates, converts, and runs; no gate reports the mismatch. It is -a convention that keeps authored and exported workflows distinguishable on sight. +## Step fields and state -## Caveats +The structural step `type` enum is `tool`, `subworkflow`, `pause`, and `pick_value`, with `null` also admitted. gxformat2's source schema defaults an omitted type to `tool`. A step can also carry `id`, `label`, `doc`, `position`, `tool_id`, `tool_shed_repository`, `tool_version`, `in`, `out`, `state`, `tool_state`, `run`, `runtime_inputs`, `when`, `errors`, and `uuid`. The structural schema does not require subtype-specific fields such as `tool_id` on a tool step, so validation must continue beyond JSON Schema. -- **Doc strings dropped.** Effect-TS `JSONSchema.make` does not preserve description fields. Narrative grounding for what each input/step type means lives in this note and in upstream gxformat2 SALAD (`$GXFORMAT2/schema/v19_09/workflow.yml`). -- **Repeated `$id: "/schemas/unknown"` markers.** The `Schema.Unknown` placeholder for opaque values (such as `default`) emits a duplicate `$id` 15× in the gxformat2 schema. Strict Ajv refuses to compile; pass `{ strict: false }` or strip the duplicates before compile. -- **`collection_type` is not a closed enum.** The schema accepts any string; valid composite shapes are pinned in [[galaxy-collection-semantics]]. +For hand-authored tool steps, use `state` for typed nested YAML, including conditionals and repeats. A converted workflow may contain `tool_state`, as the pinned IWC SRA manifest example does. Treat that as the representation received, not an authoring recommendation. Current import accepts `tool_state` as a map or JSON string, but gives `state` precedence if both fields are present. The structural schema permits both, so use only one on a step. Preserve an exported representation when reading or carrying it forward, and use `state` when authoring a new step. + +## What schema validation leaves open + +The root and array-form entries constrain their listed properties, but generated map-form `inputs` and `outputs` each include a permissive object branch. Passing this schema therefore does not prove that every map entry has only documented fields. In map form, the `type` enum and `format` shape can also escape the documented constraints. Every form still needs semantic checks for connections and tool configuration. + +The generated JSON also omits field descriptions and repeats `$id: "/schemas/unknown"` on opaque values. Some strict Ajv configurations reject those duplicate identifiers. If that occurs, compile with the project's supported non-strict configuration or remove duplicate markers in a validation copy. Keep the vendored companion unchanged. The upstream [gxformat2 source schema](https://github.com/galaxyproject/gxformat2/blob/main/schema/v19_09/workflow.yml) supplies field documentation that this generated artifact lacks. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md index c7ce559..38a7704 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/nextflow-conditional-to-galaxy-subworkflow-when.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-08 -revised: 2026-05-08 -revision: 1 +revised: 2026-11-07 +revision: 2 related_notes: - "[[nextflow-to-galaxy-reference-data-mapping]]" - "[[nextflow-to-galaxy-channel-shape-mapping]]" @@ -19,7 +19,7 @@ related_molds: sources: - "https://github.com/galaxyproject/gxformat2" - "https://github.com/iwc-workflows" -summary: "Stub. Translate Nextflow conditionals into Galaxy `when:` (single-workflow v1). Subworkflow vs inline is an aesthetic call, not a rule." +summary: "Stub. Translate Nextflow conditionals into Galaxy `when:` (single workflow). Subworkflow vs inline is an aesthetic call, not a rule." --- # Nextflow conditional to Galaxy subworkflow / when @@ -28,9 +28,11 @@ Stub. Surfaced from sarek emulation (2026-05-08). Companion to [[nextflow-to-gal ## Posture -For v1 of the Nextflow-to-Galaxy translation Molds the output is a single Galaxy workflow per source pipeline, even when the source has substantial branching. IWC reviewers historically prefer sibling workflows for what looks like one pipeline with toggles, and we agree; but for the *translation step* a single artifact keeps the Mold pipeline deterministic, the harness simple, and the reviewer's mental model of "this draft maps 1:1 to the source" intact. Sibling-extraction is a polish pass a human or follow-up Mold runs *after* translation, not a decision the translation Mold makes. +The Nextflow-to-Galaxy translation Molds output a single Galaxy workflow per source pipeline, even when the source has substantial branching, and capture as much of the source's scope as conditionals allow. IWC workflows don't typically dispatch on different tools within a workflow, but that reflects the corpus, not a design constraint. When the source offers a tool choice, carry it in one workflow with conditionals, especially when porting a workflow that already dispatches this way. A second workflow just for a tool choice is not a goal. -The question this note addresses is: given that v1 is one Galaxy workflow, *how* should source conditionals translate? The two structural tools available are: +IWC reviewers have historically preferred sibling workflows for what looks like one pipeline with toggles. A human or follow-up Mold can still extract siblings after translation, but the translation Mold doesn't aim for that split. A single artifact also keeps the Mold pipeline deterministic, the harness simple, and the reviewer's mental model of "this draft maps 1:1 to the source" intact. + +The question this note addresses is: given one Galaxy workflow, *how* should source conditionals translate? The two structural tools available are: - **Inline `when:` on a tool step.** Step runs or doesn't, based on an ECMAScript 5.1 expression over workflow inputs. Lightweight; doesn't change the DAG shape. - **`when:` on a subworkflow step.** A bundle of steps runs or doesn't as a unit. Adds a subworkflow boundary and the wiring overhead that comes with it (typed inputs, typed outputs, explicit pass-through). When the parent's `when:` is false the inner steps inherit the skip via the engine's `when_values` propagation. @@ -63,7 +65,7 @@ These are the cases where the call is genuinely judgment, and an agent rendering - **Sarek `aligner` (bwa-mem | bwa-mem2 | dragmap | sentieon | parabricks).** Five-way. Source has `FASTQ_ALIGN_BWAMEM_MEM2_DRAGMAP_SENTIEON` as one subworkflow, plus a separate `FASTQ_PREPROCESS_PARABRICKS` because parabricks bundles align+sort+dedup. Mixed signal: the source leans subworkflow for four of the five, parabricks is its own beast. Probably means: subworkflow for the four-way alignment swap (matches source), a sibling `when:`-gated subworkflow step for parabricks (matches source). Inline `when:` would scatter five alignment tool choices through the main workflow body. - **Sarek `wes`.** Two-way boolean. Source scopes WES handling inside `PREPARE_INTERVALS` and a few variant-calling internals. Strong "subworkflow `when:`" signal — source already structured it that way, cascade is wide, interface is narrow (intervals BED in, intervals BED out). -- **Sarek `tools` selector (variant callers).** Source scopes each caller in its own subworkflow (`BAM_VARIANT_CALLING_HAPLOTYPECALLER`, `BAM_VARIANT_CALLING_FREEBAYES`, …). Each caller's outputs land in a downstream merge. Strong "subworkflow `when:` per caller" signal — but the wiring at the merge is delicate (each branch contributes a different VCF format), so the merge step itself becomes the awkward part. Maybe the right answer here is sibling Galaxy workflows after all, even at v1 — flag for the human reviewer. +- **Sarek `tools` selector (variant callers).** Source scopes each caller in its own subworkflow (`BAM_VARIANT_CALLING_HAPLOTYPECALLER`, `BAM_VARIANT_CALLING_FREEBAYES`, …). Each caller's outputs land in a downstream merge. Strong "subworkflow `when:` per caller" signal — but the wiring at the merge is delicate (each branch contributes a different VCF format), so the merge step itself becomes the awkward part. Flag that wiring for the human reviewer rather than splitting the workflow. ## Output fan-in: use `pick_value`, not a generic merge @@ -86,7 +88,7 @@ IWC uses the `when:` + `pick_value` pattern in production. Two confirmed example - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-artic-ivar-analysis/pe-wgs-ivar-analysis.gxwf.yml` — two parallel tool steps both gated by `when: $(inputs.when)` from a shared boolean, merged downstream with a `pick_value` step. - `comparative_genomics/hyphy/capheine-core-and-compare.gxwf.yml` — multiple HyPhy analyses guarded by optional foreground/background booleans, merged downstream. -So `when:` + `pick_value` for branch selection is corpus-observed and IWC-acceptable. **What is *not* observed in IWC**: dynamic tool selection via `when:` (e.g. "run bwa-mem or parabricks based on a `tools` selector input"). Every IWC workflow commits to a fixed tool stack at authoring time. A Nextflow-to-Galaxy translation that surfaces aligner choice as a runtime `when:`-gated swap would be a novel pattern in IWC. The mechanism is well-supported by the engine; the precedent is missing. Worth flagging in any draft brief that proposes it: "this works mechanically, but IWC has no exemplar for tool-selection-via-when, and a reviewer may push back toward sibling workflows." +So `when:` + `pick_value` for branch selection is corpus-observed and IWC-acceptable. **What IWC doesn't typically do**: dynamic tool selection via `when:` (e.g. "run bwa-mem or parabricks based on a `tools` selector input"). IWC workflows typically commit to a fixed tool stack at authoring time, so a Nextflow-to-Galaxy translation that keeps aligner choice as a runtime `when:`-gated swap has little corpus precedent. The engine supports the mechanism, and the missing precedent is not a reason to cut the choice or split the workflow. A draft brief can record it as evidence for the reviewer. ## Confirmed engine behavior @@ -103,7 +105,7 @@ The engine's design here is more deliberate and more tolerant than the working m ## When this note's posture stops working -- **A pipeline whose conditionals don't cleanly bucket either way.** If the heuristics keep producing "mixed signals" across most conditionals, the source pipeline is probably one where sibling Galaxy workflows really are the right answer at v1, and the translation Mold should surface that as an Open Question on the brief rather than force one workflow. +- **A pipeline whose conditionals don't cleanly bucket either way.** If the heuristics keep producing "mixed signals" across most conditionals, the translation Mold should surface that as an Open Question on the brief. Consider sibling workflows only when one workflow can't carry the branches, not because of a tool choice alone. - **Galaxy invocation engine doesn't behave the way this note assumes.** Confirmed against source on 2026-05-08; the engine is robust here. Risk reopens only if a future Galaxy refactor changes `when:` semantics or the `pick_value` module is deprecated. - **A second Nextflow pipeline emulation surfaces a third pattern.** sarek covers tool-swap (`aligner`), boolean phase toggle (`wes`), and selector (`tools`). RNAseq might add quantifier choice (`salmon | star_salmon | hisat2`) — likely a subworkflow case but worth checking. Taxprofiler might add database-driven branching. New patterns sharpen the heuristics. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md index bd4ddc7..b00b1d5 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-collection-patterns.md @@ -13,9 +13,9 @@ tags: - topic/collection-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy collection transformation patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." related_notes: - "[[iwc-transformations-survey]]" - "[[iwc-conditionals-survey]]" @@ -25,10 +25,13 @@ related_patterns: - "[[reshape-relabel-remap-by-collection-axis]]" - "[[fan-in-bundle-consume-and-flatten]]" - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" - "[[sync-collections-by-identifier]]" - "[[harmonize-by-sortlist-from-identifiers]]" - "[[regex-relabel-via-tabular]]" - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" - "[[collection-swap-nesting-with-apply-rules]]" - "[[collection-split-identifier-via-rules]]" - "[[collection-build-list-paired-with-apply-rules]]" @@ -42,45 +45,46 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: collection patterns -This is the runtime-facing map for Galaxy collection transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. -## Cleanup +## Enter or leave a collection -- [[collection-cleanup-after-mapover-failure]] — use `__FILTER_EMPTY_DATASETS__` or `__FILTER_FAILED_DATASETS__` after map-over when empty or errored elements would break downstream steps. -- [[collection-unbox-singleton]] — use `__EXTRACT_DATASET__` with `which: first` when a known one-element collection must become a dataset. +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. -## Map-over Lifecycle Recipes +## Keep mapped siblings aligned -- [[manifest-to-mapped-collection-lifecycle]] — build collection elements from manifest/table rows, map a tool over them, then relabel or reshape outputs. -- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped outputs, sync siblings from surviving identifiers, then gate final reports. -- [[reshape-relabel-remap-by-collection-axis]] — correct the mapped axis after domain fan-out using Apply Rules and deterministic relabeling. -- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection consumer, then flatten or aggregate results. +First identify what changed. Membership, order, and labels are separate decisions. -## Identifiers +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. -- [[sync-collections-by-identifier]] — membership sync: extract identifiers from one collection and filter or relabel a sibling collection. -- [[harmonize-by-sortlist-from-identifiers]] — order sync: sort one sibling collection by another collection's identifier order. -- [[regex-relabel-via-tabular]] — label rewrite: derive new element identifiers in tabular form and apply them with `__RELABEL_FROM_FILE__`. -- [[relabel-via-rules-and-find-replace]] — relabel inside a structural reshape, currently grounded in the influenza fan-out pattern. +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. -## Structural Reshape +## Change the collection axes -- [[collection-flatten-after-fanout]] — collapse nested collection output to a flat list when the outer axis no longer matters. -- [[collection-build-named-bundle]] — assemble individual outputs into a named collection bundle for publishing or downstream fan-in. -- [[collection-swap-nesting-with-apply-rules]] — use Apply Rules to swap `list:list` axes. -- [[collection-split-identifier-via-rules]] — use Apply Rules regex columns to derive nested list identifiers from one identifier string. -- [[collection-build-list-paired-with-apply-rules]] — use Apply Rules to promote identifier columns into `list:paired` structure. +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. -## Bridges +## Follow a map-over lifecycle -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. ## See also diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md index 23fae29..2dfe890 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md @@ -12,9 +12,9 @@ tags: - topic/galaxy-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy when and pick_value conditional patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." related_notes: - "[[iwc-conditionals-survey]]" related_patterns: @@ -30,29 +30,33 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: conditionals patterns -This is the runtime-facing map for Galaxy conditional workflow choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. -## Direct Gates +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. -- [[conditional-run-optional-step]] — expose or derive a boolean, connect it as `inputs.when`, and use `when: $(inputs.when)` to skip optional steps. -- [[conditional-gate-on-nonempty-result]] — compute a boolean from empty/non-empty dataset or collection state before gating downstream reporting/export. The MGnify recipe is corpus-backed but clunky pending verified-pattern workflow work. +## Skip work without replacing its output -## Routes and Fallbacks +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. -- [[conditional-route-between-alternative-outputs]] — run one of several `when`-gated alternatives, then merge possible outputs with `pick_value`. -- [[conditional-transform-or-pass-through]] — optionally transform a value, then use `pick_value` to choose transformed output or original fallback. +## Continue with one value -## Not Conditionals +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. -- [[collection-cleanup-after-mapover-failure]] — use collection filters when empty or failed mapped elements should be dropped or replaced. This is collection-state cleanup, not a `when` gate. -- `__FILTER_NULL__` — catalog capability for null outputs after conditional steps, but no IWC-backed pattern candidate in the conditionals survey. Zero uptake is not itself an anti-pattern call. +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. ## See Also - [[iwc-conditionals-survey]] — conditionals survey and evidence trail. -- [[galaxy-collection-patterns]] — companion MOC for collection transforms and cleanup. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md index 8123493..de9ec07 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-interval-patterns.md @@ -13,8 +13,8 @@ tags: - topic/interval-transform status: draft created: 2026-06-10 -revised: 2026-08-03 -revision: 2 +revised: 2026-09-23 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." related_notes: - "[[iwc-interval-operations-survey]]" @@ -34,6 +34,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md index 445ed6c..591b3f5 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md @@ -13,8 +13,8 @@ tags: - topic/tabular-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-10-15 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." related_notes: - "[[iwc-tabular-operations-survey]]" @@ -40,38 +40,39 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: tabular patterns -This is the runtime-facing map for Galaxy tabular transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation pages are the actionable references. +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. -## Row And Column Operations +## Rows and columns in one table -- [[tabular-filter-by-column-value]] — keep/drop rows by string column value with `Filter1`. -- [[tabular-filter-by-regex]] — keep/drop rows by regex or pattern matching. -- [[tabular-cut-and-reorder-columns]] — project and reorder columns with `Cut1`. -- [[tabular-compute-new-column]] — compute row-wise values into a new column. +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. -## Joins And Aggregation +## Combine or summarize tables -- [[tabular-join-on-key]] — join two tabular datasets by key columns. -- [[tabular-group-and-aggregate-with-datamash]] — group and aggregate rows with `datamash_ops`. -- [[tabular-sql-query]] — use SQL when filtering, joining, and projection are clearer as one query. +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. -## Text-Processing Recipes +## Text-processing recipes -- [[tabular-prepend-header]] — add a header row with an awk/text-processing step. -- [[tabular-synthesize-bed-from-3col]] — build BED from three-column tabular inputs. -- [[tabular-split-taxonomy-string]] — split taxonomy-like strings into useful columns. -- [[tabular-relabel-by-row-counter]] — synthesize row labels from row order. +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. -## Bridges +## Between tables and collections -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. ## See also diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json index 3a39f43..dd3a94a 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json @@ -75,7 +75,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_version": { "anyOf": [ @@ -108,7 +108,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "out": { "anyOf": [ @@ -309,7 +309,7 @@ ] }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "release": { "anyOf": [ @@ -412,22 +412,28 @@ "items": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" } ] }, @@ -437,22 +443,28 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" }, { "type": "string" @@ -927,7 +939,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "label": { "anyOf": [ @@ -1040,7 +1052,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_state": { "anyOf": [ @@ -1856,7 +1868,9 @@ "text", "File", "data", - "collection" + "collection", + "color", + "directory_uri" ], "type": "string" } @@ -1952,10 +1966,10 @@ ] }, "restrictions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "suggestions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "type": { "enum": [ @@ -1963,6 +1977,9 @@ "string" ], "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" } }, "required": [ @@ -1971,6 +1988,96 @@ "type": "object" }, "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { "additionalProperties": false, "properties": { "default": { @@ -2016,7 +2123,7 @@ "required": [], "type": "object" }, - "Shared10": { + "Shared11": { "additionalProperties": false, "properties": { "default": { @@ -2124,7 +2231,7 @@ ], "type": "object" }, - "Shared11": { + "Shared12": { "additionalProperties": false, "properties": { "default": { @@ -2230,7 +2337,7 @@ "required": [], "type": "object" }, - "Shared12": { + "Shared13": { "additionalProperties": false, "properties": { "default": { @@ -2337,7 +2444,7 @@ ], "type": "object" }, - "Shared13": { + "Shared14": { "additionalProperties": false, "properties": { "default": { @@ -2424,7 +2531,143 @@ ], "type": "object" }, - "Shared14": { + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { "anyOf": [ { "items": { @@ -2457,7 +2700,7 @@ } ] }, - "Shared15": { + "Shared18": { "anyOf": [ { "type": "null" @@ -2488,11 +2731,11 @@ } ] }, - "Shared16": { + "Shared19": { "anyOf": [ { "items": { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, "type": "array" }, @@ -2500,7 +2743,7 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, { "type": "string" @@ -2522,7 +2765,7 @@ } ] }, - "Shared17": { + "Shared20": { "anyOf": [ { "type": "null" @@ -2610,7 +2853,7 @@ "$ref": "#/$defs/Shared1" }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "uuid": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-summary-to-galaxy-template/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/SKILL.md b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/SKILL.md new file mode 100644 index 0000000..ab53aa4 --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/SKILL.md @@ -0,0 +1,84 @@ +--- +name: nextflow-summary-to-workflow-brief +description: "Turn a nextflow source summary and selected scope into a Workflow Brief without Galaxy design assumptions." +--- + +# nextflow-summary-to-workflow-brief + +Follow the procedure below and use the artifact/reference sections as the runtime contract. + +## When To Use + +- Turn a nextflow source summary and selected scope into a Workflow Brief without Galaxy design assumptions. + +## Inputs + +- Read artifact `summary-nextflow`. Schema: summary-nextflow. Produced by `summarize-nextflow`. Source evidence supplied by the preceding summarizer; preserve uncertainty and the caller’s selected scope. + +## Outputs + +- Write artifact `workflow-brief` as `workflow-brief.md`. Format: `markdown`. Schema: workflow-brief-schema. Expert-editable workflow intent and agent environment, with optional details and explicit workflow or environment blockers. + +## Required Tools + +- **`foundry`** (foundry). `npm install -g @galaxy-foundry/gxwf-foundry`. + Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. + Check: `foundry --help`. + Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md + +## Load Upfront + +- `references/notes/workflow-brief-design.md`: Research note copied verbatim into the bundle. Apply the brief boundaries: evidence and expert intent, no inferred Galaxy design, and separate workflow/environment blockers. +- `references/schemas/summary-nextflow.schema.json`: Schema file copied verbatim into the bundle. Read and validate the structured source evidence without turning its processes or containers into Galaxy choices. +- `references/schemas/workflow-brief-schema.schema.json`: Schema file copied verbatim into the bundle. Carry the Markdown section declaration and its structural validator. + +## Load On Demand + +- `references/cli/check-workflow-brief.json`: CLI command reference packaged as a sidecar. Report declared blockers separately from structural validity. Use when: after structural validation and before reporting whether the brief is clear of declared blockers. +- `references/cli/validate-workflow-brief.json`: CLI command reference packaged as a sidecar. Validate the emitted brief before returning it. Use when: after writing workflow-brief.md. + +## Validation + +- Before starting, check that the installed validator carries the workflow-brief-schema schema this bundle was cast with: run `foundry validate-workflow-brief --schema-sha256 8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `workflow-brief.md` before returning it: run `foundry validate-workflow-brief --schema-sha256 8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0 workflow-brief.md` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-workflow-brief --schema-sha256 8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0 workflow-brief.md`. This checks artifact `workflow-brief` against the workflow-brief-schema schema. + +## Procedure + +Read the structured `summary-nextflow` and the caller’s selected scientific scope, then emit a Workflow Brief for expert review. + +Record the selected analyses and source pin. Keep process topology, source container references, parameter evidence, reference assets, and test candidates in the linked source summary. A Nextflow process is not evidence of an available Galaxy wrapper or of a required Galaxy step. + +### Evidence and scope + +Read the source summary and the caller's stated objective and selection. Identify which scientific work is included and excluded; do not copy every source analysis into scope. If the selection is too ambiguous to proceed, state that uncertainty explicitly in Workflow/Blockers rather than choosing an analysis for the expert. + +Write `workflow-brief.md` with `## Workflow`, `### Objective`, and `## Agent Environment`. Add optional subsections only when evidence or expert intent supports them. Identify the source and caller instructions, distinguish source facts from expert requirements, and keep scientific inputs, outputs, and acceptance criteria high-level. Missing optional detail does not require filler sections. + +The writing agent must not infer Galaxy datatypes, collections, interface labels, available tools, or wrapper versions. It must not write step graphs, concrete tool states, or final test declarations. Named source tools and formats are evidence, not Galaxy choices. Explicit expert-provided Galaxy requirements may be recorded with attribution. + +### Agent environment + +Use supplied environment evidence or harmless availability/version probes to describe expected `gxwf`, `foundry`, and `planemo` tooling, agent constraints, container preferences, and Docker/Singularity/Apptainer usability. Distinguish requested requirements, observed facts, and unknowns. Do not claim a container engine works based only on an executable being on PATH. Do not install tools or change the environment as part of brief production. + +Place known issues preventing work in `### Blockers` under the appropriate parent. Put other unresolved choices in Workflow/Open questions, each with a proposed answer and its evidence. Omit or leave Blockers empty when there are none. Any text there, including “none”, is blocking. + +### Validate and hand off + +Run `foundry validate-workflow-brief workflow-brief.md`. Fix structural failures without inventing scientific intent. Run `foundry check-workflow-brief workflow-brief.md --json` and retain its result in the run report. Exit `4` means a valid but blocked brief: return that brief and its blockers for expert/environment resolution. Do not suppress the blockers to obtain a clear check. + +Return the brief for expert editing and review. Producing a brief does not authorize Galaxy design or implementation, and a clear static check does not replace review or environment preflight. Keep source evidence available to the later consumer. + +## Feedback Mode + +- Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. +- When enabled, read `_feedback.md` before doing the work and use its registered `foundry-feedback.ledger.yml` protocol. +- Preserve harness-owned run and phase state. Append only concrete observations about a canonical Foundry source asset or a related project that this run showed to be at fault; do not put ordinary workflow requirements in this ledger. +- Before reporting completion, make one explicit pass over the work you just did. Do not ask yourself whether anything was unclear — recall what happened: where you guessed at something the instructions should have settled, needed information this bundle does not carry, hit an instruction that contradicted another or contradicted the artifacts in front of you, used a packaged reference that did not cover your case, or did something the procedure never describes. +- Append an entry for each such event that clears the protocol's bar. If none do, append nothing and report `no feedback` explicitly. Silence and a clean pass are not the same thing, and nothing downstream can tell them apart unless you say which one it was. +- Pass the same ledger path to any subagent used for this work, and merge updates serially so one writer cannot overwrite another. + +## Runtime Notes + +- Do not read Foundry source files at runtime; use only files packaged in this skill bundle and user-supplied artifacts. +- Preserve declared artifact filenames unless the user or harness supplies explicit paths. +- Carry unresolved assumptions into the output artifact instead of silently inventing missing source evidence. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_feedback.md b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_feedback.md new file mode 100644 index 0000000..61a4bd4 --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_feedback.md @@ -0,0 +1,143 @@ +# Foundry feedback ledger + +The `foundry-feedback-ledger` is an opt-in runtime artifact for observations about the Foundry +assets used during a real cast-skill or pipeline run. It records actionable gaps, defects, +friction, and wishes about Molds and their authored references. It does not record unmet +requirements in the workflow being built; those belong in the open-requirements ledger. + +Every entry is destined for `galaxyproject/foundry`. When a run shows that a related project is +at fault — Galaxy, a tool wrapper, the IWC corpus, a harness CLI — that observation still belongs +here, recorded with the analysis that points at the upstream. The Foundry is where the run +context lives, so it is where the issue is tracked; a maintainer forwards from there. Never tell +a user to take an observation to another tracker instead. + +The registered filename is `foundry-feedback.ledger.yml`. Feedback-aware callers pass the same +file through every skill invocation in a run. + +## Ledger shape + +```yaml +run: + pipeline: nextflow-to-galaxy + run_slug: sarek-galaxy + status: running + phases: + - { n: 1, kind: mold, skill: summarize-nextflow, status: done, feedback_checked: true } + - { n: 2, kind: mold, skill: nextflow-summary-to-galaxy-interface, status: running } +entries: + - id: channel-shape-note-silent-on-optional-outputs + raised_by: nextflow-summary-to-galaxy-interface + observed_in: + mold: nextflow-summary-to-galaxy-interface + path: content/molds/nextflow-summary-to-galaxy-interface/index.md + revision: 6 + content_hash: 82d8... + foundry_head: d9e0b09 + subject: + kind: research + label: nextflow-to-galaxy-channel-shape-mapping + locator: content/research/nextflow-to-galaxy-channel-shape-mapping/index.md + content_hash: a18c... + kind: gap + severity: major + what: "The note does not cover optional outputs." + expected: "State how an absent optional channel maps to the Galaxy interface." + evidence: "Interface design stopped at the optional output decision." + status: open + issue: null +``` + +Entry `kind` is `gap`, `defect`, `friction`, or `wish`. Severity is `blocker`, `major`, or +`minor`. Status is `open`, `filed`, `duplicate`, or `wontfix`. + +## Run lifecycle + +For a pipeline run, the harness creates the file before phase 1 with `run.status: running`, an +empty `entries` list, and the complete top-level phase roster from its assembly record. All +phases begin `pending`. The harness changes a phase to `running` immediately before invocation +and to `done` on success. A phase that terminates the run and the run itself become `failed`; a +graceful user stop makes the run `cancelled`. A hard interruption naturally leaves the run +`running`, which is distinguishable from success without a recovery write. + +A phase also carries `feedback_checked`. The harness sets it true only when that phase reported a +feedback outcome — entry ids it appended, or an explicit `no feedback`. It is absent or false +otherwise, including for a phase that simply never said. This is what separates a run that looked +for friction and found none from a run that never looked. + +Loop phases remain one row and increment `iterations` after each successful iteration. Branch +phases remain one row and record the chosen path in `selected`. After every intended phase is +done, the harness sets `run.status: complete`. + +For a standalone skill invocation, the first feedback-aware skill creates a ledger with +`pipeline: null`, the caller's run slug when one exists, and one phase row for itself. It owns +that row's lifecycle as well as any entry it appends. + +No ledger means feedback mode was off. An empty `entries` list is evidence of a clean run only +when `run.status` is `complete` **and** every phase is `feedback_checked`. A complete run with +unchecked phases is complete but unreviewed; an empty incomplete ledger is neither. + +## When to append + +Appending is a deliberate pass, not a background reflex. Before a skill reports completion it +recalls what actually happened during its work — where it guessed at something the instructions +should have settled, needed information its bundle does not carry, hit an instruction that +contradicted another or contradicted the artifacts in front of it, used a reference that did not +cover the case, or did something its procedure never describes. Those are recallable events. "Was +anything unclear?" is not a useful question to ask at that moment and reliably answers itself no. + +The pass ends in one of two outcomes, and the outcome is always stated: entries appended, or +`no feedback`. A skill that says nothing has not reported a clean pass. + +Two entry shapes are admissible. Both require that `what` names a concrete problem, `expected` +states a proposed correction, and the entry is not already present in this ledger for the same +subject and correction. + +**A Foundry source asset.** The observation concerns Foundry-authored content, implementation, or +a registry that could be changed directly, and `subject.locator` canonically identifies it. This +is the ordinary case and the one the clustering and hash checks are built for. + +**A related project.** The run showed that a project the Foundry drives or cites is at fault — +Galaxy, a Tool Shed wrapper, the IWC corpus, `gxwf`, planemo. Use `subject.kind: related-project`, +name the project and the specific component in `label`, and put its public identity in +`locator` (a repository URL, a tool id, an IWC workflow id). There is no `content_hash`. State +in `what` and `evidence` what the run actually observed, and in `expected` what the upstream +would have to do. The entry is still filed against the Foundry. + +Do not use feedback entries for ordinary workflow requirements, tool-selection uncertainty, or +an unsupported preference with nothing concrete to change. + +## Canonical identity and provenance + +For a Foundry subject, use a repository-relative authored path as `subject.locator`; for a +package-export schema, use `package://#`. Generated skills and harnesses are +never subjects; identify the Mold, Pipeline, caster, assembler, or registry that must change. + +Initial subject kinds are `mold`, `pipeline`, `pattern`, `source-pattern`, `research`, `schema`, +`prompt`, `cli-tool`, `cli-command`, `meta`, `implementation`, and `registry`, plus +`related-project` for the upstream case above. `label` is for display only; the locator is the +identity and clustering key. + +A `related-project` subject is the one kind whose locator is not a Foundry path: it is public +upstream identity, carries no content hash, and is never checked against Foundry main. + +For the running Mold, copy `mold.name`, `mold.path`, `mold.revision`, and `mold.content_hash` +from `_provenance.json` into `observed_in`. Record its `mold.commit` value as `foundry_head`, but +never use that commit as exact content identity: it can be null or predate the bytes in the cast. +For a packaged reference, copy its `refs[].src` and `refs[].src_hash` into the subject locator and +content hash. When the subject is the Mold itself, use its authored path and +`observed_in.content_hash`. + +## Safe updates + +Read the complete ledger immediately before changing it. Preserve run state and all entries you +do not own. Append a complete entry in one rewrite; do not leave a partial YAML document. The +harness is the sole writer of pipeline lifecycle fields, while each skill writes only entries. + +Keep evidence concise and safe to upstream. Never copy credentials, tokens, private repository +URLs, proprietary source text, patient or participant data, or user-identifying filesystem +paths into the ledger. Replace a run-directory prefix with `/` and summarize sensitive +evidence instead of quoting it. + +The reporting skill may set an open entry to `filed`, `duplicate`, or `wontfix` and record the +resolved issue URL after the corresponding disposition is confirmed. It never deletes an entry; +the ledger remains the run audit trail. diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_provenance.json b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_provenance.json new file mode 100644 index 0000000..6e19f17 --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_provenance.json @@ -0,0 +1,111 @@ +{ + "provenance_schema_version": 4, + "cast_target": "claude", + "mold": { + "name": "nextflow-summary-to-workflow-brief", + "path": "content/molds/nextflow-summary-to-workflow-brief/index.md", + "revision": 2, + "content_hash": "c2761e698f7913963c3324379d597b72cd6af456baf08742507ebf31b1b7d599", + "commit": "871b5e6deadfd56aeb5e2ba92eb8f6f50c8449ee" + }, + "cast_at": "2026-11-10T14:49:42.424Z", + "refs": [ + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[check-workflow-brief]]", + "src": "content/cli/foundry/check-workflow-brief.md", + "dst": "references/cli/check-workflow-brief.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Report declared blockers separately from structural validity.", + "trigger": "After structural validation and before reporting whether the brief is clear of declared blockers.", + "verification": "Check the committed fixtures and review a worked run for faithful evidence and unchanged brief input.", + "src_hash": "7fd8bccdd0f5a2a62cc51e00589e9e263ce633a4c9a75ba3753d4d01ae822895", + "dst_hash": "a5ad5bcca89d0e0db6f2e4503bf48cfec962c5282e89d2d9a3b3738eda11e8b3", + "source": "deterministic" + }, + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[validate-workflow-brief]]", + "src": "content/cli/foundry/validate-workflow-brief.md", + "dst": "references/cli/validate-workflow-brief.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Validate the emitted brief before returning it.", + "trigger": "After writing workflow-brief.md.", + "verification": "Check the committed fixtures and review a worked run for faithful evidence and unchanged brief input.", + "src_hash": "85992477f91f2131695e7d9ec5d798f99e4800b276aaefd1d763aeb2b10ea105", + "dst_hash": "4e4a4cb44d6d34f90080ecc6f9cdaff68abce96babfdcf289acfa7337cf8b737", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[workflow-brief-design]]", + "src": "content/research/workflow-brief-design/index.md", + "dst": "references/notes/workflow-brief-design.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Apply the brief boundaries: evidence and expert intent, no inferred Galaxy design, and separate workflow/environment blockers.", + "verification": "Check the committed fixtures and review a worked run for faithful evidence and unchanged brief input.", + "src_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "dst_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[summary-nextflow]]", + "src": "package://@galaxy-foundry/summarize-nextflow#summaryNextflowSchema", + "dst": "references/schemas/summary-nextflow.schema.json", + "used_at": "both", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Read and validate the structured source evidence without turning its processes or containers into Galaxy choices.", + "license": "MIT", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[workflow-brief-schema]]", + "src": "package://@galaxy-foundry/gxwf-foundry#workflowBriefSchema", + "dst": "references/schemas/workflow-brief-schema.schema.json", + "used_at": "both", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Carry the Markdown section declaration and its structural validator.", + "src_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "dst_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "source": "deterministic" + } + ], + "artifacts": { + "produces": [ + { + "id": "workflow-brief", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "description": "Expert-editable workflow intent and agent environment, with optional details and explicit workflow or environment blockers." + } + ], + "consumes": [ + { + "id": "summary-nextflow", + "description": "Source evidence supplied by the preceding summarizer; preserve uncertainty and the caller’s selected scope.", + "inherited_schema": "[[summary-nextflow]]", + "producers": [ + "summarize-nextflow" + ] + } + ] + } +} diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_required_tools.json b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_required_tools.json new file mode 100644 index 0000000..b6d5500 --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_required_tools.json @@ -0,0 +1,16 @@ +[ + { + "tool": "foundry", + "origin": "npm", + "package": "@galaxy-foundry/gxwf-foundry", + "invoke": "foundry", + "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", + "availability_check": "foundry --help", + "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", + "source": "implied", + "implied_by": [ + "foundry check-workflow-brief", + "foundry validate-workflow-brief" + ] + } +] diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_verify.json b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_verify.json new file mode 100644 index 0000000..52b1e1b --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/_verify.json @@ -0,0 +1,33 @@ +{ + "verify_schema_version": 1, + "entries": [ + { + "artifact_id": "summary-nextflow", + "direction": "input", + "kind": "json", + "default_filename": "summary-nextflow.json", + "schema": "[[summary-nextflow]]", + "validator_bin": "foundry", + "args": [ + "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "{artifact_path}" + ] + }, + { + "artifact_id": "workflow-brief", + "direction": "output", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "validator_bin": "foundry", + "args": [ + "validate-workflow-brief", + "--schema-sha256", + "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "{artifact_path}" + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/cli/check-workflow-brief.json b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/cli/check-workflow-brief.json new file mode 100644 index 0000000..f0c765e --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/cli/check-workflow-brief.json @@ -0,0 +1,31 @@ +{ + "type": "cli-command", + "tool": "foundry", + "command": "check-workflow-brief", + "summary": "Check Workflow Brief structure and stop on explicit workflow or agent environment blockers.", + "source_path": "content/cli/foundry/check-workflow-brief.md", + "source_revision": 1, + "package": "@galaxy-foundry/gxwf-foundry", + "description": "Check Workflow Brief structure and explicit workflow/environment blockers.", + "synopsis": "foundry check-workflow-brief [options] ", + "args": [ + { + "raw": "brief.md", + "name": "brief.md", + "required": true, + "variadic": false + } + ], + "options": [ + { + "flags": "--json", + "name": "json", + "description": "Emit machine-readable structure and blocker diagnostics", + "takesArgument": false, + "optionalArgument": false, + "negatable": false, + "defaultValue": false + } + ], + "body": "# `foundry check-workflow-brief`\n\nRead a [[workflow-brief-design]] document and check its structure and declared blockers.\n\n## Output\n\nExit `0` means structurally valid with no declared blockers. Structural failure exits `3`; a valid brief with nonempty Blockers under Workflow or Agent Environment exits `4`. Input read failures exit `1`.\n\nWithout `--json`, ready status prints on stdout; errors and blockers print on stderr. With `--json`, stdout contains `valid`, `ready`, `errors`, and `blockers`. Blockers identify their parent section, heading line, and Markdown body. Input read errors still print on stderr.\n\n## Examples\n\n```sh\nfoundry check-workflow-brief workflow-brief.md\nfoundry check-workflow-brief workflow-brief.md --json\n```\n\n## Gotchas\n\n- Any content in `### Blockers` blocks work, including “none”. Omit or leave the section empty when there are no blockers. Comments and separators alone are empty.\n- The two Blockers sections are scoped to their respective Workflow and Agent Environment parents.\n- A clear check establishes document shape and absence of declared blockers. Expert review, semantic completeness, and observed tooling/container availability still need their own checks.\n- This command reads the document without editing it." +} diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/cli/validate-workflow-brief.json b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/cli/validate-workflow-brief.json new file mode 100644 index 0000000..cef0d8c --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/cli/validate-workflow-brief.json @@ -0,0 +1,31 @@ +{ + "type": "cli-command", + "tool": "foundry", + "command": "validate-workflow-brief", + "summary": "Validate the title and required sections of a Markdown Workflow Brief.", + "source_path": "content/cli/foundry/validate-workflow-brief.md", + "source_revision": 4, + "package": "@galaxy-foundry/gxwf-foundry", + "description": "Validate a Workflow Brief Markdown document (experimental RFC).", + "synopsis": "foundry validate-workflow-brief [options] [brief.md]", + "args": [ + { + "raw": "brief.md", + "name": "brief.md", + "required": false, + "variadic": false + } + ], + "options": [ + { + "flags": "--schema-sha256 ", + "name": "schemaSha256", + "description": "Exit 4 unless this validator's schema has this sha256 (a cast bundle's _verify.json pin); with no artifact, check only the pin", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + } + ], + "body": "# `foundry validate-workflow-brief`\n\nValidate a Markdown document against the section contract described in [[workflow-brief-design]].\n\n## Output\n\nPrints `: valid` to stdout on success (exit `0`). Missing, duplicate, or empty required sections produce diagnostics on stderr and exit `3`. Input read failures exit `1`. With `--schema-sha256 ` it first compares the hash of the schema it carries against ``, the pin a cast bundle records in `_verify.json`: a mismatch exits `4` without validating, because the validator and the bundle disagree about the contract (see [[schema-packages]]). Given the pin and no artifact, it checks only the pin.\n\n## Examples\n\n```sh\nfoundry validate-workflow-brief workflow-brief.md\n```\n\n## Gotchas\n\n- This checks the document title, required sections, their content, and parent-scoped subsections. It does not verify semantic completeness, expert approval, or environment readiness.\n- Headings inside code fences, block quotes, and HTML comments do not satisfy section requirements.\n- A valid brief may contain blockers. Use `foundry check-workflow-brief` to stop on workflow or environment blockers.\n- Additional sections, prose, tables, and explicit unknowns are allowed. Heading case and section order are flexible." +} diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/notes/workflow-brief-design.md b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/notes/workflow-brief-design.md new file mode 100644 index 0000000..8bc1e6d --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/notes/workflow-brief-design.md @@ -0,0 +1,96 @@ +--- +type: research +title: "Workflow Brief" +tags: + - meta +status: draft +created: 2026-09-17 +revised: 2026-10-19 +revision: 6 +summary: "Markdown Workflow Brief definition and section contract for scope, constraints, agent environment, and explicit blockers." +--- + +# Workflow Brief + +A Workflow Brief is an expert-editable document describing what a workflow should achieve, what it should leave out, and the constraints under which an agent will build and run it. It is an input to design and implementation. The draft workflow captures the proposed steps, wiring, and tool state. + +## Document shape + +The brief is Markdown with a title and two sections: `## Workflow` and `## Agent Environment`. The workflow needs an objective; most details are optional. The agent environment must contain a description, which can explicitly record what still needs to be checked. + +The `workflowBriefSchema` declaration uses the shared engine described in [[markdown-document-contract]] and lives in `packages/gxwf-foundry/src/workflow-brief.ts`. An illustrative document lives in `packages/gxwf-foundry/test/fixtures/workflow-brief/read-alignment.md`. + +| Heading | Required | Content | +| --- | --- | --- | +| `## Workflow` | Yes | Scientific intent and scope. | +| `### Objective` | Yes, under Workflow | Scientific goal and desired outcome. | +| `### Sources` | No | Papers, interviews, repositories, relevant source pins, and evidence availability. | +| `### Scope` | No | Optional `#### Included` and `#### Excluded` describe selected work and boundaries. | +| `### Inputs and outputs` | No | Scientific roles and source-supported formats or organization. | +| `### Requirements and preferences` | No | Expert or source requirements, their strength, rationale, and evidence. | +| `### Acceptance criteria` | No | High-level behaviors that establish success. | +| `### Open questions` | No | Unresolved choices that do not prevent work, each with a proposed answer and its evidence. | +| `### Decisions` | No | Expert decisions and rationale. | +| `### Related artifacts` | No | Links to source evidence and other relevant inputs. | +| `### Blockers` | No, under Workflow | Unresolved issues that prevent design or implementation. Any content blocks work. | +| `## Agent Environment` | Yes | Agent tooling and operating conditions. | +| `### Tooling` | No | Expected Foundry tooling and versions, and what is known about availability. | +| `### Constraints` | No | Agent workspace, network, installation, resource, and access restrictions. | +| `### Containerization` | No | Container preference and Docker, Singularity, or Apptainer availability. | +| `### Blockers` | No, under Agent Environment | Environment issues that prevent work. Any content blocks work. | + +Headings are case-insensitive; section order is flexible and additional sections are allowed. Recognized sections are unique within their parent. If present, ordinary sections need content beyond headings, comments, or separators. Blockers may be empty. Scope's Included and Excluded subsections are optional. + +## Workflow content + +The writing agent must record evidence and expert intent without making Galaxy design choices. It must not assume Galaxy datatypes, dataset collection structures, tool availability, interface labels, or concrete tool versions. Source formats and explicitly named source tools can be cited as evidence; they do not establish a Galaxy mapping or an installed wrapper. An expert may supply specific Galaxy requirements here, and the agent must identify them as expert-provided rather than inferred. + +The agent must not write a step graph, concrete tool state, or final test declarations into the brief. Interface design, data-flow design, tool discovery, and test development belong to the downstream design and implementation Molds. Inputs and outputs stay at scientific roles and source-supported descriptions. Acceptance criteria stay high-level; detailed fixtures and assertions belong to test development. + +Requirements and preferences describe what the expert or source asks for. Agent Environment Constraints describe restrictions on the agent's work. Uncertainty that prevents proceeding belongs in Blockers. Record every other unresolved choice in Open questions with a proposed answer and the evidence for it, such as source behavior, the expert's wording, or the available test data. Propose the answer the evidence best supports. Where expert intent and source behavior conflict, propose the expert's intent and cite the conflict. When the evidence is thin, still propose an answer, mark it low confidence, and say what evidence is missing. The expert accepts a proposal by leaving it unedited and overrides it by editing the answer. + +Write each open question as a bullet with `Proposed:` and `Evidence:` sub-bullets. Add `Confidence: low` and `Missing:` when the evidence is thin: + +```md +- Should runs of one sample be merged before trimming? + - Proposed: yes, merge them as the source does. + - Evidence: the source merges sample-sheet rows sharing a sample ID. The selection does not mention merging. + - Confidence: low + - Missing: whether the expert's samples ever span several runs. +``` + +## Agent environment + +Tooling records whether expected tools such as `gxwf`, `foundry`, and `planemo` are available at the required versions. Distinguish intended requirements from observed availability. Containerization records the preference and whether Docker, Singularity, or Apptainer is actually usable, rather than assuming that an executable name proves a working engine. + +A separate preflight result should record observed versions, container availability, source/reference access, writable paths, timestamps, and diagnostics. A requested installation strategy is not evidence of an installed executable. + +## Validation and blockers + +```sh +foundry validate-workflow-brief workflow-brief.md +foundry check-workflow-brief workflow-brief.md --json +``` + +Validation checks the title, required sections, unique recognized headings, content, and parent-scoped subsections. Example headings in fences, quotes, lists, or comments do not satisfy requirements. Success exits `0`, structural failures exit `3`, and input read failures exit `1`. + +The static readiness check adds a check for `### Blockers` under both Workflow and Agent Environment. A structurally valid document with content in either section exits `4`; a structurally valid document without blockers exits `0`. Omit or leave Blockers empty when there are no blockers. Do not write “none” there: any content, including that word, is blocking. Comments and separators alone are empty. The JSON result includes `valid`, `ready`, `errors`, and blocker bodies with their parent and source line. + +A clear static check only establishes structure and the absence of declared blockers. It does not establish semantic completeness, expert approval, or successful environment preflight. The harness must stop on declared blockers and apply its own review and preflight checks before implementation. + +The exported `WorkflowBrief` and `WorkflowBriefSection` types describe the parsed document. `WorkflowBriefReadinessResult` describes the static check. These checks read the document without changing it. + +## Lifecycle + +1. A source-to-brief producer assembles evidence, proposed scope, and unanswered questions without Galaxy design assumptions. +2. The expert edits the brief and settles blockers needed before the next stage. +3. The harness checks structure, declared blockers, review, and environment readiness. +4. Brief-to-Galaxy consumes the brief and referenced evidence, then runs design, draft, implementation, and test development. + +Brief-to-Galaxy and every design or implementation step must leave the brief unchanged. Record durable workflow knowledge in `open-requirements.ledger.yml`: evidence as discoveries, authorized choices as decisions, unmet needs as obligations, and changes to expert intent as proposed brief-change recommendations. Ordinary phase progress remains harness state. Updating the brief belongs to a separate expert editing step; an implementation agent must not resolve a blocker by rewriting its input. + +A hand-authored brief is a first-class input. Implementation should work in a fresh session using the document and its referenced artifacts. + +## Open design questions + +- Should expert questions live here or link to a separate decision record? diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/schemas/summary-nextflow.schema.json new file mode 100644 index 0000000..d1b42ad --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/schemas/summary-nextflow.schema.json @@ -0,0 +1,1851 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://galaxyproject.org/foundry/schemas/summary-nextflow.schema.json", + "$comment": "Canonical source: packages/summarize-nextflow/src/schema/summary-nextflow.schema.json in galaxyproject/foundry. Mold frontmatter cites this schema via [[summary-nextflow]] wiki-links; the cast pipeline imports the `summaryNextflowSchema` runtime export and serializes it into cast bundles.", + "title": "Nextflow Pipeline Summary", + "description": "Structured per-source summary emitted by the summarize-nextflow Mold.\n\nPer-source schema by design — paper, Nextflow, and CWL each have their own summary shape; downstream Molds (data flow, templates, tool wrappers) consume any source's summary and handle the polymorphism.\n\nField names mirror gxy-sketches' SketchSource / ToolSpec / TestDataRef / ExpectedOutputRef where parity exists; see content/research/gxy-sketches-alignment/index.md.", + "$ref": "#/$defs/Summary", + "$defs": { + "Summary": { + "title": "Summary", + "description": "Top-level shape. Every Nextflow summary is exactly this object.", + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "params", + "sample_sheets", + "profiles", + "tools", + "processes", + "subworkflows", + "workflow", + "reference_assets", + "reference_rebuilds", + "test_candidates", + "test_selection" + ], + "properties": { + "source": { + "$ref": "#/$defs/SourceRecord" + }, + "params": { + "type": "array", + "items": { + "$ref": "#/$defs/Param" + } + }, + "sample_sheets": { + "type": "array", + "items": { + "$ref": "#/$defs/SampleSheet" + }, + "description": "Structured sample-sheet inputs. Each entry binds one `params[]` parameter to a row schema (column names, types, path-vs-meta classification, required flags, enums, patterns). Promoted from prose inside `params[].description` so downstream target translations (Galaxy `sample_sheet*` collections, CWL records-of-arrays) can choose collection variants without re-parsing the source pipeline. Empty array when no sample-sheet idiom is detected. Discovery sources: nf-schema `schema:` references, `samplesheetToList()` calls, and `splitCsv(header: true)` materializations." + }, + "profiles": { + "type": "array", + "items": { + "$ref": "#/$defs/Profile" + }, + "description": "Every profile declared in a `profiles { }` block, with the role(s) each one plays. Kinds are classified from the profile body (and its resolved `includeConfig` chain), not from the profile name: a name-shaped guess mislabels both directions — `replikation/What_the_Phage`'s `test` profile only flips a mode boolean, and `nf-core/sarek`'s `mutect` profile includes `conf/test_mutect2.config`. Across the 26-pipeline fixture corpus 103 of 371 profiles carry profile-level test-input evidence. This evidence feeds candidate discovery but does not replace test-case selection; mode profiles can also encode target-scope choices, while container and executor profiles describe the source execution environment." + }, + "tools": { + "type": "array", + "items": { + "$ref": "#/$defs/Tool" + } + }, + "processes": { + "type": "array", + "items": { + "$ref": "#/$defs/Process" + } + }, + "subworkflows": { + "type": "array", + "items": { + "$ref": "#/$defs/Subworkflow" + } + }, + "workflow": { + "$ref": "#/$defs/Workflow" + }, + "reference_assets": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferenceAsset" + }, + "description": "Curated, target-agnostic view of reference-data inputs the source pipeline consumes (FASTAs, indexes, dictionaries, annotation tables, …). Each entry is a foreign-key projection onto `params[]` plus source-side metadata (asset kind, format hint, source expression, schema group, callers). Empty array when no reference assets are detected. Downstream Molds (e.g. nextflow-summary-to-galaxy-reference-data) classify these against Galaxy translation rungs; classification decisions are not encoded here." + }, + "reference_rebuilds": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferenceRebuildRule" + }, + "description": "Compute-if-missing rebuild branches detected in workflow/subworkflow bodies. Each entry binds an asset param to the guard and builder process that recomputes it when absent. Empty when no rebuild idioms are found. Source evidence only — Galaxy in-tool-rebuild decisions live downstream." + }, + "test_candidates": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/TestCandidate" + }, + "description": "The maximally useful statically recoverable whole-pipeline test cases. When pipeline-level nf-test exists, each literal `test(...)` block is a candidate (dynamically generated files retain one explicit aggregate candidate and a warning). Otherwise input-bearing profiles are candidates; otherwise pipeline defaults form the fallback candidate. Every candidate carries its own resolved profiles, parameter delta, inputs, assertions, scale/coverage classification, and selection disposition." + }, + "test_selection": { + "$ref": "#/$defs/TestSelection", + "description": "The deterministic selection result over `test_candidates[]`. A null selected id is an explicit scope-choice result, never an implicit fallback to the first filename or to a profile named `test`." + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Cast-skill-emitted advisory messages (e.g. DSL1 detected, meta.yml/script disagreement, lossy assertion summarization). Empty array on a clean run." + } + } + }, + "SourceRecord": { + "title": "SourceRecord", + "description": "Provenance block. Mirrors gxy-sketches SketchSource field names so a Foundry summary is structurally consumable by anyone using that shape.", + "type": "object", + "additionalProperties": false, + "required": [ + "ecosystem", + "workflow", + "url", + "version", + "slug" + ], + "properties": { + "ecosystem": { + "type": "string", + "enum": [ + "nf-core", + "nextflow" + ], + "description": "Pipeline flavor. `nf-core` when `manifest.name` is `nf-core/` and the canonical layout is present; `nextflow` for ad-hoc DSL2 pipelines. Superset of gxy-sketches' enum (which adds `iwc`, `snakemake-workflows`, `wdl` for its other ingestors)." + }, + "workflow": { + "type": "string", + "description": "Pipeline name (typically the part after `nf-core/`)." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Git remote URL of the pipeline repository." + }, + "version": { + "type": "string", + "description": "Tag, branch, or commit SHA the summary was derived from." + }, + "license": { + "type": [ + "string", + "null" + ], + "description": "SPDX license identifier when detectable from a LICENSE file." + }, + "slug": { + "type": "string", + "description": "Kebab-case identifier; `-` for nf-core, repo basename otherwise." + } + } + }, + "Profile": { + "title": "Profile", + "description": "One entry from a `profiles { }` block, classified by role. A `test` kind supplies profile-level candidate evidence without claiming that the profile is the selected, cheapest, or most representative test case. Pipeline-level `test_candidates[]` entries derived from nf-test are the higher-fidelity candidate unit because they preserve per-test parameter overrides.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "kinds", + "source_path", + "includes", + "signals" + ], + "properties": { + "name": { + "type": "string", + "description": "Profile name as declared, i.e. the value passed to `-profile`." + }, + "kinds": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "test", + "container", + "executor", + "resources", + "mode", + "dev", + "unknown" + ] + }, + "description": "The roles the profile body evidences. An array rather than a scalar because real profiles combine roles — 25 of the 371 corpus profiles select a container engine *and* an executor in one block (`CRG-CNAG/CalliNGS-NF`'s `cluster` sets `process.container`, `singularity.enabled`, and a `process { }` block with `executor = 'crg'`; seven of `biocorecrg/MOP2`'s eight profiles do the same).\n\nValues: `test` supplies input data matching the recognized test conventions — includes a `conf/test*.config`, or assigns `params.input` / `samplesheet` to a non-null value; `test_full` is this kind too, with its config family visible in `includes`. `container` selects a container or package-management engine (`docker.enabled`, `process.conda`, `process.arch`, ...). `executor` selects an executor, cloud backend, or `workDir`. `resources` tunes process directives only (`memory`, `cpus`, `time`, `withLabel:` selectors). `mode` assigns params that change pipeline behavior without supplying input data (`nf-core/rnaseq`'s `prokaryotic`, `nf-core/smrnaseq`'s protocol presets). `dev` is a developer-ergonomics profile (`debug`, `gitpod`). `unknown` means no classifying signal was found.\n\nNot every role is additive. `test`, `container`, `executor`, and `dev` accumulate. `mode` and `resources` are assigned **only when none of those matched**, because their signals are far weaker — nf-core's `docker` profile assigns `params.use_gpu` inside a ternary, and most test profiles set process resource caps, so accumulating them would label most of a corpus `mode`. A consumer that needs the suppressed evidence reads `signals[]`, where `params-assignment` and `process-directives` survive regardless.\n\n`test` is profile-level candidate evidence, not a final selection or a claim about cost or representative coverage. Prefer nf-test-derived `test_candidates[]` entries when present because they preserve per-test parameter overrides; use an input-bearing profile as a fallback when no pipeline-level nf-test case exists. Mode profiles may encode science-scope choices that target translation must surface, unlike execution-only container and executor profiles.\n\n`test` detection keys on the nf-core input conventions (`conf/test*.config`, `params.input`, `params.samplesheet`). All 103 corpus `test` profiles are in the 16 nf-core fixtures and none in the 10 ad-hoc ones — an ad-hoc pipeline naming its inputs `reads` / `genome` yields no `test` profile even where one exists in spirit. Absence of a `test` kind therefore means 'not detected', not 'none declared'." + }, + "source_path": { + "type": "string", + "description": "Repo-relative path of the config file whose `profiles { }` block declared this profile. Not always `nextflow.config` — `epi2me-labs/wf-human-variation` declares all six in `base.config`, `biocorecrg/MOP2` in `nextflow.global.config`." + }, + "includes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "`includeConfig` targets named directly in the profile body, verbatim and unresolved (e.g. `conf/test_full.config`). Interpolated paths are recorded as written. The classifier follows these to reach the real settings; downstream consumers can use the config family as one ranking signal, but it does not by itself establish runtime cost or representative coverage." + }, + "signals": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "include:test-config", + "sets-input-data", + "container-keys", + "executor-keys", + "params-assignment", + "process-directives", + "dev-profile-name" + ] + }, + "description": "Evidence the classifier acted on, so a surprising `kinds` value is auditable without re-reading the pipeline config — and so evidence suppressed by the `mode` / `resources` fallback rule is still visible. Empty exactly when `kinds` is `[\"unknown\"]`: nothing matched, or the `includeConfig` chain holding the settings was deeper than the resolver follows." + } + } + }, + "Param": { + "title": "Param", + "description": "One pipeline parameter. Sourced from `nextflow.config`'s `params { ... }` block, augmented from `nextflow_schema.json` when present (nf-core).", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "required" + ], + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string", + "description": "JSON Schema-style type when nextflow_schema.json supplies it (string, integer, number, boolean, path); free-form when inferred from a default value." + }, + "default": { + "description": "Default value verbatim from the source. May be null." + }, + "description": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "enum": { + "type": "array", + "items": {}, + "description": "Allowed values when nextflow_schema.json declares them." + }, + "format": { + "type": [ + "string", + "null" + ], + "description": "JSON Schema `format` keyword from nextflow_schema.json. nf-schema vocabulary: `file-path`, `directory-path`, `path`, `file-path-pattern`. Distinguishes path-typed params (datasets/collections) from plain strings without re-reading the source schema. Null when undeclared." + }, + "hidden": { + "type": [ + "boolean", + "null" + ], + "description": "nf-schema `hidden` keyword. True for CLI-plumbing params (e.g. `validate_params`, `pipelines_testdata_base_path`, `version`) that shouldn't surface in user-facing target interfaces. Null when undeclared." + }, + "mimetype": { + "type": [ + "string", + "null" + ], + "description": "nf-schema `mimetype` keyword (e.g. `text/csv`, `text/plain`, `application/gzip`). Useful for seeding Galaxy `format` on path-typed params. Null when undeclared." + }, + "schema_group": { + "type": [ + "string", + "null" + ], + "description": "Human-readable group label from the parent nextflow_schema.json `$defs` section's `title` (e.g. `Input/output options`, `Reference genome options`). Preserves nf-schema sectioning so target interfaces can group inputs without re-parsing the source. Null when the param is not under a titled section." + }, + "fa_icon": { + "type": [ + "string", + "null" + ], + "description": "Font Awesome icon class from the parent `$defs` section's `fa_icon` (e.g. `fas fa-terminal`). Optional UI hint; null when undeclared." + }, + "source_kind": { + "type": [ + "string", + "null" + ], + "enum": [ + "nextflow_schema", + "params_block", + "getGenomeAttribute", + "computed", + null + ], + "description": "How this param entry was derived. `nextflow_schema`: declared in `nextflow_schema.json`. `params_block`: declared (or defaulted) in `nextflow.config`'s `params { ... }` block. `getGenomeAttribute`: synthesized from `params.X = getGenomeAttribute('X')` assignments in `nextflow.config` (nf-core key-expanded bundle pattern). `computed`: derived from another non-getGenomeAttribute expression. Null when provenance was not classified." + }, + "source_expression": { + "type": [ + "string", + "null" + ], + "description": "Verbatim right-hand-side expression when the param was assigned in Groovy (e.g. `getGenomeAttribute('fasta_fai')`). Null when the param comes from a JSON Schema entry only." + }, + "source_path": { + "type": [ + "string", + "null" + ], + "description": "Repo-relative path of the source file where the assignment lives (e.g. `nextflow.config`, `conf/igenomes.config`). Null when undefined or when the param comes only from `nextflow_schema.json`." + } + } + }, + "SampleSheet": { + "title": "SampleSheet", + "description": "One sample-sheet-shaped pipeline input. Captures the row schema, path-vs-meta column classification, and discovery provenance so downstream Molds can map onto Galaxy `sample_sheet[:variant]` collections, CWL records-of-arrays, or other target shapes without re-reading the source.", + "type": "object", + "additionalProperties": false, + "required": [ + "param", + "discovered_via", + "columns" + ], + "properties": { + "param": { + "type": "string", + "description": "Foreign key into `params[].name` — the parameter whose value points at the sample-sheet file." + }, + "schema_path": { + "type": [ + "string", + "null" + ], + "description": "Repo-relative path of the JSON Schema file describing rows (e.g. `assets/schema_input.json`), resolved from the param's nf-schema entry's `schema:` keyword. Null when discovered via `samplesheetToList` without a schema reference, or via `splitCsv(header: true)` ad-hoc parsing." + }, + "discovered_via": { + "type": "string", + "enum": [ + "nf-schema", + "samplesheetToList", + "splitCsv", + "ad-hoc" + ], + "description": "How the sample-sheet shape was detected. `nf-schema`: nextflow_schema.json entry has `schema:` keyword. `samplesheetToList`: workflow imports nf-schema's helper and calls it on the param. `splitCsv`: workflow uses `splitCsv(header: true)` on the param's path. `ad-hoc`: pipeline-specific CSV/TSV parsing inferred from script bodies." + }, + "format": { + "type": [ + "string", + "null" + ], + "enum": [ + "csv", + "tsv", + "csv-or-tsv", + null + ], + "description": "Tabular format declared by the param's `mimetype` or inferred from `samplesheetToList` / `splitCsv` semantics. Null when undeclared." + }, + "header": { + "type": [ + "boolean", + "null" + ], + "description": "Whether the file is expected to have a header row. Null when the discovery source doesn't pin it." + }, + "columns": { + "type": "array", + "items": { + "$ref": "#/$defs/SampleSheetColumn" + }, + "description": "Row schema in declaration order. nf-schema's `samplesheetToList` emits columns in property order, not source-column order — preserve property order here so downstream translations match runtime channel item layout." + } + } + }, + "SampleSheetColumn": { + "title": "SampleSheetColumn", + "description": "One column in a sample-sheet row schema. Type vocabulary is JSON Schema-style scalar; `kind` separates dataset-reference columns from per-row metadata. Validation hints (`pattern`, `enum`, `exists`, `mimetype`) are preserved verbatim — target Molds decide which of them survive translation (e.g. Galaxy's `sample_sheet` validator allowlist is regex/in_range/length).", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "kind", + "required" + ], + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "string", + "integer", + "number", + "boolean" + ], + "description": "JSON Schema-style scalar type. Path columns are typed `string` with a `format: file-path|directory-path|path` qualifier — see `format` and `kind`." + }, + "kind": { + "type": "string", + "enum": [ + "data", + "meta" + ], + "description": "`data`: path-typed column whose values are dataset references (becomes element/inner-collection slot in Galaxy `sample_sheet`). `meta`: scalar metadata column (becomes a `column_definitions` entry, populated per row in `columns[]`). Includes the nf-schema `meta:` annotation when present; in its absence inferred from `format` (`file-path`/`directory-path`/`path` → `data`, anything else → `meta`)." + }, + "format": { + "type": [ + "string", + "null" + ], + "description": "JSON Schema `format` keyword. nf-schema vocabulary: `file-path`, `directory-path`, `path`, `file-path-pattern`. Null for scalar metadata columns without a format hint." + }, + "required": { + "type": "boolean" + }, + "default": { + "description": "Default value verbatim from the sample-sheet schema. May be null." + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "enum": { + "type": "array", + "items": {}, + "description": "Allowed values when the column schema declares an `enum`. Empty array when absent." + }, + "pattern": { + "type": [ + "string", + "null" + ], + "description": "JSON Schema regex `pattern` when present." + }, + "exists": { + "type": [ + "boolean", + "null" + ], + "description": "nf-schema `exists` flag for path columns. True when the parser must verify the file exists." + }, + "mimetype": { + "type": [ + "string", + "null" + ], + "description": "nf-schema `mimetype` keyword (e.g. `text/csv`, `application/gzip`). Useful for target datatypes." + } + } + }, + "Tool": { + "title": "Tool", + "description": "One tool/dependency the pipeline runs. Mirrors gxy-sketches ToolSpec (name + version) and augments it with the resolved container/conda strings the bridge to author-galaxy-tool-wrapper needs.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string", + "description": "Canonical tool name (e.g. `fastp`, `samtools`). Used as the foreign key from `processes[].tool`." + }, + "version": { + "type": "string", + "description": "Exact version when the conda spec pins one (`name=1.0`, `name==1.0`, or the space-separated `name 1.0`; a trailing `=` is a conda build string and is not part of the version). The literal `unknown` when no exact pin was declared — either because the spec names no version at all (a legitimate authoring choice, e.g. `conda-forge::r-base`) or because it declares an inexact constraint, in which case `version_constraint` carries it. A spec the parser cannot read at all is reported in `warnings[]` rather than landing here as `unknown`." + }, + "version_constraint": { + "type": [ + "string", + "null" + ], + "description": "Verbatim version constraint when the conda spec declares an inexact one (e.g. `>=1.17`, `>=1.0,<2.0`). Null when the spec pins exactly or names no version. Present so a range is preserved rather than flattened into the `unknown` sentinel; a consumer that needs an exact version should read a non-null constraint as \"the pipeline constrains this tool, but not to a single version\"." + }, + "biocontainer": { + "type": [ + "string", + "null" + ], + "description": "BioContainers image reference when one was found. Includes both `quay.io/biocontainers/:--` and the docker.io alias `biocontainers/:--` (modern nf-core modules typically use the docker.io form in the docker branch and the depot.galaxyproject.org form in the singularity branch)." + }, + "bioconda": { + "type": [ + "string", + "null" + ], + "description": "`bioconda::=` spec when a `conda` directive provides one — directly or by reference to a `${moduleDir}/environment.yml` whose `dependencies:` list contains a single bioconda entry. The latter is the modern nf-core convention." + }, + "docker": { + "type": [ + "string", + "null" + ], + "description": "Non-biocontainer Docker registry image string when present." + }, + "singularity": { + "type": [ + "string", + "null" + ], + "description": "Singularity image reference when present (e.g. `https://depot.galaxyproject.org/singularity/...`)." + }, + "wave": { + "type": [ + "string", + "null" + ], + "description": "Seqera Wave / community-cr registry reference (e.g. `community.wave.seqera.io/library/:--` or `https://community-cr-prod.seqera.io/.../sha256//data`). Wave-built containers are increasingly common in nf-core; kept distinct from `docker` because the resolution rules and provenance differ." + }, + "mulled_components": { + "type": "array", + "items": { + "$ref": "#/$defs/ToolSpec" + }, + "description": "Constituent Bioconda packages for an opaque `mulled-v2-*` multi-package container when resolved from a cached BioContainers multi-package-containers TSV. Omitted when no matching cached row is available." + }, + "versions": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Every distinct version of this tool declared anywhere in the pipeline, sorted, when processes disagree. nf-core modules pin independently, so one pipeline routinely runs several versions of a shared dependency (e.g. `samtools` at 1.17, 1.18, 1.19.2, 1.20). The sibling `version`, `biocontainer`, `docker`, `singularity`, and `wave` fields all come from whichever declaration was read last, so on a divergent tool they describe one arbitrary process rather than the pipeline as a whole — treat this array as the authoritative version set and read `processes[].conda` / `processes[].container` for what a given process actually runs. Declarations that name no version contribute nothing here, so a tool pinned by some processes and left unpinned by others lists only its real versions. Omitted when fewer than two distinct versions were declared." + } + } + }, + "ToolSpec": { + "title": "ToolSpec", + "description": "One constituent package in a decomposed multi-package container.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "bioconda" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "bioconda": { + "type": "string", + "description": "Exact Bioconda requirement spec, e.g. `bioconda::samtools=1.20`." + } + } + }, + "ChannelIO": { + "title": "ChannelIO", + "description": "One declared input or output channel of a process. Channel shape is a string, not a structured type — `tuple(meta, [path,path])` is enough for downstream Molds and avoids a research project on NF channel typing.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "shape" + ], + "properties": { + "name": { + "type": "string" + }, + "shape": { + "type": "string", + "description": "String-encoded channel shape, e.g. `tuple(meta, [path,path])`, `path`, `val(integer)`." + }, + "description": { + "type": "string" + }, + "topic": { + "type": [ + "string", + "null" + ], + "description": "Nextflow channel-topic name when this output is bound to a topic (e.g. `versions` for the standard nf-core version-aggregation topic). Topics are a Nextflow 24+ feature; the module-level `topic: ` annotation on a process output emits to a global named channel that any consumer can `channel.topic('')` to read. Null for non-topic outputs and for all inputs." + } + } + }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, + "Process": { + "title": "Process", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "module_path", + "meta", + "module_tests", + "inputs", + "outputs" + ], + "properties": { + "name": { + "type": "string", + "description": "Canonical process name (uppercase NF convention) as declared by the `process ` line in `module_path`." + }, + "aliases": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alias names this process is imported as in workflow scopes via `include { as }`. Real nf-core pipelines re-import a single module multiple times under different aliases to invoke it with distinct runtime args (e.g. `MINIMAP2_ALIGN as MINIMAP2_CONSENSUS`, `MINIMAP2_ALIGN as MINIMAP2_POLISH`). Each alias appears in `workflow.edges[].from`/`.to` exactly as written; the canonical `name` does not appear in edges unless an unaliased import exists. Default `[]` when the process is only imported under its canonical name." + }, + "in_subworkflows": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Denormalized FK: names of `subworkflows[]` whose `calls[]` invoke this process (canonical name or any alias). Empty `[]` when the process is only invoked from the primary `workflow` body. Useful for grouping the flat process-tier DAG into a subworkflow-tier view client-side. Multi-entry when the same module is invoked from several subworkflows (typically via aliases)." + }, + "module_path": { + "type": "string", + "description": "Relative path from the pipeline root to the file declaring the process." + }, + "meta": { + "anyOf": [ + { + "$ref": "#/$defs/ModuleMeta" + }, + { + "type": "null" + } + ], + "description": "Normalized `meta.yml` from the module directory when present. Null for local/ad-hoc processes without module metadata." + }, + "module_tests": { + "type": "array", + "items": { + "$ref": "#/$defs/NfTest" + }, + "description": "Module-scoped nf-test files under the module's `tests/` directory. Empty for local/ad-hoc modules without unit tests." + }, + "tool": { + "type": [ + "string", + "null" + ], + "description": "Foreign key into `tools[].name` when the process runs a single recognizable tool; null for multi-tool or pure-Groovy processes." + }, + "container": { + "type": [ + "string", + "null" + ], + "description": "Verbatim text of the process's `container` directive (or null). Modern nf-core modules use a ternary expression `\"${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '' : '' }\"` — keep that text intact here. The two resolved branches are split out into `tools[].docker` / `tools[].singularity` (and `tools[].wave` / `tools[].biocontainer` as appropriate)." + }, + "conda": { + "type": [ + "string", + "null" + ], + "description": "Verbatim text of the process's `conda` directive (or null). Two common shapes: a literal `bioconda::=` (legacy) or a file reference `${moduleDir}/environment.yml` (modern nf-core convention). Either way the resolved bioconda spec lands in `tools[].bioconda`." + }, + "inputs": { + "type": "array", + "items": { + "$ref": "#/$defs/ChannelIO" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/$defs/ChannelIO" + } + }, + "when": { + "type": [ + "string", + "null" + ], + "description": "Verbatim `when:` guard expression, or null if absent." + }, + "script_summary": { + "type": "string", + "description": "One-line LLM-derived summary of what the `script:` body does. Free text. The deterministic CLI does not emit it — it emits `script_excerpt` instead, and the cast skill's LLM pass fills this in." + }, + "script_excerpt": { + "type": [ + "string", + "null" + ], + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." + }, + "publish_dir": { + "type": [ + "string", + "null" + ], + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." + } + } + }, + "ModuleMeta": { + "title": "ModuleMeta", + "description": "Normalized subset of nf-core module `meta.yml`, captured next to a process so module-scoped downstream Molds can consume `processes[i]` without reading the full pipeline summary.", + "type": "object", + "additionalProperties": false, + "required": [ + "keywords", + "authors", + "maintainers", + "tools", + "input", + "output" + ], + "properties": { + "description": { + "type": "string" + }, + "keywords": { + "type": "array", + "items": { + "type": "string" + } + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + }, + "maintainers": { + "type": "array", + "items": { + "type": "string" + } + }, + "tools": { + "type": "array", + "items": { + "$ref": "#/$defs/ModuleMetaEntry" + } + }, + "input": { + "type": "array", + "items": { + "$ref": "#/$defs/ModuleMetaEntry" + } + }, + "output": { + "type": "array", + "items": { + "$ref": "#/$defs/ModuleMetaEntry" + } + } + } + }, + "ModuleMetaEntry": { + "title": "ModuleMetaEntry", + "description": "One named entry from a module `meta.yml` section, normalized from nf-core's single-key YAML maps into `{ name, ...fields }` objects.", + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "documentation": { + "type": "string" + }, + "tool_dev_url": { + "type": "string" + }, + "doi": { + "type": "string" + }, + "licence": { + "type": "array", + "items": { + "type": "string" + } + }, + "identifier": { + "type": "string" + }, + "type": { + "type": "string" + }, + "pattern": { + "type": "string" + } + } + }, + "Subworkflow": { + "title": "Subworkflow", + "description": "One named `workflow { ... }` block other than the primary workflow. Includes nf-core utility wrappers (PIPELINE_INITIALISATION, PIPELINE_COMPLETION, UTILS_NFCORE_PIPELINE) which exist to compose helper-function calls rather than to invoke processes.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "path", + "kind", + "calls", + "tests" + ], + "properties": { + "name": { + "type": "string" + }, + "path": { + "type": "string", + "description": "Relative path from the pipeline root." + }, + "kind": { + "type": "string", + "enum": [ + "pipeline", + "utility" + ], + "description": "`pipeline` when the subworkflow invokes pipeline processes (data-flow contributor). `utility` when it composes helper functions only — common nf-core template subworkflows like PIPELINE_INITIALISATION (which calls `paramsHelp`, `samplesheetToList`, `UTILS_NFSCHEMA_PLUGIN`) and PIPELINE_COMPLETION (`completionEmail`, `completionSummary`). Utility subworkflows have empty `calls[]` for processes but may still expose channel outputs the primary workflow consumes." + }, + "aliases": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alias names this subworkflow is imported as via `include { as }`. Calls retain the spelling used at each call site, while this field maps those aliases back to the canonical `name`. Empty when the subworkflow is never aliased." + }, + "calls": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of processes and nested subworkflows this subworkflow invokes. Free-function calls (e.g. `paramsHelp`, `completionEmail`) are NOT recorded here; they're implied by `kind = utility`." + }, + "inputs": { + "type": "array", + "items": { + "$ref": "#/$defs/ChannelIO" + }, + "description": "Parsed entries of the subworkflow's `take:` block. `name` is the take name, `description` carries any adjacent `// ...` comment, `shape` carries the verbatim source line." + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/$defs/ChannelIO" + } + }, + "invocations": { + "type": "array", + "items": { + "$ref": "#/$defs/SubworkflowInvocation" + }, + "description": "Call-sites where this subworkflow was invoked. Each entry records the calling workflow/subworkflow, the verbatim positional arguments, and a per-argument binding onto this subworkflow's `inputs[].name` (take names). Drives reference-asset attribution (`reference_assets[].used_by`) without re-parsing call sites. Empty when uncalled or when the call could not be positionally aligned." + }, + "tests": { + "type": "array", + "items": { + "$ref": "#/$defs/NfTest" + }, + "description": "Subworkflow-scoped nf-test files under the subworkflow's `tests/` directory. Empty when absent." + } + } + }, + "SubworkflowInvocation": { + "title": "SubworkflowInvocation", + "description": "One call-site of a subworkflow, with positional arguments bound to its `take:` names.", + "type": "object", + "additionalProperties": false, + "required": [ + "caller", + "arguments", + "bindings" + ], + "properties": { + "caller": { + "type": "string", + "description": "Name of the calling workflow or subworkflow." + }, + "caller_path": { + "type": [ + "string", + "null" + ], + "description": "Repo-relative path of the file the caller is defined in. Null when not resolvable." + }, + "arguments": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Verbatim positional argument expressions in source order (e.g. `params.fasta`, `PREPARE_GENOME.out.fai`)." + }, + "bindings": { + "type": "array", + "items": { + "$ref": "#/$defs/InvocationBinding" + }, + "description": "Per-argument binding onto the callee's `inputs[].name` (take names), in source order. Length matches `arguments[]` when the call positionally aligns to the callee's take block; shorter when only a prefix aligns." + } + } + }, + "InvocationBinding": { + "title": "InvocationBinding", + "description": "Binding of one positional argument to a callee `take:` name.", + "type": "object", + "additionalProperties": false, + "required": [ + "take", + "argument" + ], + "properties": { + "take": { + "type": "string", + "description": "Callee `take:` name (FK into the callee subworkflow's `inputs[].name`)." + }, + "argument": { + "type": "string", + "description": "Verbatim caller-side argument expression." + } + } + }, + "Channel": { + "title": "Channel", + "description": "One top-level channel constructed in the workflow. Sources include `Channel.fromPath`, `Channel.fromFilePairs`, `samplesheetToList`-driven `Channel.fromList`, `splitCsv`, `file`/`files`, and plain `params.*` references.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "source", + "shape", + "construct", + "from_param", + "required_runtime" + ], + "properties": { + "name": { + "type": "string" + }, + "source": { + "type": "string", + "description": "Verbatim channel-construction expression." + }, + "shape": { + "type": "string", + "description": "String-encoded channel shape; same convention as ChannelIO." + }, + "construct": { + "type": "string", + "enum": [ + "fromPath", + "fromFilePairs", + "fromList", + "samplesheetToList", + "splitCsv", + "file", + "files", + "of", + "value", + "empty", + "topic", + "other" + ], + "description": "Classifies the channel's primary materialization factory or shape-determining operator. Selection precedence: (1) `samplesheetToList` when the chain contains `samplesheetToList(...)` (typically wrapped in `Channel.fromList`); (2) `splitCsv` when the chain ends in `.splitCsv(header: true)` over a path; (3) otherwise the outermost factory (`Channel.fromPath` → `fromPath`, `Channel.fromFilePairs` → `fromFilePairs`, `file(...)` → `file`, etc.); (4) `other` for unrecognized constructions. The verbatim `source` retains the full chain — `construct` is the typed lookup the converter would otherwise re-parse." + }, + "from_param": { + "type": [ + "string", + "null" + ], + "description": "Foreign key into `params[].name` when the construction expression directly references `params.X` (e.g. `Channel.fromPath(params.reads)`, `samplesheetToList(params.input, ...)`, `file(params.fasta)`). Null for literal-glob construction, expressions that compose multiple params, and channels derived from other channels. v1 resolution is direct-only; one-hop Groovy bindings (`def reads = params.reads; Channel.fromPath(reads)`) are not chased — see galaxyproject/foundry#211." + }, + "required_runtime": { + "type": "boolean", + "description": "True when the construction chain ends in `.ifEmpty { error ... }` (or an equivalent imperative emptiness-throw guard). Captures runtime requiredness even when the param's nf-schema entry does not mark it required. Combine with `params[].required` and any imperative pre-construction `error` checks for the full requiredness picture." + } + } + }, + "Edge": { + "title": "Edge", + "description": "One edge in the workflow's call graph. The deterministic parser records the literal operator chain in `via`; reconciliation of the source and target shapes is the LLM step.", + "type": "object", + "additionalProperties": false, + "required": [ + "from", + "to" + ], + "properties": { + "from": { + "type": "string", + "description": "Channel name or `.out.` reference." + }, + "to": { + "type": "string", + "description": "Process or subworkflow name receiving this channel." + }, + "via": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Operators on the path between source and target, in order (e.g. `[\"map\", \"join\", \"groupTuple\"]`). Empty for a direct edge." + }, + "notes": { + "type": "string", + "description": "LLM-emitted note when the reconciliation is low-confidence (e.g. deeply nested closures)." + } + } + }, + "Conditional": { + "title": "Conditional", + "description": "One workflow-level conditional that gates a subgraph.", + "type": "object", + "additionalProperties": false, + "required": [ + "guard", + "branch", + "affects" + ], + "properties": { + "guard": { + "type": "string", + "description": "Verbatim guard expression, e.g. `params.skip_alignment`." + }, + "branch": { + "type": "string", + "enum": [ + "default", + "alternate" + ], + "description": "Which side of the conditional this entry describes. `default` is the truthy branch; `alternate` is the else branch." + }, + "affects": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Process or subworkflow names whose execution is gated by this conditional." + } + } + }, + "Workflow": { + "title": "Workflow", + "description": "Primary workflow: channel construction plus the call-graph DAG. Real nf-core pipelines have multiple named `workflow` blocks (an anonymous entrypoint `workflow {}` in main.nf that wires PIPELINE_INITIALISATION → NFCORE_ → PIPELINE_COMPLETION; a named NFCORE_ wrapper; a substantive named workflow under workflows/.nf). The summary collapses this into one primary `workflow` plus `subworkflows[]`. Selection rule: pick the workflow that invokes the most pipeline processes — typically the one under `workflows/.nf`. The anonymous `workflow {}` glue and the NFCORE_ wrapper land in `subworkflows[]`.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "channels", + "edges" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the selected primary workflow (uppercase NF convention). The anonymous `workflow {}` entrypoint is never the primary; if it is the only workflow block, the pipeline is too small to summarize and the cast skill should emit a warning." + }, + "channels": { + "type": "array", + "items": { + "$ref": "#/$defs/Channel" + } + }, + "edges": { + "type": "array", + "items": { + "$ref": "#/$defs/Edge" + } + }, + "conditionals": { + "type": "array", + "items": { + "$ref": "#/$defs/Conditional" + } + } + } + }, + "TestDataRef": { + "title": "TestDataRef", + "description": "One test-fixture input. Field names mirror gxy-sketches' TestDataRef verbatim. The sketch-bundle constraint that `path` must live under `test_data/` is intentionally dropped; the Foundry summary describes fixtures as data, it does not bundle them.", + "type": "object", + "additionalProperties": false, + "required": [ + "role" + ], + "properties": { + "role": { + "type": "string", + "description": "Logical role of the input (e.g. `samplesheet`, `reference_fasta`)." + }, + "path": { + "type": [ + "string", + "null" + ], + "description": "Repo-relative path when the fixture lives in-tree, or local filesystem path when the CLI fetched the remote fixture into a caller-provided test-data directory." + }, + "url": { + "type": [ + "string", + "null" + ], + "format": "uri" + }, + "sha1": { + "type": [ + "string", + "null" + ], + "description": "SHA-1 integrity hash when published alongside the fixture." + }, + "filetype": { + "type": [ + "string", + "null" + ], + "description": "File format, e.g. `fastq.gz`, `csv`." + }, + "description": { + "type": [ + "string", + "null" + ] + } + }, + "anyOf": [ + { + "required": [ + "path" + ] + }, + { + "required": [ + "url" + ] + } + ] + }, + "ExpectedOutputRef": { + "title": "ExpectedOutputRef", + "description": "One expected output. Field names mirror gxy-sketches' ExpectedOutputRef verbatim. At least one of path / url / assertions is required.", + "type": "object", + "additionalProperties": false, + "required": [ + "role" + ], + "properties": { + "role": { + "type": "string" + }, + "path": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ], + "format": "uri" + }, + "kind": { + "type": [ + "string", + "null" + ], + "description": "Coarse output kind: `report`, `tabular`, `image`, `archive`, `sequence`, `other`." + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "assertions": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Assertion strings. Simple equality / regex / contains-string checks are preserved verbatim; complex Groovy assertions are summarized to prose with a `warnings[]` flag in the parent Summary." + } + }, + "anyOf": [ + { + "required": [ + "path" + ] + }, + { + "required": [ + "url" + ] + }, + { + "required": [ + "assertions" + ] + } + ] + }, + "TestCandidate": { + "title": "TestCandidate", + "description": "One whole-pipeline execution candidate. The shape unifies nf-test cases, input-bearing profiles, and the no-profile defaults fallback so inputs and assertions never live in a separate singular record.", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "kind", + "name", + "path", + "effective_profiles", + "params_delta", + "inputs", + "outputs", + "execution_mode", + "scope", + "disposition", + "rationale", + "assert_workflow_success", + "snapshot", + "prose_assertions" + ], + "properties": { + "id": { + "type": "string", + "description": "Stable candidate identifier: `::` for nf-test, `profile:` for a profile, or `pipeline-defaults`. Duplicate names in one file receive a numeric suffix." + }, + "kind": { + "type": "string", + "enum": [ + "nf-test", + "profile", + "pipeline-defaults" + ] + }, + "name": { + "type": "string", + "description": "Human-readable test name, profile name, or `pipeline defaults`." + }, + "path": { + "type": [ + "string", + "null" + ], + "description": "Repo-relative `.nf.test`, profile-declaring config, or selected entrypoint path." + }, + "effective_profiles": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Resolved profile chain after nf-test.config, file/test directives, and an explicit caller override. `+name` directives append; other directives replace." + }, + "params_delta": { + "type": "object", + "additionalProperties": true, + "description": "Candidate-specific parameter assignments after profile config and per-test overrides, excluding unchanged pipeline defaults." + }, + "inputs": { + "type": "array", + "items": { + "$ref": "#/$defs/TestDataRef" + }, + "description": "Input fixtures resolved for this candidate, including fetched samplesheet members when requested." + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/$defs/ExpectedOutputRef" + } + }, + "execution_mode": { + "type": "string", + "enum": [ + "real", + "stub", + "mixed", + "unknown" + ] + }, + "scope": { + "type": "string", + "enum": [ + "primary", + "bootstrap-only", + "reference-scale", + "unknown" + ], + "description": "Coverage/scale classification used by selection. Names are evidence: full-scale is deferred by default; minimal/tiny/stub is bootstrap-only." + }, + "disposition": { + "type": "string", + "enum": [ + "selected", + "eligible", + "deferred" + ] + }, + "rationale": { + "type": "string", + "description": "Why the candidate exists and how its disposition was classified." + }, + "assert_workflow_success": { + "type": [ + "boolean", + "null" + ], + "description": "nf-test workflow-success assertion, or null for profile/default candidates." + }, + "snapshot": { + "anyOf": [ + { + "$ref": "#/$defs/SnapshotFixture" + }, + { + "type": "null" + } + ] + }, + "prose_assertions": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "TestSelection": { + "title": "TestSelection", + "description": "Selection outcome over the complete candidate list.", + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "selected_candidate_id", + "rationale" + ], + "properties": { + "status": { + "type": "string", + "enum": [ + "selected", + "needs-scope-choice" + ] + }, + "selected_candidate_id": { + "type": [ + "string", + "null" + ], + "description": "Foreign key into `test_candidates[].id`; null exactly when status is needs-scope-choice." + }, + "rationale": { + "type": "string" + } + } + }, + "NfTest": { + "title": "NfTest", + "description": "One `tests/*.nf.test` file. nf-core templates use a near-uniform shape: a `nextflow_pipeline { test(\"\") { when { params { ... } } then { assertAll(...) } } }` block, with one test() per profile, asserting `workflow.success` plus a snapshot match. This shape captures the data so downstream test-conversion molds (e.g. nextflow-test-to-target-tests) don't have to re-parse Groovy.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "path", + "profiles", + "params_overrides", + "execution_mode", + "assert_workflow_success", + "snapshot", + "prose_assertions" + ], + "properties": { + "name": { + "type": "string", + "description": "Description string passed to `test(\"...\")`. Often duplicates the profile name." + }, + "path": { + "type": "string", + "description": "Repo-relative path of the .nf.test file (e.g. `tests/dfast.nf.test`)." + }, + "profiles": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Profile name(s) the test runs under, from the file-level `profile \"\"` declaration and/or per-test config overrides. Usually a single profile." + }, + "params_overrides": { + "type": "object", + "additionalProperties": true, + "description": "The `when { params { ... } }` block as a key→value map. Most templates set only `outdir`; pipeline-specific overrides land here." + }, + "execution_mode": { + "type": "string", + "enum": [ + "real", + "stub", + "mixed", + "unknown" + ], + "description": "Whether the test requests real execution, stub execution, or mixes both." + }, + "assert_workflow_success": { + "type": "boolean", + "description": "Whether the test asserts `workflow.success`. Almost always true for nf-core templates." + }, + "snapshot": { + "anyOf": [ + { + "$ref": "#/$defs/SnapshotFixture" + }, + { + "type": "null" + } + ], + "description": "Structured representation of the `assert snapshot(...).match()` clause when present; null when the test uses no snapshot." + }, + "prose_assertions": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Other assertions (regex matches, count checks, content equality on specific files) summarized to prose strings. Empty for snapshot-only tests, which is the common nf-core case." + } + } + }, + "SnapshotFixture": { + "title": "SnapshotFixture", + "description": "Structured shape of an nf-test `snapshot(...).match()` assertion. nf-core templates pass a small set of values into snapshot() — succeeded-task count, version YAML (with Nextflow version stripped), stable file-name list, stable file-content list — and prune the comparison via `getAllFilesFromDir(..., ignoreFile: ..., ignore: [...])` helpers. This breakdown lets downstream consumers reconstruct equivalent assertions in target test frameworks without re-parsing Groovy.", + "type": "object", + "additionalProperties": false, + "required": [ + "captures", + "helpers", + "ignore_files", + "ignore_globs" + ], + "properties": { + "captures": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Logical names of values passed to snapshot(), in order. Common set: `succeeded_task_count`, `versions_yml`, `stable_names`, `stable_paths`. Free-form strings rather than an enum because pipelines can pass other values." + }, + "helpers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "nf-test helper functions invoked in the snapshot expression (e.g. `getAllFilesFromDir`, `removeNextflowVersion`). Used to detect whether a target framework can replicate the comparison." + }, + "ignore_files": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Repo-relative paths passed as `ignoreFile:` to helpers (e.g. `tests/.nftignore`, `tests/.nftignore_files_entirely`). The files themselves are line-delimited globs." + }, + "ignore_globs": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Inline `ignore: [...]` glob list passed to helpers (e.g. `['Prokka/**', '**/multiqc_busco.yaml']`). Distinct from ignore_files which references on-disk lists." + }, + "snap_path": { + "type": [ + "string", + "null" + ], + "description": "Repo-relative path of the corresponding `.nf.test.snap` file when present." + }, + "parsed_content": { + "type": "array", + "items": { + "$ref": "#/$defs/SnapshotContent" + }, + "description": "Parsed entries from the `.nf.test.snap` JSON sidecar. Empty when the sidecar is absent, too large for compact summary output, or not valid JSON. Each entry preserves the snapshot name plus channel-keyed file md5 assertions and non-file scalar values." + } + } + }, + "SnapshotContent": { + "title": "SnapshotContent", + "description": "One top-level snapshot entry from an nf-test `.snap` JSON sidecar.", + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "channels" + ], + "properties": { + "name": { + "type": "string", + "description": "Top-level key in the `.snap` file, usually the `test(\"...\")` name." + }, + "channels": { + "type": "array", + "items": { + "$ref": "#/$defs/SnapshotChannel" + }, + "description": "Channel-keyed snapshot values. Pipeline-template flat file lists use `key: null`." + } + } + }, + "SnapshotChannel": { + "title": "SnapshotChannel", + "description": "One output channel or flat snapshot list inside a `.snap` entry.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "files", + "values" + ], + "properties": { + "key": { + "type": [ + "string", + "null" + ], + "description": "Channel key from the snapshot object, e.g. `0`, `1`, or `genome_gtf`; null for flat file-list snapshots." + }, + "files": { + "type": "array", + "items": { + "$ref": "#/$defs/SnapshotFile" + }, + "description": "File md5 assertions parsed from `:md5,` strings." + }, + "values": { + "type": "array", + "items": {}, + "description": "Non-file snapshot values preserved verbatim for consumers that need versions, counts, or scalar tuple emissions." + } + } + }, + "Evidence": { + "title": "Evidence", + "description": "Shared provenance block for source-detected facts. `source_path` localizes the finding; `confidence` is a coarse self-assessment; `evidence[]` carries free-form snippets (matched lines, normalized expressions, detector notes). Reused by ReferenceAsset and ReferenceRebuildRule.", + "type": "object", + "additionalProperties": false, + "required": [ + "source_path", + "confidence", + "evidence" + ], + "properties": { + "source_path": { + "type": [ + "string", + "null" + ], + "description": "Repo-relative path of the source file where the finding was detected (e.g. `subworkflows/local/prepare_genome/main.nf`, `nextflow.config`). Null when the finding spans files or cannot be localized." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ], + "description": "Detector self-assessment. `high`: all signals match the expected idiom; `medium`: some signals match but the guard or builder is partial / mixes non-param locals; `low`: heuristic match, downstream should treat as a hint." + }, + "evidence": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Free-form evidence snippets (matched source lines, normalized expressions, detector notes). Empty when none." + } + } + }, + "ReferenceAsset": { + "title": "ReferenceAsset", + "description": "Curated source-side view of one reference-data input the pipeline consumes. A foreign-key projection onto `params[].name` plus the metadata downstream classification Molds need. Target-agnostic: no Galaxy translation decisions are encoded here.", + "type": "object", + "additionalProperties": false, + "required": [ + "param", + "asset_kind", + "required", + "used_by", + "evidence" + ], + "properties": { + "param": { + "type": "string", + "description": "FK into `params[].name`." + }, + "asset_kind": { + "type": "string", + "description": "Coarse asset classification (e.g. `fasta`, `fasta_index`, `sequence_dictionary`, `bwa_index`, `bwamem2_index`, `tabix_index`, `gtf`, `gff`, `bed`, `vcf`, `database`, `reference_sheet`, `other`). Free-form string; downstream Molds choose how to map onto target datatypes." + }, + "format_hint": { + "type": [ + "string", + "null" + ], + "description": "Format hint, often the nf-schema `format` keyword (`file-path`, `directory-path`, `path`, `file-path-pattern`) or a domain hint like `fai`, `dict`. Null when undetermined." + }, + "required": { + "type": "boolean", + "description": "True when the source pipeline treats this asset as required (no compute-if-missing branch, no optional guard). When false, see `reference_rebuilds[]` for any rebuild branch covering this param." + }, + "source_kind": { + "type": [ + "string", + "null" + ], + "enum": [ + "nextflow_schema", + "params_block", + "getGenomeAttribute", + "computed", + null + ], + "description": "Mirrors `params[].source_kind`. Duplicated here so reference-data consumers don't have to dereference." + }, + "source_expression": { + "type": [ + "string", + "null" + ], + "description": "Mirrors `params[].source_expression`. Duplicated for the same reason." + }, + "schema_group": { + "type": [ + "string", + "null" + ], + "description": "Mirrors `params[].schema_group`. Useful for grouping (`Reference genome options`)." + }, + "used_by": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of processes / subworkflows that receive this param as an argument (via direct invocation binding or channel construction). Empty when no caller could be attributed." + }, + "evidence": { + "$ref": "#/$defs/Evidence" + } + } + }, + "ReferenceRebuildRule": { + "title": "ReferenceRebuildRule", + "description": "One compute-if-missing rebuild branch detected in a workflow or subworkflow body. Binds an asset param to the guard expression and builder process that produces it when absent. Source evidence only; in-tool-rebuild decisions live downstream.", + "type": "object", + "additionalProperties": false, + "required": [ + "asset_param", + "guard", + "builder", + "builder_outputs", + "evidence" + ], + "properties": { + "asset_param": { + "type": "string", + "description": "FK into `params[].name` — the asset this branch reconstitutes when missing." + }, + "guard": { + "type": "string", + "description": "Verbatim guard expression, e.g. `!fasta_fai_in && step != \"annotate\"`." + }, + "guard_params": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Param names referenced by the guard. Empty when the guard binds only non-param locals." + }, + "builder": { + "type": "string", + "description": "Builder process or subworkflow name invoked inside the branch (e.g. `SAMTOOLS_FAIDX`, `BWA_INDEX`)." + }, + "builder_outputs": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Output channel names from the builder that are assigned to the asset (e.g. `fai`, `dict`)." + }, + "fallback_for": { + "type": [ + "string", + "null" + ], + "description": "Name of the `_in`-style param that, when present, supplies the asset directly and skips the rebuild. Null when no such pairing was detected." + }, + "evidence": { + "$ref": "#/$defs/Evidence" + } + } + }, + "SnapshotFile": { + "title": "SnapshotFile", + "description": "One file digest assertion from a `.snap` sidecar.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "basename", + "md5", + "stub" + ], + "properties": { + "path": { + "type": "string", + "description": "Path portion before `:md5,` exactly as stored in the sidecar." + }, + "basename": { + "type": "string", + "description": "Final path segment, suitable for Galaxy test `file:` assertions when the snapshot stores only basenames." + }, + "md5": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "MD5 digest from the sidecar." + }, + "stub": { + "type": "boolean", + "description": "True when the digest is the empty-file md5 emitted by many `-stub` tests." + } + } + } + } +} diff --git a/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/schemas/workflow-brief-schema.schema.json b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/schemas/workflow-brief-schema.schema.json new file mode 100644 index 0000000..92c431e --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-summary-to-workflow-brief/references/schemas/workflow-brief-schema.schema.json @@ -0,0 +1,103 @@ +{ + "name": "Workflow Brief", + "title": "required", + "sections": [ + { + "heading": "Workflow", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Objective", + "min": 1, + "max": 1, + "content": true + }, + { + "heading": "Sources", + "max": 1, + "content": true + }, + { + "heading": "Scope", + "max": 1, + "content": true, + "sections": [ + { + "heading": "Included", + "max": 1, + "content": true + }, + { + "heading": "Excluded", + "max": 1, + "content": true + } + ] + }, + { + "heading": "Inputs and outputs", + "max": 1, + "content": true + }, + { + "heading": "Requirements and preferences", + "max": 1, + "content": true + }, + { + "heading": "Acceptance criteria", + "max": 1, + "content": true + }, + { + "heading": "Open questions", + "max": 1, + "content": true + }, + { + "heading": "Decisions", + "max": 1, + "content": true + }, + { + "heading": "Related artifacts", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + }, + { + "heading": "Agent Environment", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Tooling", + "max": 1, + "content": true + }, + { + "heading": "Constraints", + "max": 1, + "content": true + }, + { + "heading": "Containerization", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_provenance.json b/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_provenance.json index f8509c4..c1e9bc0 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/nextflow-test-to-cwl-test-plan/index.md", "revision": 3, "content_hash": "d06d87fce9160be3e5d69485d310bf3e51bff90a9752540de6431302ffb97424", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "commit": "871b5e6deadfd56aeb5e2ba92eb8f6f50c8449ee" }, - "cast_at": "2026-09-15T23:16:25.154Z", + "cast_at": "2026-11-10T14:49:42.501Z", "refs": [ { "kind": "research", @@ -36,8 +36,8 @@ "evidence": "corpus-observed", "purpose": "Read summarized nf-test profiles, snapshot fixtures, selected test data, params, and expected outputs.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], diff --git a/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_verify.json b/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_verify.json index 667ae3b..d508d96 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-cwl-test-plan/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/SKILL.md b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/SKILL.md index c71490e..08ffa4b 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/SKILL.md +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/SKILL.md @@ -41,11 +41,14 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. +- Before starting, check that the installed validator carries the galaxy-workflow-test-plan schema this bundle was cast with: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. ## Procedure -Translate Nextflow test evidence into a Galaxy workflow test plan. The output is a reviewable YAML handoff conforming to galaxy-workflow-test-plan, not a concrete `tests-format` file: preserve profile, fixture, snapshot, ignored-file, expected-output, and rationale provenance so implement-galaxy-workflow-test can author the final Galaxy test artifact with the right labels and assertions. Because this plan is translated from real nf-test evidence, set `source.derived_from: test-evidence` and prefer `evidence: test-evidence` on the assertions it carries. +Translate Nextflow test evidence into a Galaxy workflow test plan. The output is a reviewable YAML handoff conforming to galaxy-workflow-test-plan, not a concrete `tests-format` file: preserve profile, fixture, snapshot, ignored-file, expected-output, and rationale provenance so implement-galaxy-workflow-test can author the final Galaxy test artifact with the right labels and assertions. Because this plan is translated from real nf-test evidence, set `source.derived_from: test-evidence`. + +Set each assertion's `evidence` by where its expected value came from. Use `test-evidence` only for a value translated from an upstream assertion, checksum, or expected file. Use `fixture-derived` for a value measured from the upstream fixture bytes, and `intent` for one read from a wrapper XML or the translated workflow's step state. A snapshot `md5` token translated to `checksum` is `test-evidence`. A snapshot capture that records only names or paths (`stable_name`, `.nftignore`-pruned) carries no expected content: author those assertions as `fixture-derived` or `intent`, and when a test case ends up with no `test-evidence` assertions, record a `warnings[]` entry with code `no-upstream-expected-outputs`. Consume the candidate selected by summarize-nextflow; do not re-rank profiles here. Use nextflow-test-case-selection to preserve whether that case is primary coverage, a topology bootstrap, or reference-scale evidence, and to keep deferred science branches visible rather than presenting one translated test as exhaustive. diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_provenance.json b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_provenance.json index f580977..9981e28 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "nextflow-test-to-galaxy-test-plan", "path": "content/molds/nextflow-test-to-galaxy-test-plan/index.md", - "revision": 7, - "content_hash": "2a8aaa5228eaa632c760ad10de9bfcdc1a7e4f034332b68cadc8a7f92cbe1e8c", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "revision": 8, + "content_hash": "fa769216f621ef25470c9fb907df1278635c0a1e022b647b9462ff7b92469291", + "commit": "871b5e6deadfd56aeb5e2ba92eb8f6f50c8449ee" }, - "cast_at": "2026-09-15T23:16:25.267Z", + "cast_at": "2026-11-10T14:49:42.528Z", "refs": [ { "kind": "research", @@ -53,8 +53,8 @@ "evidence": "corpus-observed", "purpose": "Distinguish accepted IWC-style test shortcuts from assertion smells while translating tests.", "trigger": "When deciding whether to use existence-only, size-only, image-dimension, or tolerant output checks.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -68,8 +68,8 @@ "evidence": "corpus-observed", "purpose": "Emit Galaxy/IWC-style job input fixtures, remote locations, hashes, and collection input shapes.", "trigger": "When writing job inputs or deciding whether fixtures belong in test-data, Zenodo, ENA/SRA, or CVMFS.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -113,8 +113,8 @@ "evidence": "corpus-observed", "purpose": "Describe Galaxy workflow-test assertion intent and tolerances for translated expected outputs.", "trigger": "When turning Nextflow expected outputs or snapshots into Galaxy test-plan assertions.", - "src_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", - "dst_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", "source": "deterministic" }, { @@ -129,8 +129,8 @@ "purpose": "Output contract: the emitted plan conforms to [[galaxy-workflow-test-plan]]. Cast bundles the JSON Schema; validate with `foundry validate-galaxy-workflow-test-plan`.", "verification": "Cast the skill on an nf-core/bacass summary and confirm the emitted YAML validates and downstream [[implement-galaxy-workflow-test]] consumes it.", "license": "MIT", - "src_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", - "dst_hash": "325e9cf1bb5e075fa818dc68868647de95389ca8df8581ee8cf1e2be61196e37", + "src_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "dst_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "source": "deterministic" }, { @@ -144,8 +144,8 @@ "evidence": "corpus-observed", "purpose": "Read summarized nf-test profiles, snapshot fixtures, selected test data, params, and expected outputs.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" }, { @@ -161,8 +161,8 @@ "trigger": "When mapping expected outputs, tolerances, snapshots, or fixture assertions into Galaxy workflow-test assertion intent.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", - "dst_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", + "src_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "dst_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_verify.json b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_verify.json index 2c1b718..ffd276c 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] }, @@ -22,6 +24,8 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-workflow-test-plan", + "--schema-sha256", + "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md index 6d0fef7..bd1160e 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-11 -revision: 6 +revised: 2026-10-21 +revision: 10 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,231 +16,109 @@ related_notes: - "[[validate-tests]]" - "[[iwc-tabular-operations-survey]]" - "[[galaxy-discover-datasets]]" -summary: "Decision and idiom guide for picking planemo workflow-test assertions: which family per output type, how to size tolerances, when to validate." +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." --- -# Planemo asserts: idiom and decision guide +# Choosing Planemo workflow-test assertions -Companion to [[iwc-test-data-conventions]] (input shapes), [[galaxy-workflow-testability-design]] (workflow structure before test YAML exists), and [[iwc-shortcuts-anti-patterns]] (what's accepted vs smell). This note is forward-looking: when authoring a new `-tests.yml`, which assertion family fits which output, and what the recommended tolerances and operators are. +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. -The **vocabulary itself is not restated here** — every assertion's parameter list, types, defaults, required fields, and Python docstring is rendered from the test-format JSON Schema at [[tests-format]]. Assertion names below deep-link into that page (e.g. [[tests-format#has_text_model|has_text]] jumps straight to that `$def`). +## 1. Choose by the property, then the output type -## 1. Choose by output type +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | -The single most useful decision table. Pick the row that matches the file format the workflow emits; default to the recommended assertion family. +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. -| Output type | Default assertion family | Why | Fallback | -|---|---|---|---| -| **Plain text reports / logs** (FastQC summary, MultiQC text section) | [[tests-format#has_text_model|has_text]] (substring on a known stable token) + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Substring is robust across versions; line count catches truncation | [[tests-format#has_text_matching_model|has_text_matching]] (regex) when token text changes per run | -| **HTML reports** (MultiQC HTML, custom dashboards) | [[tests-format#has_text_model|has_text]] against stable section names | HTML embeds timestamps and asset hashes; byte-diff is hopeless | [[tests-format#has_size_model|has_size]] + large `delta` as last resort | -| **Tabular** (TSV, CSV, BED-like) | [[tests-format#has_n_columns_model|has_n_columns]] + [[tests-format#has_text_model|has_text]] for headers + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Schema-shape + content probes; tolerant to row reordering | [[tests-format#has_line_matching_model|has_line_matching]] for a known canonical row | -| **VCF** | `compare: diff` with `lines_diff: 6` | The `lines_diff: 6` constant matches the typical VCF header preamble that embeds `##fileDate=` and `##source=` | [[tests-format#has_text_matching_model|has_text_matching]] for known variants | -| **BAM** | [[tests-format#has_size_model|has_size]] + [[tests-format#has_archive_member_model|has_archive_member]] (BAM is a gzipped block format) | Headers contain `@PG` lines with command lines and Galaxy job IDs; never byte-diff | Convert to SAM in a downstream step and assert on text | -| **FASTA** (deterministic — assemblies, consensus) | `file:` exact comparison or [[tests-format#has_text_model|has_text]] for known sequence | Output is byte-stable when the upstream tool is deterministic | [[tests-format#has_n_lines_model|has_n_lines]] if line wrapping varies | -| **FASTA** (non-deterministic — RepeatModeler libraries) | `compare: sim_size` with large `delta:` | Family content varies run-to-run | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | -| **FASTQ** (rare as workflow output) | [[tests-format#has_n_lines_model|has_n_lines]] (must be multiple of 4) | Quality scores are read-id-dependent | `compare: sim_size` | -| **JSON** (deterministic — config dumps, params) | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] / [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] | Surgical: assert on the property you care about, ignore the rest | `compare: diff` if the JSON is fully canonical | -| **JSON** (stochastic — HyPhy stats, MCMC results) | `has_text: text: "{"` (existence-only) | Embedded floats break any structural assertion; see [[iwc-shortcuts-anti-patterns]] §1 | [[tests-format#has_h5_keys_model|has_h5_keys]]-equivalent for top-level structural keys | -| **HDF5 / AnnData** | [[tests-format#has_h5_keys_model|has_h5_keys]] + [[tests-format#has_h5_attribute_model|has_h5_attribute]] for known structure | Asserts shape, not values — values are float-noisy | [[tests-format#has_size_model|has_size]] with `delta:` | -| **XML** | [[tests-format#is_valid_xml_model|is_valid_xml]] + [[tests-format#has_element_with_path_model|has_element_with_path]] + `element_text_is`/`element_text_matches` | Surgical structural probes; byte-diff fails on whitespace/attribute-order | [[tests-format#has_n_elements_with_path_model|has_n_elements_with_path]] for cardinality | -| **PNG / image plots** | [[tests-format#has_image_width_model|has_image_width]] + [[tests-format#has_image_height_model|has_image_height]] + [[tests-format#has_size_model|has_size]] with `delta:` (5–10%) | Catches "rendered something with the right dimensions"; corpus norm | [[tests-format#has_image_center_of_mass_model|has_image_center_of_mass]] if mask/segmentation outputs are deterministic | -| **TIFF / multipage images** | [[tests-format#has_image_frames_model|has_image_frames]] + [[tests-format#has_image_channels_model|has_image_channels]] + [[tests-format#has_size_model|has_size]] with `delta:` | Same logic, channel/frame structure matters | — | -| **Archives** (zip, tar.gz) | `has_archive_member: path: "regex"` with nested `asserts:` | Asserts on a specific member; archive timestamps never byte-stable | [[tests-format#has_size_model|has_size]] with `delta:` | -| **GFF / GTF** | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` + [[tests-format#has_text_model|has_text]] for known feature | Tools reorder freely; hard to byte-diff | `has_n_columns: 9` (GFF spec) | -| **Cool / HiC matrices** | `compare: sim_size` with multi-MB `delta:` | Binary, run-to-run variance | [[tests-format#has_archive_member_model|has_archive_member]] for HDF5 component | +## 2. Compare operators and expected artifacts -When in doubt: start with [[tests-format#has_size_model|has_size]] + `delta_frac: 0.1`. It catches the catastrophic failure mode (empty / 10x bigger output). Then add a content probe. +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. -### 1a. If the assertion is too weak, revisit the workflow output +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. -Assertion choice sometimes reveals a workflow-design problem. If the only available output can support only a size check or image-dimension smoke test, check whether the workflow should expose a stronger checkpoint before settling for the weak assertion. +## 3. Set tolerances from the failure boundary -- Scanpy's plot outputs use size and image-dimension assertions, but the same workflow also exposes AnnData HDF5 checkpoints and a cluster-count table (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end uses coarse size bands for coverage/mapped-read outputs, but expression/count tables get stronger regex and exact-line checks (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. -Rule: if a translated workflow exposes only weakly assertable final reports, consult [[galaxy-workflow-testability-design]] and consider promoting a table/text/HDF5 checkpoint before writing the final assertions. +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. -## 2. The `compare:` operators +## 4. Text assertions and repeated checks -Top-level on a `file:` output assertion. Vocabulary, in decreasing strictness: - -- **`diff`** (default). Byte-for-byte equality with optional `lines_diff:` tolerance for a fixed number of header lines. Use only when the upstream tool is deterministic on fixed inputs *and* the output has no embedded timestamps, command lines, version banners, or hash-ordered Python-dict-style keys. -- **`re_match` / `re_match_multiline`**. Each line of the expected fixture is a regex that must match the corresponding output line. Useful when a few fields per row are timestamped but the rest is canonical. Rare in the corpus. -- **`contains`**. The expected fixture is a substring of the output. Cheap; weak. Prefer `asserts: has_text` for new code unless you genuinely have a multi-line block to assert as a whole. -- **`sim_size`**. Output file size matches the fixture's size within `delta:` (bytes) or `delta_frac:` (fraction). Use when the output is necessarily non-deterministic but its rough size is reproducible (RepeatModeler libraries, HiC matrices, Bayesian sampler outputs). - -Picking `lines_diff:`: count the mutable header lines in the output format. VCF: ~6 (`##fileformat`, `##fileDate`, `##source`, `##reference`, contig/info lines vary). SAM/text headers: count `@HD`/`@PG`/`@CO` lines. Set `lines_diff:` to the count exactly — looser values mask real diffs. - -## 3. Tolerance picking - -**`delta:`** is bytes (for [[tests-format#has_size_model|has_size]] and `compare: sim_size`) or absolute count (for [[tests-format#has_n_lines_model|has_n_lines]], [[tests-format#has_n_columns_model|has_n_columns]], [[tests-format#has_image_width_model|has_image_width]], etc.). Suffix multipliers documented in the schema — `1K`, `1M`, `1G` work. - -**`delta_frac:`** is a fraction (0.1 = 10%). Use when expected size scales with input volume. Three IWC tests use it (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`); the rest use absolute `delta:`. - -**Picking magnitudes** (from corpus survey in [[iwc-shortcuts-anti-patterns]] §2): - -- Image dimensions: `delta: 25–30` pixels (5% of typical matplotlib defaults). -- Image file size: `delta: 5K–60K` (5–10% of file size). -- Small text reports: `delta: 1K–10K`. -- HTML reports: `delta: 25K–100K`. -- BAM files: `delta: 1M–10M`. -- RepeatModeler / Bayesian sampler outputs: `delta: 30K–90M` (extreme, but justified by the underlying nondeterminism). - -**Heuristic for new outputs:** `delta_frac: 0.1` is a defensible default. Tighten if the output proves more deterministic than expected. - -## 4. Text family — [[tests-format#has_text_model|has_text]] vs [[tests-format#has_text_matching_model|has_text_matching]] vs [[tests-format#has_line_model|has_line]] vs [[tests-format#has_line_matching_model|has_line_matching]] vs [[tests-format#has_n_lines_model|has_n_lines]] - -All five are common; choose by what you're verifying. - -- **[[tests-format#has_text_model|has_text]]** — output contains the substring `text:`. Anywhere in the output, any number of times. Add `n:` / `min:` / `max:` to constrain occurrence count. Add `delta:` to allow slack on the count. -- **[[tests-format#has_text_matching_model|has_text_matching]]** — output matches the regex `expression:`. Use sparingly; prefer literal [[tests-format#has_text_model|has_text]] when you can. -- **[[tests-format#has_line_model|has_line]]** — output has at least one *line* matching `line:` exactly. Use when line boundaries matter (e.g. asserting on a specific row in a table). -- **[[tests-format#has_line_matching_model|has_line_matching]]** — same but with regex. -- **[[tests-format#has_n_lines_model|has_n_lines]]** — assert the line count is `n:` ± `delta:`. - -A common combination in IWC: `has_n_lines: n: 100, delta: 5` + `has_text: text: "expected_token"` — line-count sanity-check plus a content marker. This catches both truncation and content drift in one assertion pair. - -`negate: true` is supported on every assertion. Used for the "this output should NOT contain X" case. - -## 5. Collection output assertions (`element_tests:`) - -For Galaxy collection outputs, the test format keys element assertions by element identifier: +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. ```yaml -my_collection_output: - element_tests: - sample_1: +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: asserts: - has_text: - text: "expected" - sample_2: - file: test-data/expected_sample_2.txt -``` - -Optional `attributes:` at the collection level can assert on the produced collection shape: - -```yaml -my_collection_output: - attributes: {collection_type: list:list} - element_tests: - ... + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 ``` -Nested collections: outer `element_tests:` keyed by outer identifier; inner uses `elements:` (note plural, no `_tests` suffix on the inner). See [[iwc-test-data-conventions]] §2f for the live example. - -For a list-of-files where every element should pass the same minimal check, the existence-probe pattern (`has_text: "{"` for JSON; `has_size: min: 100` for any non-empty binary) is widely used and accepted in IWC. - -## 6. The validate-against-workflow inner loop - -A `-tests.yml` file can be **structurally invalid** in two distinct ways: - -1. **Schema-invalid** — wrong field names, wrong nesting, wrong types. Caught by the test-format JSON Schema. -2. **Workflow-incoherent** — schema-valid YAML, but the input/output labels don't match the actual workflow. Renaming an output in the `.ga` and forgetting to update its sibling `-tests.yml` produces this case. Planemo will surface it as an "output not found" error at test-runtime, but only after a full workflow run. - -The `@galaxy-tool-util/schema` npm package ships **two** validators that catch both cases statically — no Galaxy or Planemo invocation needed: +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. -- **`validateTestsFile(yaml)`** — runs the file against `tests.schema.json` (AJV). Reports schema violations with paths. -- **`checkTestsAgainstWorkflow(workflow, tests)`** — cross-checks a `.ga` / format2 workflow against a tests file: missing input labels, missing output labels, type incompatibilities (e.g. test supplies a `File` for a parameter typed `int`). +## 5. Collection and structured-output examples -Both are pure-JS, take milliseconds, and have no Galaxy dependency. Wire them into the inner authoring loop: +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" ``` -edit -tests.yml - → validateTestsFile() # schema gate - → checkTestsAgainstWorkflow(.ga, tests) # coherence gate - → planemo workflow_test_on_invocation # assertion gate (no full re-run) - → planemo test # full integration (slow) -``` - -The first two gates short-circuit cheap mistakes before a slow planemo run. They are the static-validation equivalent of `gxwf` for tests, and the `implement-galaxy-workflow-test` mold should reference them as its primary inner-loop tooling. Source: galaxy-tool-util-ts package, `src/test-format/index.ts` exports. - -## 7. Authoring loop — generation, then refinement - -Reviewer convention is to **generate** the initial `-tests.yml` rather than hand-write it. Two planemo subcommands cover this: - -- **`planemo workflow_test_init --from_invocation `** ([[planemo-workflow_test_init]]) — given a successful Galaxy invocation ID, emit a `-tests.yml` with a `job:` block that captures all inputs (with SHA-1 hashes) and an `outputs:` block with `file:` references to the actual outputs (downloaded into `test-data/`). Hand-tighten the assertions afterward. -- **`planemo workflow_test_on_invocation `** ([[planemo-workflow_test_on_invocation]]) — re-evaluate an edited `-tests.yml` against a saved invocation without re-running the workflow. The fast inner loop for assertion iteration; complements the static gates in §6. - -Together these cut the assertion-iteration cost dramatically. An agent should: - -1. Run the workflow once on usegalaxy.* (or local) to get a known-good invocation. -2. `--from_invocation` to bootstrap the test file. -3. Replace the autogenerated `file:` exact-comparison assertions with assertion-family-appropriate alternatives per §1. -4. [[planemo-workflow_test_on_invocation]] after each edit; full [[planemo-test]] at the end. - -## 8. What the schema gives you for free - -When the test-format schema lands as a Foundry-rendered note, the agent can consult any assertion's `$def` directly for: parameter types, defaults, required fields, the `that` discriminator constant, and the original Python docstring (carried through as `description`). This note does **not** restate that vocabulary — it complements it with the corpus-grounded *which-and-when*. - -What's still missing from the schema and worth keeping in research notes: - -- This decision table (§1) — output-type → assertion family. -- Tolerance magnitudes (§3) — corpus-derived defaults. -- The `validateTestsFile` / `checkTestsAgainstWorkflow` integration story (§6). -- Anti-pattern flags — see [[iwc-shortcuts-anti-patterns]]. -## 9. Common combinations (recipes) +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. -Six recipes worth memorizing. +## 6. Validate labels and schema before execution -**Stable text report (FastQC summary, simple stats).** -```yaml -my_report: - asserts: - has_n_lines: { n: 12, delta: 2 } - has_text: { text: "Total Sequences" } -``` - -**MultiQC HTML report.** -```yaml -multiqc_report: - asserts: - has_text: { text: "Filtered Reads" } - has_text: { text: "FastQC" } -``` +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: -**VCF (pinned tool, fixed reference).** -```yaml -called_variants: - file: test-data/expected.vcf - compare: diff - lines_diff: 6 +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga ``` -**Stochastic JSON (HyPhy-style).** -```yaml -hyphy_meme: - element_tests: - geneA: { asserts: { has_text: { text: "{" } } } - geneB: { asserts: { has_text: { text: "{" } } } -``` +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. -**Matplotlib plot.** -```yaml -umap_plot: - asserts: - has_size: { size: 68416, delta: 6000 } - has_image_width: { width: 601, delta: 30 } - has_image_height: { height: 429, delta: 25 } -``` +## 7. Create and refine the test -**AnnData (HDF5).** -```yaml -clustered_anndata: - asserts: - has_h5_keys: { keys: "obs/louvain" } - has_h5_keys: { keys: "var/highly_variable" } - has_h5_keys: { keys: "uns/rank_genes_groups" } - has_size: { size: 12000000, delta: 1500000 } -``` +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. -## 10. Cross-references +## 8. Source of the exact syntax -- [[galaxy-workflow-testability-design]] — decide which workflow outputs and checkpoints to expose before choosing assertions. -- [[iwc-test-data-conventions]] — input-side conventions (job inputs, collection shapes, `hashes:`, CVMFS). -- [[iwc-shortcuts-anti-patterns]] — accepted-vs-smell catalog and corpus prevalence; this note's mirror image. -- Test-format schema (`@galaxy-tool-util/schema` npm package) — authoritative vocabulary; will be vendored into a Foundry-rendered schema note. See `content/meta/casting.md` for the casting story. -- Planemo test-format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- [[galaxy-xsd]] — Galaxy XSD assertion source of truth, vendored from upstream. -- Tightening of the schema and Pydantic source: [galaxyproject/galaxy#22566](https://github.com/galaxyproject/galaxy/pull/22566). -- TS schema sync into npm: [jmchilton/galaxy-tool-util-ts#75](https://github.com/jmchilton/galaxy-tool-util-ts/pull/75). +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json index 1539281..82e4529 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json @@ -358,7 +358,7 @@ "properties": { "family": { "type": "string", - "description": "tests-format assertion family or compare operator this intent maps to, by name, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys, file, compare:diff, compare:sim_size. Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." + "description": "tests-format assertion family, compare operator, or output-attribute field this intent maps to, by name. Assertion families go inside the output's `asserts:`, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys. Compare operators and output-attribute fields sit beside `asserts:`: file, compare:diff, compare:sim_size, element_count (collection outputs), count (nested sub-collections), ftype, checksum, metadata. Do not use collection_type (state it in ExpectedOutput.collection_shape) or element_tests (a non-null element_identifier implies it). Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." }, "intent": { "type": "string", @@ -390,15 +390,16 @@ "string", "null" ], - "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions." + "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions. Address an element of a nested collection by joining identifiers outermost first with `/`, e.g. WT_REP1/forward in a list:paired." }, "evidence": { "type": "string", "enum": [ "test-evidence", + "fixture-derived", "intent" ], - "description": "Whether this assertion was translated from upstream test evidence or synthesized from intent." + "description": "Basis of the expected value: test-evidence means translated from an upstream assertion, checksum, or expected file; fixture-derived means measured from upstream test fixture bytes where no upstream assertion exists (e.g. snapshot-only nf-tests); intent means synthesized from workflow intent, wrapper XML, or step state with no upstream anchor." }, "confidence": { "type": "string", diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json index 8de90e7..99d551f 100644 --- a/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json +++ b/plugins/foundry-skills/skills/nextflow-test-to-galaxy-test-plan/references/schemas/tests-format.schema.json @@ -1110,6 +1110,18 @@ "default": "Collection", "title": "Class" }, + "count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Count" + }, "element_tests": { "anyOf": [ { @@ -1171,6 +1183,30 @@ ], "default": null, "title": "Elements" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Maximum Count" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum Count" } }, "title": "TestCollectionCollectionElementAssertions", @@ -1366,6 +1402,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestCollectionDatasetElementAssertions", @@ -1679,6 +1728,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestDataOutputAssertions", diff --git a/plugins/foundry-skills/skills/nextflow-to-test-data/SKILL.md b/plugins/foundry-skills/skills/nextflow-to-test-data/SKILL.md index 7f10011..8f67640 100644 --- a/plugins/foundry-skills/skills/nextflow-to-test-data/SKILL.md +++ b/plugins/foundry-skills/skills/nextflow-to-test-data/SKILL.md @@ -18,7 +18,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Outputs -- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Test data resolved from the pipeline's declared fixtures, expressed as URLs/paths plus expected shapes for downstream test authoring. Shared id with find-test-data — the branch's search fallback fills only the inputs this Mold leaves unresolved. +- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Schema: test-data-refs. Test data resolved from the pipeline's declared fixtures, expressed as URLs/paths plus expected shapes for downstream test authoring. Shared id with find-test-data — the branch's search fallback fills only the inputs this Mold leaves unresolved. ## Required Tools @@ -27,6 +27,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load Upfront - `references/schemas/summary-nextflow.schema.json`: Schema file copied verbatim into the bundle. Input contract: read `test_selection` and the chosen `test_candidates[].inputs[]` — role, url/path, sha1, filetype — as the declared source of test data. +- `references/schemas/test-data-refs.schema.json`: Schema file copied verbatim into the bundle. Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning. ## Load On Demand @@ -37,7 +38,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- None declared. +- Before starting, check that the installed validator carries the test-data-refs schema this bundle was cast with: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `test-data-refs.json` before returning it: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json`. This checks artifact `test-data-refs` against the test-data-refs schema. ## Procedure @@ -50,8 +52,8 @@ This skill is the source-specific first leg of the harness's `test-data-resoluti 1. **Enumerate Galaxy inputs and their required shape.** From the interface brief, list each workflow input: label, Galaxy collection shape (File / list / paired / list:paired / record), and datatype. This is the *target shape* every ref must satisfy. 2. **Resolve the candidate.** Read `test_selection`. If it names a candidate, collect that candidate's declared inputs: `role`, `url`/`path`, `sha1`, `filetype`, and description. If it reports `needs-scope-choice`, compare the candidates' scope, execution mode, and input coverage and record the caller's choice; do not infer scale or representativeness from a profile name alone. 3. **Map each declared fixture onto a Galaxy input.** Match by role and shape onto the interface's input labels. A samplesheet-driven Nextflow input often expands into a Galaxy collection — record the element identifiers and any split/concatenation prep needed to reach the Galaxy collection shape (galaxy-workflow-testability-design). -4. **Emit refs.** Write one `test-data-refs.json` entry per resolved input: prefer the declared remote `url` + `sha1` (remote-URL-first, iwc-test-data-conventions); fall back to the in-tree `path` plus provenance only when no URL is published. Carry datatype, collection element identifiers, and any subset/split prep. Each entry maps to an addressable workflow input label. -5. **Report genuine gaps.** An input with no declared fixture of the right shape stays `resolved: false` with a reason — this is what the harness hands to find-test-data. Do not search public sources here; searching is find-test-data's job, not this skill's. +4. **Emit refs.** Write one `test-data-refs.json` entry per resolved input: prefer the declared remote `url` + `sha1` (remote-URL-first, iwc-test-data-conventions); fall back to the in-tree `path` plus provenance only when no URL is published. Carry datatype, collection element identifiers, and any subset/split prep. Each entry maps to an addressable workflow input label. Shape each entry per test-data-refs with `resolved_by: nextflow-to-test-data`, and validate the file with `foundry validate-test-data-refs` before returning. +5. **Report genuine gaps.** An input with no declared fixture of the right shape stays `resolved: false` with a reason and `storage: unresolved` — this is what the harness hands to find-test-data. Do not search public sources here; searching is find-test-data's job, not this skill's. ### No fabrication diff --git a/plugins/foundry-skills/skills/nextflow-to-test-data/_provenance.json b/plugins/foundry-skills/skills/nextflow-to-test-data/_provenance.json index 7823479..6f43e19 100644 --- a/plugins/foundry-skills/skills/nextflow-to-test-data/_provenance.json +++ b/plugins/foundry-skills/skills/nextflow-to-test-data/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "nextflow-to-test-data", "path": "content/molds/nextflow-to-test-data/index.md", - "revision": 3, - "content_hash": "7326b3810ca2c2b6216d868e1c10c1d95b0b8ecca5a3314f1a800a1b3f67443f", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "revision": 4, + "content_hash": "842e097edfb5bfbc37b62a35d5870ee2087df33919212a7965fd70dae799b189", + "commit": "871b5e6deadfd56aeb5e2ba92eb8f6f50c8449ee" }, - "cast_at": "2026-09-15T23:16:25.343Z", + "cast_at": "2026-11-10T14:49:42.555Z", "refs": [ { "kind": "research", @@ -53,8 +53,8 @@ "evidence": "corpus-observed", "purpose": "Map each declared fixture to an addressable Galaxy input label and the collection shape it must populate.", "trigger": "When mapping a declared fixture (often a samplesheet-driven input) onto a Galaxy input's collection shape.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -68,8 +68,8 @@ "evidence": "corpus-observed", "purpose": "Express each ref remote-URL-first with SHA-1 integrity and per-input collection layout when recording resolved fixtures.", "trigger": "When writing each `test-data-refs` entry.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -83,8 +83,24 @@ "evidence": "corpus-observed", "purpose": "Input contract: read `test_selection` and the chosen `test_candidates[].inputs[]` — role, url/path, sha1, filetype — as the declared source of test data.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[test-data-refs]]", + "src": "package://@galaxy-foundry/gxwf-foundry#testDataRefsSchema", + "dst": "references/schemas/test-data-refs.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning.", + "verification": "Run the test-data-resolution chain on a pinned scenario and confirm each leg's test-data-refs.json validates and implement-galaxy-workflow-test stages from it.", + "license": "MIT", + "src_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "dst_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", "source": "deterministic" } ], @@ -94,6 +110,7 @@ "id": "test-data-refs", "kind": "json", "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", "description": "Test data resolved from the pipeline's declared fixtures, expressed as URLs/paths plus expected shapes for downstream test authoring. Shared id with [[find-test-data]] — the branch's search fallback fills only the inputs this Mold leaves unresolved." } ], diff --git a/plugins/foundry-skills/skills/nextflow-to-test-data/_verify.json b/plugins/foundry-skills/skills/nextflow-to-test-data/_verify.json index 667ae3b..1e2da49 100644 --- a/plugins/foundry-skills/skills/nextflow-to-test-data/_verify.json +++ b/plugins/foundry-skills/skills/nextflow-to-test-data/_verify.json @@ -10,6 +10,22 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "{artifact_path}" + ] + }, + { + "artifact_id": "test-data-refs", + "direction": "output", + "kind": "json", + "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", + "validator_bin": "foundry", + "args": [ + "validate-test-data-refs", + "--schema-sha256", + "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/nextflow-to-test-data/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/nextflow-to-test-data/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/nextflow-to-test-data/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/nextflow-to-test-data/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/nextflow-to-test-data/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/nextflow-to-test-data/references/schemas/test-data-refs.schema.json b/plugins/foundry-skills/skills/nextflow-to-test-data/references/schemas/test-data-refs.schema.json new file mode 100644 index 0000000..45df249 --- /dev/null +++ b/plugins/foundry-skills/skills/nextflow-to-test-data/references/schemas/test-data-refs.schema.json @@ -0,0 +1,291 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://galaxyproject.org/foundry/schemas/test-data-refs.schema.json", + "$comment": "Canonical source: packages/gxwf-foundry/src/schemas/test-data-refs/test-data-refs.schema.json in galaxyproject/foundry. Mold frontmatter cites this schema via [[test-data-refs]] wiki-links; the cast pipeline imports the `testDataRefsSchema` runtime export and serializes it into cast bundles. $defs.Fixture is a verbatim copy of the galaxy-workflow-test-plan Fixture; a test keeps them equal.", + "title": "Galaxy workflow test-data refs", + "description": "Test data for each Galaxy workflow input, written by the test-data-resolution chain (a source-specific Mold, then find-test-data) and read by implement-galaxy-workflow-test. One ref per workflow input; an input no leg could resolve stays in the list as an honest gap.", + "$ref": "#/$defs/TestDataRefs", + "$defs": { + "TestDataRefs": { + "type": "object", + "additionalProperties": false, + "required": [ + "refs" + ], + "properties": { + "refs": { + "type": "array", + "items": { + "$ref": "#/$defs/Ref" + }, + "description": "One entry per workflow input label. A later leg of the chain keeps every entry an earlier leg resolved, unchanged, and replaces only unresolved ones." + } + } + }, + "Ref": { + "type": "object", + "additionalProperties": false, + "required": [ + "workflow_label", + "collection_shape", + "datatype", + "resolved", + "resolved_by", + "reason", + "fixture" + ], + "properties": { + "workflow_label": { + "type": "string", + "minLength": 1, + "description": "Workflow input this ref feeds: its label in the Galaxy interface brief, or its input port in the CWL design." + }, + "collection_shape": { + "type": [ + "string", + "null" + ], + "description": "Galaxy collection shape when the input is a collection, e.g. list, paired, list:paired, record; null for a single dataset." + }, + "datatype": { + "type": [ + "string", + "null" + ], + "description": "Galaxy datatype/extension, e.g. fastqsanger.gz, fasta, tabular; null when undecided." + }, + "resolved": { + "type": "boolean", + "description": "true when the ref points at real data of the required shape; false for an honest gap." + }, + "resolved_by": { + "type": [ + "string", + "null" + ], + "description": "Name of the Mold that resolved this input, e.g. nextflow-to-test-data or find-test-data; null while unresolved." + }, + "reason": { + "type": [ + "string", + "null" + ], + "description": "Why the input is unresolved; null when resolved." + }, + "fixture": { + "$ref": "#/$defs/Fixture", + "description": "Where the data lives. For a collection whose elements carry their own fixtures, storage and location may be null." + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + }, + "description": "Collection elements, nested for nested shapes (list:paired → sample → forward/reverse). Identifiers must match the staged test file." + }, + "prep": { + "type": [ + "string", + "null" + ], + "description": "Data-import-boundary prep needed to reach the Galaxy shape from the source data: a subset, split, or concatenation. A note on the ref, not an analysis step." + } + }, + "allOf": [ + { + "if": { + "properties": { + "resolved": { + "const": false + } + } + }, + "then": { + "properties": { + "resolved_by": { + "type": "null" + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "properties": { + "storage": { + "const": "unresolved" + }, + "location": { + "type": "null" + } + } + } + } + }, + "else": { + "properties": { + "resolved_by": { + "type": "string", + "minLength": 1 + }, + "reason": { + "type": "null" + } + }, + "anyOf": [ + { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + } + }, + { + "if": { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + "then": { + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + } + } + } + } + } + } + ] + }, + "Element": { + "type": "object", + "additionalProperties": false, + "required": [ + "identifier" + ], + "properties": { + "identifier": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "$ref": "#/$defs/Fixture" + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + } + } + } + }, + "DataElement": { + "description": "An element of a resolved collection: it carries a located fixture, or nested elements that each do.", + "anyOf": [ + { + "required": [ + "fixture" + ], + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + }, + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + }, + "Fixture": { + "type": "object", + "additionalProperties": false, + "required": [ + "storage", + "location", + "checksum", + "provenance" + ], + "properties": { + "storage": { + "type": [ + "string", + "null" + ], + "enum": [ + "remote-url", + "in-repo", + "cvmfs-string", + "generated-toy", + "unresolved", + null + ], + "description": "Where the fixture data lives or should live: remote-url (Zenodo/EBI/SRA location), in-repo (committed test-data path), cvmfs-string (data-table/built-in-index bare string), generated-toy (to be synthesized), or unresolved (not yet sourced)." + }, + "location": { + "type": [ + "string", + "null" + ], + "description": "Concrete URL, repo-relative path, or data-table string when known; null when the fixture still needs resolution." + }, + "checksum": { + "type": [ + "string", + "null" + ], + "description": "Integrity hash when available, e.g. SHA-1:; null when unknown." + }, + "provenance": { + "type": "string", + "description": "Where this fixture comes from or how it was chosen, e.g. a translated source test-data reference, a Zenodo record, or a note that test-data-resolution must supply it." + } + } + } + } +} diff --git a/plugins/foundry-skills/skills/paper-to-test-data/SKILL.md b/plugins/foundry-skills/skills/paper-to-test-data/SKILL.md index fa87e38..07531a3 100644 --- a/plugins/foundry-skills/skills/paper-to-test-data/SKILL.md +++ b/plugins/foundry-skills/skills/paper-to-test-data/SKILL.md @@ -1,6 +1,6 @@ --- name: paper-to-test-data -description: "Derive workflow test inputs and expected outputs from a paper." +description: "Resolve the test data a paper names into refs for each workflow input." --- # paper-to-test-data @@ -9,15 +9,17 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## When To Use -- Derive workflow test inputs and expected outputs from a paper. +- Resolve the test data a paper names into refs for each workflow input. ## Inputs - Read artifact `freeform-summary`. Produced by `interview-to-freeform-summary`, `summarize-paper`. Free-form summary from summarize-paper; sample data and reference evidence the test fixtures derive from. +- Read artifact `freeform-galaxy-interface`. Produced by `freeform-summary-to-galaxy-interface`. Galaxy interface brief from freeform-summary-to-galaxy-interface pinning the workflow input labels, collection shapes, and datatypes each ref must map onto, in PAPER → GALAXY. +- Read artifact `freeform-cwl-design`. Produced by `freeform-summary-to-cwl-design`. CWL design from freeform-summary-to-cwl-design naming the workflow input ports each ref must map onto, in PAPER → CWL. ## Outputs -- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Resolved workflow test inputs and expected outputs derived from paper evidence (URLs, file shapes, expected hashes). +- Write artifact `test-data-refs` as `test-data-refs.json`. Format: `json`. Schema: test-data-refs. Test data the paper names, one ref per workflow input (URLs, shapes, hashes). Shared id with find-test-data, which fills only the inputs this Mold leaves unresolved. ## Required Tools @@ -25,7 +27,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Load Upfront -- None declared. +- `references/schemas/test-data-refs.schema.json`: Schema file copied verbatim into the bundle. Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning. ## Load On Demand @@ -33,11 +35,12 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- None declared. +- Before starting, check that the installed validator carries the test-data-refs schema this bundle was cast with: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `test-data-refs.json` before returning it: run `foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-test-data-refs --schema-sha256 534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8 test-data-refs.json`. This checks artifact `test-data-refs` against the test-data-refs schema. ## Procedure -Read the paper free-form summary from summarize-paper and derive concrete workflow test inputs and expected outputs — resolvable URLs, file shapes, and expected hashes — emitted as `test-data-refs`. +Read the paper free-form summary from summarize-paper and resolve the data the paper names (resolvable URLs, file shapes, expected hashes) onto the workflow inputs the target declares — the Galaxy interface brief's input labels, or the CWL design's input ports — as `test-data-refs` with `resolved_by: paper-to-test-data`. An input the paper names no data for stays `resolved: false` with a reason, for find-test-data to search. Expected outputs are the test plan's to derive, not this skill's. Validate the file with `foundry validate-test-data-refs` before returning. ## Feedback Mode diff --git a/plugins/foundry-skills/skills/paper-to-test-data/_provenance.json b/plugins/foundry-skills/skills/paper-to-test-data/_provenance.json index 1e5d275..134ea77 100644 --- a/plugins/foundry-skills/skills/paper-to-test-data/_provenance.json +++ b/plugins/foundry-skills/skills/paper-to-test-data/_provenance.json @@ -4,19 +4,37 @@ "mold": { "name": "paper-to-test-data", "path": "content/molds/paper-to-test-data/index.md", - "revision": 2, - "content_hash": "863d7b503a77fa63449899f124d1669e181fdfd9a72f81f516a219cdc89cb6bb", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 3, + "content_hash": "cdb6a8d7ca4c84f6dddd457ceb9f48a1286d92a865f3577c8d647d0d0a9765e5", + "commit": "2bdd9b9b68a71fd161362922e9d1ae797d10a8ce" }, - "cast_at": "2026-09-13T20:58:29.564Z", - "refs": [], + "cast_at": "2026-10-16T22:32:04.310Z", + "refs": [ + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[test-data-refs]]", + "src": "package://@galaxy-foundry/gxwf-foundry#testDataRefsSchema", + "dst": "references/schemas/test-data-refs.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Output contract: one ref per workflow input, resolved or an honest gap with a reason. Cast bundles the JSON Schema; validate with `foundry validate-test-data-refs` before returning.", + "verification": "Run the test-data-resolution chain on a pinned scenario and confirm each leg's test-data-refs.json validates and implement-galaxy-workflow-test stages from it.", + "license": "MIT", + "src_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "dst_hash": "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "source": "deterministic" + } + ], "artifacts": { "produces": [ { "id": "test-data-refs", "kind": "json", "default_filename": "test-data-refs.json", - "description": "Resolved workflow test inputs and expected outputs derived from paper evidence (URLs, file shapes, expected hashes)." + "schema": "[[test-data-refs]]", + "description": "Test data the paper names, one ref per workflow input (URLs, shapes, hashes). Shared id with [[find-test-data]], which fills only the inputs this Mold leaves unresolved." } ], "consumes": [ @@ -27,6 +45,20 @@ "interview-to-freeform-summary", "summarize-paper" ] + }, + { + "id": "freeform-galaxy-interface", + "description": "Galaxy interface brief from [[freeform-summary-to-galaxy-interface]] pinning the workflow input labels, collection shapes, and datatypes each ref must map onto, in PAPER → GALAXY.", + "producers": [ + "freeform-summary-to-galaxy-interface" + ] + }, + { + "id": "freeform-cwl-design", + "description": "CWL design from [[freeform-summary-to-cwl-design]] naming the workflow input ports each ref must map onto, in PAPER → CWL.", + "producers": [ + "freeform-summary-to-cwl-design" + ] } ] } diff --git a/plugins/foundry-skills/skills/paper-to-test-data/_verify.json b/plugins/foundry-skills/skills/paper-to-test-data/_verify.json index 67e7f8c..336d1dd 100644 --- a/plugins/foundry-skills/skills/paper-to-test-data/_verify.json +++ b/plugins/foundry-skills/skills/paper-to-test-data/_verify.json @@ -1,4 +1,19 @@ { "verify_schema_version": 1, - "entries": [] + "entries": [ + { + "artifact_id": "test-data-refs", + "direction": "output", + "kind": "json", + "default_filename": "test-data-refs.json", + "schema": "[[test-data-refs]]", + "validator_bin": "foundry", + "args": [ + "validate-test-data-refs", + "--schema-sha256", + "534dcb2f44e24a33bd59de2bca8d882af97321c602cf1b66776c22c38b0207a8", + "{artifact_path}" + ] + } + ] } diff --git a/plugins/foundry-skills/skills/paper-to-test-data/references/schemas/test-data-refs.schema.json b/plugins/foundry-skills/skills/paper-to-test-data/references/schemas/test-data-refs.schema.json new file mode 100644 index 0000000..45df249 --- /dev/null +++ b/plugins/foundry-skills/skills/paper-to-test-data/references/schemas/test-data-refs.schema.json @@ -0,0 +1,291 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://galaxyproject.org/foundry/schemas/test-data-refs.schema.json", + "$comment": "Canonical source: packages/gxwf-foundry/src/schemas/test-data-refs/test-data-refs.schema.json in galaxyproject/foundry. Mold frontmatter cites this schema via [[test-data-refs]] wiki-links; the cast pipeline imports the `testDataRefsSchema` runtime export and serializes it into cast bundles. $defs.Fixture is a verbatim copy of the galaxy-workflow-test-plan Fixture; a test keeps them equal.", + "title": "Galaxy workflow test-data refs", + "description": "Test data for each Galaxy workflow input, written by the test-data-resolution chain (a source-specific Mold, then find-test-data) and read by implement-galaxy-workflow-test. One ref per workflow input; an input no leg could resolve stays in the list as an honest gap.", + "$ref": "#/$defs/TestDataRefs", + "$defs": { + "TestDataRefs": { + "type": "object", + "additionalProperties": false, + "required": [ + "refs" + ], + "properties": { + "refs": { + "type": "array", + "items": { + "$ref": "#/$defs/Ref" + }, + "description": "One entry per workflow input label. A later leg of the chain keeps every entry an earlier leg resolved, unchanged, and replaces only unresolved ones." + } + } + }, + "Ref": { + "type": "object", + "additionalProperties": false, + "required": [ + "workflow_label", + "collection_shape", + "datatype", + "resolved", + "resolved_by", + "reason", + "fixture" + ], + "properties": { + "workflow_label": { + "type": "string", + "minLength": 1, + "description": "Workflow input this ref feeds: its label in the Galaxy interface brief, or its input port in the CWL design." + }, + "collection_shape": { + "type": [ + "string", + "null" + ], + "description": "Galaxy collection shape when the input is a collection, e.g. list, paired, list:paired, record; null for a single dataset." + }, + "datatype": { + "type": [ + "string", + "null" + ], + "description": "Galaxy datatype/extension, e.g. fastqsanger.gz, fasta, tabular; null when undecided." + }, + "resolved": { + "type": "boolean", + "description": "true when the ref points at real data of the required shape; false for an honest gap." + }, + "resolved_by": { + "type": [ + "string", + "null" + ], + "description": "Name of the Mold that resolved this input, e.g. nextflow-to-test-data or find-test-data; null while unresolved." + }, + "reason": { + "type": [ + "string", + "null" + ], + "description": "Why the input is unresolved; null when resolved." + }, + "fixture": { + "$ref": "#/$defs/Fixture", + "description": "Where the data lives. For a collection whose elements carry their own fixtures, storage and location may be null." + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + }, + "description": "Collection elements, nested for nested shapes (list:paired → sample → forward/reverse). Identifiers must match the staged test file." + }, + "prep": { + "type": [ + "string", + "null" + ], + "description": "Data-import-boundary prep needed to reach the Galaxy shape from the source data: a subset, split, or concatenation. A note on the ref, not an analysis step." + } + }, + "allOf": [ + { + "if": { + "properties": { + "resolved": { + "const": false + } + } + }, + "then": { + "properties": { + "resolved_by": { + "type": "null" + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "properties": { + "storage": { + "const": "unresolved" + }, + "location": { + "type": "null" + } + } + } + } + }, + "else": { + "properties": { + "resolved_by": { + "type": "string", + "minLength": 1 + }, + "reason": { + "type": "null" + } + }, + "anyOf": [ + { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + } + }, + { + "if": { + "properties": { + "fixture": { + "properties": { + "location": { + "type": "string" + } + } + } + } + }, + "then": { + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + } + } + } + } + } + } + ] + }, + "Element": { + "type": "object", + "additionalProperties": false, + "required": [ + "identifier" + ], + "properties": { + "identifier": { + "type": "string", + "minLength": 1 + }, + "fixture": { + "$ref": "#/$defs/Fixture" + }, + "elements": { + "type": "array", + "items": { + "$ref": "#/$defs/Element" + } + } + } + }, + "DataElement": { + "description": "An element of a resolved collection: it carries a located fixture, or nested elements that each do.", + "anyOf": [ + { + "required": [ + "fixture" + ], + "properties": { + "fixture": { + "properties": { + "storage": { + "type": "string" + }, + "location": { + "type": "string" + } + } + } + } + }, + { + "required": [ + "elements" + ], + "properties": { + "elements": { + "minItems": 1, + "items": { + "$ref": "#/$defs/DataElement" + } + } + } + } + ] + }, + "Fixture": { + "type": "object", + "additionalProperties": false, + "required": [ + "storage", + "location", + "checksum", + "provenance" + ], + "properties": { + "storage": { + "type": [ + "string", + "null" + ], + "enum": [ + "remote-url", + "in-repo", + "cvmfs-string", + "generated-toy", + "unresolved", + null + ], + "description": "Where the fixture data lives or should live: remote-url (Zenodo/EBI/SRA location), in-repo (committed test-data path), cvmfs-string (data-table/built-in-index bare string), generated-toy (to be synthesized), or unresolved (not yet sourced)." + }, + "location": { + "type": [ + "string", + "null" + ], + "description": "Concrete URL, repo-relative path, or data-table string when known; null when the fixture still needs resolution." + }, + "checksum": { + "type": [ + "string", + "null" + ], + "description": "Integrity hash when available, e.g. SHA-1:; null when unknown." + }, + "provenance": { + "type": "string", + "description": "Where this fixture comes from or how it was chosen, e.g. a translated source test-data reference, a Zenodo record, or a note that test-data-resolution must supply it." + } + } + } + } +} diff --git a/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/SKILL.md b/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/SKILL.md index cd6f6ea..d70e253 100644 --- a/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/SKILL.md @@ -1,15 +1,15 @@ --- name: pipeline-cwl-to-galaxy -description: "Path from a CWL Workflow to a Galaxy gxformat2 workflow — orchestrates the Foundry skills of the CWL → GALAXY pipeline in order, in a per-run working directory." +description: "Translate a CWL Workflow into a Galaxy gxformat2 workflow, then assemble and run a Galaxy workflow test — orchestrates the Foundry skills of the CWL → GALAXY pipeline in order, in a per-run working directory." --- # pipeline-cwl-to-galaxy -Harness for the **CWL → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/cwl-to-galaxy/index.md` (revision 2) — regenerate with `foundry-build assemble-pipeline cwl-to-galaxy` if the pipeline changes; do not hand-edit. +Harness for the **CWL → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/cwl-to-galaxy/index.md` (revision 5) — regenerate with `foundry-build assemble-pipeline cwl-to-galaxy` if the pipeline changes; do not hand-edit. ## When To Use -- Path from a CWL Workflow to a Galaxy gxformat2 workflow. Lighter upstream extraction. +- Translate a CWL Workflow into a Galaxy gxformat2 workflow, then assemble and run a Galaxy workflow test. ## Bootstrap (install these CLIs first) @@ -27,16 +27,16 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. Check: `foundry --help`. Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md -- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache`. +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. Check: `galaxy-tool-cache --help | grep -q summarize`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -45,7 +45,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -53,11 +53,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `cwl-to-galaxy-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: cwl-to-galaxy`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-cwl, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -82,7 +95,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//` and any phases handled manually (marked MANUAL above). -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/_assembly.json b/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/_assembly.json index 9ca02ef..1383811 100644 --- a/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-cwl-to-galaxy/_assembly.json @@ -1,15 +1,15 @@ { "source_pipeline": "cwl-to-galaxy", - "source_revision": 2, + "source_revision": 5, "harness_name": "pipeline-cwl-to-galaxy", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ { "tool": "cwl-utils", "origin": "pypi", "package": "cwl-utils", "invoke": "cwl-normalizer", "invoke_fallback": "uvx --from cwl-utils cwl-normalizer", "availability_check": "cwl-normalizer --help", "docs_url": "https://github.com/common-workflow-language/cwl-utils", "source": "referenced" }, { "tool": "cwltool", "origin": "pypi", "package": "cwltool", "invoke": "cwltool", "invoke_fallback": "uvx cwltool", "availability_check": "cwltool --version", "docs_url": "https://cwltool.readthedocs.io/", "source": "referenced" }, { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "referenced" }, - { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list"] }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list", "galaxy-tool-cache summarize"] }, + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-cwl", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/SKILL.md b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/SKILL.md index 3d8dbfd..ed3bc60 100644 --- a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/SKILL.md @@ -5,7 +5,7 @@ description: "Apply the pinned IWC publication checklist to an existing Galaxy w # pipeline-galaxy-workflow-maturation -Harness for the **GALAXY WORKFLOW MATURATION** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/galaxy-workflow-maturation/index.md` (revision 1) — regenerate with `foundry-build assemble-pipeline galaxy-workflow-maturation` if the pipeline changes; do not hand-edit. +Harness for the **GALAXY WORKFLOW MATURATION** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/galaxy-workflow-maturation/index.md` (revision 3) — regenerate with `foundry-build assemble-pipeline galaxy-workflow-maturation` if the pipeline changes; do not hand-edit. ## When To Use @@ -19,12 +19,12 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. Check: `foundry --help`. Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -33,7 +33,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -41,11 +41,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `galaxy-workflow-maturation-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: galaxy-workflow-maturation`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-galaxy-workflow, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -59,7 +72,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//`. -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes @@ -67,7 +82,7 @@ If the run was invoked with `--feedback`, close the ledger first: set the final - Carry unresolved assumptions forward as notes rather than inventing missing inputs. - Entry is an existing workflow, not a construction run. The harness supplies a `.gxwf.yml` or `.ga` file; phase 1 normalizes `.ga` to gxformat2 and every checklist judgement downstream is stated against that normalized baseline. - Phase 2's optional inputs have no producer in this pipeline. `galaxy-workflow-test`, `open-requirements-ledger`, `iwc-publication-context`, `iwc-workflow-readme`, `iwc-workflow-changelog`, and `iwc-dockstore-metadata` are supplied by the caller or carried in from a prior Foundry run. Absent is a normal result; the harness must not synthesize them. -- `iwc-maturation-report.md` is the run deliverable a human reads. Phases 3 and 4 add validation and Planemo evidence beside it; neither rewrites it. +- `iwc-maturation-report.md` is the run deliverable a human reads. Phases 3 and 4 add validation and Planemo evidence beside it; neither rewrites it. That evidence, and any repair attempts, never go into the README; they belong beside the report and in the pull request description. - Phases 3 and 4 declare no input artifacts, so nothing in the artifact graph binds them to phase 2's output. The harness must hand phase 2's emitted workflow to phase 3 and its emitted test to phase 4 — that obligation lives here and in `eval.md`, not in a validator gate. - A red phase 3 or phase 4 is retained evidence, not a reason to loosen the workflow or the test. Report the failure together with the phase-2 checklist items most likely to have caused it and stop; the user revises an ambiguous decision or invokes [[debug-galaxy-workflow-output]] and reruns. - When the caller supplies a prior green `workflow-test-result`, compare the phase-4 result against it and name any regression explicitly. That comparison is a harness obligation — a prior result has no declared consumer in this spine. Without one, the phase-4 outcome becomes the baseline. diff --git a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/_assembly.json b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/_assembly.json index 94a9eff..5e1e4fc 100644 --- a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-maturation/_assembly.json @@ -1,12 +1,12 @@ { "source_pipeline": "galaxy-workflow-maturation", - "source_revision": 1, + "source_revision": 3, "harness_name": "pipeline-galaxy-workflow-maturation", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "referenced" }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf validate", "gxwf validate-tests"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf validate", "gxwf validate-tests"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-galaxy-workflow", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/SKILL.md b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/SKILL.md index 7214531..92a4995 100644 --- a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/SKILL.md @@ -5,7 +5,7 @@ description: "Post-construction review journey: summarize, validate, and test an # pipeline-galaxy-workflow-review -Harness for the **GALAXY WORKFLOW REVIEW** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/galaxy-workflow-review/index.md` (revision 1) — regenerate with `foundry-build assemble-pipeline galaxy-workflow-review` if the pipeline changes; do not hand-edit. +Harness for the **GALAXY WORKFLOW REVIEW** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/galaxy-workflow-review/index.md` (revision 2) — regenerate with `foundry-build assemble-pipeline galaxy-workflow-review` if the pipeline changes; do not hand-edit. ## When To Use @@ -19,12 +19,12 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. Check: `foundry --help`. Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -33,7 +33,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -41,11 +41,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `galaxy-workflow-review-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: galaxy-workflow-review`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-galaxy-workflow, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -59,7 +72,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//`. -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/_assembly.json b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/_assembly.json index b7eb34f..5d77e0f 100644 --- a/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-galaxy-workflow-review/_assembly.json @@ -1,12 +1,12 @@ { "source_pipeline": "galaxy-workflow-review", - "source_revision": 1, + "source_revision": 2, "harness_name": "pipeline-galaxy-workflow-review", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "referenced" }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf validate", "gxwf validate-tests"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf validate", "gxwf validate-tests"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-galaxy-workflow", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/SKILL.md b/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/SKILL.md index 2a9afe8..0230ae1 100644 --- a/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/SKILL.md @@ -5,7 +5,7 @@ description: "Interview-driven path to a Galaxy gxformat2 workflow through the s # pipeline-interview-to-galaxy -Harness for the **INTERVIEW → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/interview-to-galaxy/index.md` (revision 2) — regenerate with `foundry-build assemble-pipeline interview-to-galaxy` if the pipeline changes; do not hand-edit. +Harness for the **INTERVIEW → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/interview-to-galaxy/index.md` (revision 5) — regenerate with `foundry-build assemble-pipeline interview-to-galaxy` if the pipeline changes; do not hand-edit. ## When To Use @@ -15,16 +15,16 @@ Harness for the **INTERVIEW → GALAXY** Foundry pipeline. Runs the constituent Install the harness CLIs every constituent skill invokes before driving the pipeline. Deduped across all phases; bioinformatics tools the constructed workflow installs are out of scope (the discovery phase pins those). -- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache`. +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. Check: `galaxy-tool-cache --help | grep -q summarize`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -33,7 +33,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -41,11 +41,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `interview-to-galaxy-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: interview-to-galaxy`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: interview-to-freeform-summary, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -69,7 +82,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//` and any phases handled manually (marked MANUAL above). -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/_assembly.json b/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/_assembly.json index 84b6cf9..9ec8763 100644 --- a/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-interview-to-galaxy/_assembly.json @@ -1,12 +1,12 @@ { "source_pipeline": "interview-to-galaxy", - "source_revision": 2, + "source_revision": 5, "harness_name": "pipeline-interview-to-galaxy", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ - { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list"] }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list", "galaxy-tool-cache summarize"] }, + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "interview-to-freeform-summary", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-interview-to-workflow-brief/SKILL.md b/plugins/foundry-skills/skills/pipeline-interview-to-workflow-brief/SKILL.md new file mode 100644 index 0000000..5990f55 --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-interview-to-workflow-brief/SKILL.md @@ -0,0 +1,76 @@ +--- +name: pipeline-interview-to-workflow-brief +description: "Summarize an interview source and produce a scoped Workflow Brief for expert review — orchestrates the Foundry skills of the INTERVIEW → WORKFLOW BRIEF pipeline in order, in a per-run working directory." +--- + +# pipeline-interview-to-workflow-brief + +Harness for the **INTERVIEW → WORKFLOW BRIEF** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/interview-to-workflow-brief/index.md` (revision 3) — regenerate with `foundry-build assemble-pipeline interview-to-workflow-brief` if the pipeline changes; do not hand-edit. + +## When To Use + +- Summarize an interview source and produce a scoped Workflow Brief for expert review. + +## Bootstrap (install these CLIs first) + +Install the harness CLIs every constituent skill invokes before driving the pipeline. Deduped across all phases; bioinformatics tools the constructed workflow installs are out of scope (the discovery phase pins those). + +- **`foundry`** (foundry). `npm install -g @galaxy-foundry/gxwf-foundry`. + Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. + Check: `foundry --help`. + Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md + +## Run options + +Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. + +- `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. + +## Working directory (do this first) + +Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): + +1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `interview-to-workflow-brief-run`. +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). +3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). + +Announce the chosen directory before starting. + +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: interview-to-workflow-brief`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: interview-to-freeform-summary, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + +## Pipeline + +Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. + +1. **interview-to-freeform-summary** — invoke the `interview-to-freeform-summary` skill. Normalize a free-form user interview into the shared freeform-summary workflow handoff. +2. **freeform-summary-to-workflow-brief** — invoke the `freeform-summary-to-workflow-brief` skill. Turn a freeform source summary and selected scope into a Workflow Brief without Galaxy design assumptions. + +## Done + +Report the final artifacts in `.//`. + +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. + +## Notes + +- Do not re-implement any skill's internal logic here; this harness only sequences and routes. +- Carry unresolved assumptions forward as notes rather than inventing missing inputs. +- Brief production ends with workflow-brief.md for expert editing and review; it does not continue into Galaxy design or implementation. +- The caller supplies the desired objective and selected scope. Ambiguous selection becomes a Workflow blocker rather than an agent choice. +- Retain source evidence and run both structural validation and the static blocker check on the emitted brief. A valid blocked brief is an honest production result, not permission to implement. +- When reporting at Done, list the brief's open questions with their proposed answers, low-confidence proposals first. Tell the expert that leaving a proposal unedited accepts it and editing the answer overrides it. diff --git a/plugins/foundry-skills/skills/pipeline-interview-to-workflow-brief/_assembly.json b/plugins/foundry-skills/skills/pipeline-interview-to-workflow-brief/_assembly.json new file mode 100644 index 0000000..b06ea06 --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-interview-to-workflow-brief/_assembly.json @@ -0,0 +1,13 @@ +{ + "source_pipeline": "interview-to-workflow-brief", + "source_revision": 3, + "harness_name": "pipeline-interview-to-workflow-brief", + "options": ["use-subagents", "checkpoint", "feedback"], + "required_tools": [ + { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "implied", "implied_by": ["foundry validate-workflow-brief", "foundry check-workflow-brief"] } + ], + "phases": [ + { "phase": 1, "kind": "mold", "skill": "interview-to-freeform-summary", "cast_present": true, "loop": false }, + { "phase": 2, "kind": "mold", "skill": "freeform-summary-to-workflow-brief", "cast_present": true, "loop": false } + ] +} diff --git a/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/SKILL.md b/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/SKILL.md index e8612fc..c84c12f 100644 --- a/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/SKILL.md @@ -5,7 +5,7 @@ description: "Direct path from a Nextflow pipeline to a CWL Workflow + CommandLi # pipeline-nextflow-to-cwl -Harness for the **NEXTFLOW → CWL** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/nextflow-to-cwl/index.md` (revision 2) — regenerate with `foundry-build assemble-pipeline nextflow-to-cwl` if the pipeline changes; do not hand-edit. +Harness for the **NEXTFLOW → CWL** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/nextflow-to-cwl/index.md` (revision 3) — regenerate with `foundry-build assemble-pipeline nextflow-to-cwl` if the pipeline changes; do not hand-edit. ## When To Use @@ -19,12 +19,12 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. Check: `foundry --help`. Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -33,7 +33,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -41,11 +41,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `nextflow-to-cwl-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: nextflow-to-cwl`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-nextflow, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -67,7 +80,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//`. -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/_assembly.json b/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/_assembly.json index 7fdf187..9d55b4c 100644 --- a/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-nextflow-to-cwl/_assembly.json @@ -1,12 +1,12 @@ { "source_pipeline": "nextflow-to-cwl", - "source_revision": 2, + "source_revision": 3, "harness_name": "pipeline-nextflow-to-cwl", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "referenced" }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf validate-tests"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf validate-tests"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-nextflow", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/SKILL.md b/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/SKILL.md index 287d8d6..896fd5e 100644 --- a/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/SKILL.md @@ -5,7 +5,7 @@ description: "Direct path from a Nextflow pipeline to a Galaxy gxformat2 workflo # pipeline-nextflow-to-galaxy -Harness for the **NEXTFLOW → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/nextflow-to-galaxy/index.md` (revision 4) — regenerate with `foundry-build assemble-pipeline nextflow-to-galaxy` if the pipeline changes; do not hand-edit. +Harness for the **NEXTFLOW → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/nextflow-to-galaxy/index.md` (revision 5) — regenerate with `foundry-build assemble-pipeline nextflow-to-galaxy` if the pipeline changes; do not hand-edit. ## When To Use @@ -19,16 +19,16 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. Check: `foundry --help`. Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md -- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache`. +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. Check: `galaxy-tool-cache --help | grep -q summarize`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -37,7 +37,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -45,11 +45,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `nextflow-to-galaxy-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: nextflow-to-galaxy`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-nextflow, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -75,7 +88,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//` and any phases handled manually (marked MANUAL above). -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/_assembly.json b/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/_assembly.json index 666148e..c9b4f9e 100644 --- a/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-nextflow-to-galaxy/_assembly.json @@ -1,13 +1,13 @@ { "source_pipeline": "nextflow-to-galaxy", - "source_revision": 4, + "source_revision": 5, "harness_name": "pipeline-nextflow-to-galaxy", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "referenced" }, - { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list"] }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list", "galaxy-tool-cache summarize"] }, + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-nextflow", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-nextflow-to-workflow-brief/SKILL.md b/plugins/foundry-skills/skills/pipeline-nextflow-to-workflow-brief/SKILL.md new file mode 100644 index 0000000..6910b22 --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-nextflow-to-workflow-brief/SKILL.md @@ -0,0 +1,76 @@ +--- +name: pipeline-nextflow-to-workflow-brief +description: "Summarize a Nextflow pipeline and produce a scoped Workflow Brief for expert review — orchestrates the Foundry skills of the NEXTFLOW → WORKFLOW BRIEF pipeline in order, in a per-run working directory." +--- + +# pipeline-nextflow-to-workflow-brief + +Harness for the **NEXTFLOW → WORKFLOW BRIEF** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/nextflow-to-workflow-brief/index.md` (revision 3) — regenerate with `foundry-build assemble-pipeline nextflow-to-workflow-brief` if the pipeline changes; do not hand-edit. + +## When To Use + +- Summarize a Nextflow pipeline and produce a scoped Workflow Brief for expert review. + +## Bootstrap (install these CLIs first) + +Install the harness CLIs every constituent skill invokes before driving the pipeline. Deduped across all phases; bioinformatics tools the constructed workflow installs are out of scope (the discovery phase pins those). + +- **`foundry`** (foundry). `npm install -g @galaxy-foundry/gxwf-foundry`. + Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. + Check: `foundry --help`. + Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md + +## Run options + +Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. + +- `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. + +## Working directory (do this first) + +Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): + +1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `nextflow-to-workflow-brief-run`. +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). +3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). + +Announce the chosen directory before starting. + +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: nextflow-to-workflow-brief`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-nextflow, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + +## Pipeline + +Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. + +1. **summarize-nextflow** — invoke the `summarize-nextflow` skill. Read a Nextflow pipeline source tree (nf-core or ad-hoc DSL2) and emit a structured JSON summary for downstream translation Molds. +2. **nextflow-summary-to-workflow-brief** — invoke the `nextflow-summary-to-workflow-brief` skill. Turn a nextflow source summary and selected scope into a Workflow Brief without Galaxy design assumptions. + +## Done + +Report the final artifacts in `.//`. + +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. + +## Notes + +- Do not re-implement any skill's internal logic here; this harness only sequences and routes. +- Carry unresolved assumptions forward as notes rather than inventing missing inputs. +- Brief production ends with workflow-brief.md for expert editing and review; it does not continue into Galaxy design or implementation. +- The caller supplies the desired objective and selected scope. Ambiguous selection becomes a Workflow blocker rather than an agent choice. +- Retain source evidence and run both structural validation and the static blocker check on the emitted brief. A valid blocked brief is an honest production result, not permission to implement. +- When reporting at Done, list the brief's open questions with their proposed answers, low-confidence proposals first. Tell the expert that leaving a proposal unedited accepts it and editing the answer overrides it. diff --git a/plugins/foundry-skills/skills/pipeline-nextflow-to-workflow-brief/_assembly.json b/plugins/foundry-skills/skills/pipeline-nextflow-to-workflow-brief/_assembly.json new file mode 100644 index 0000000..b191e96 --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-nextflow-to-workflow-brief/_assembly.json @@ -0,0 +1,13 @@ +{ + "source_pipeline": "nextflow-to-workflow-brief", + "source_revision": 3, + "harness_name": "pipeline-nextflow-to-workflow-brief", + "options": ["use-subagents", "checkpoint", "feedback"], + "required_tools": [ + { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "referenced", "implied_by": ["foundry validate-workflow-brief", "foundry check-workflow-brief"] } + ], + "phases": [ + { "phase": 1, "kind": "mold", "skill": "summarize-nextflow", "cast_present": true, "loop": false }, + { "phase": 2, "kind": "mold", "skill": "nextflow-summary-to-workflow-brief", "cast_present": true, "loop": false } + ] +} diff --git a/plugins/foundry-skills/skills/pipeline-paper-to-cwl/SKILL.md b/plugins/foundry-skills/skills/pipeline-paper-to-cwl/SKILL.md index 7a1a801..8180603 100644 --- a/plugins/foundry-skills/skills/pipeline-paper-to-cwl/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-paper-to-cwl/SKILL.md @@ -5,7 +5,7 @@ description: "Direct path from a paper to a CWL Workflow + CommandLineTool set # pipeline-paper-to-cwl -Harness for the **PAPER → CWL** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/paper-to-cwl/index.md` (revision 2) — regenerate with `foundry-build assemble-pipeline paper-to-cwl` if the pipeline changes; do not hand-edit. +Harness for the **PAPER → CWL** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/paper-to-cwl/index.md` (revision 3) — regenerate with `foundry-build assemble-pipeline paper-to-cwl` if the pipeline changes; do not hand-edit. ## When To Use @@ -15,12 +15,12 @@ Harness for the **PAPER → CWL** Foundry pipeline. Runs the constituent skills Install the harness CLIs every constituent skill invokes before driving the pipeline. Deduped across all phases; bioinformatics tools the constructed workflow installs are out of scope (the discovery phase pins those). -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -29,7 +29,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -37,11 +37,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `paper-to-cwl-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: paper-to-cwl`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-paper, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -53,7 +66,7 @@ Run these phases in order. After each, confirm the expected artifact exists in t 5. **implement-cwl-tool-step** (loop) — invoke the `implement-cwl-tool-step` skill, once per step. No shared endstate oracle yet; iterate over the tools enumerated in the CWL template, doing each by hand. 6. **validate-cwl** (loop) — invoke the `validate-cwl` skill, once per step. No shared endstate oracle yet; iterate over the tools enumerated in the CWL template, doing each by hand. 7. **test-data-resolution** (branch) — resolve in order; stop at the first that yields acceptable output: - - Try `paper-to-test-data` — Derive workflow test inputs and expected outputs from a paper. + - Try `paper-to-test-data` — Resolve the test data a paper names into refs for each workflow input. - Otherwise try `find-test-data` — Search IWC fixtures and public sources for test data matching a data-flow shape. - **user-supplied** — if nothing above yields acceptable output, ask the user to supply it directly. 8. **implement-cwl-workflow-test** — invoke the `implement-cwl-workflow-test` skill. Assemble CWL job file(s) and expected-output assertions. @@ -65,7 +78,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//` and any phases handled manually (marked MANUAL above). -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-paper-to-cwl/_assembly.json b/plugins/foundry-skills/skills/pipeline-paper-to-cwl/_assembly.json index 6da3c75..92b9a47 100644 --- a/plugins/foundry-skills/skills/pipeline-paper-to-cwl/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-paper-to-cwl/_assembly.json @@ -1,11 +1,11 @@ { "source_pipeline": "paper-to-cwl", - "source_revision": 2, + "source_revision": 3, "harness_name": "pipeline-paper-to-cwl", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf validate-tests"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf validate-tests"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-paper", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/SKILL.md b/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/SKILL.md index 029caea..41c72f7 100644 --- a/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/SKILL.md @@ -5,26 +5,26 @@ description: "Direct path from a paper to a Galaxy gxformat2 workflow — orches # pipeline-paper-to-galaxy -Harness for the **PAPER → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/paper-to-galaxy/index.md` (revision 2) — regenerate with `foundry-build assemble-pipeline paper-to-galaxy` if the pipeline changes; do not hand-edit. +Harness for the **PAPER → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/paper-to-galaxy/index.md` (revision 6) — regenerate with `foundry-build assemble-pipeline paper-to-galaxy` if the pipeline changes; do not hand-edit. ## When To Use -- Direct path from a paper to a Galaxy gxformat2 workflow. No CWL intermediate. +- Direct path from a paper to a Galaxy gxformat2 workflow. No Workflow Brief or CWL intermediate. ## Bootstrap (install these CLIs first) Install the harness CLIs every constituent skill invokes before driving the pipeline. Deduped across all phases; bioinformatics tools the constructed workflow installs are out of scope (the discovery phase pins those). -- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache`. +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. Check: `galaxy-tool-cache --help | grep -q summarize`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -33,7 +33,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -41,11 +41,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `paper-to-galaxy-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: paper-to-galaxy`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-paper, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -57,7 +70,7 @@ Run these phases in order. After each, confirm the expected artifact exists in t 5. **freeform-summary-to-galaxy-template** — invoke the `freeform-summary-to-galaxy-template` skill. gxformat2 skeleton with per-step TODOs from a free-form summary and Galaxy design brief. 6. **advance-galaxy-draft-step** (loop) — invoke the `advance-galaxy-draft-step` skill, once per step. It owns its own endstate oracle (`gxwf draft-next-step`) and concretizes one drafty step per call; re-invoke until it reports `draft: false`, then it extracts the concrete `galaxy-workflow.gxwf.yml` (via `gxwf draft-extract`) and continues. 7. **test-data-resolution** (branch) — resolve in order; stop at the first that yields acceptable output: - - Try `paper-to-test-data` — Derive workflow test inputs and expected outputs from a paper. + - Try `paper-to-test-data` — Resolve the test data a paper names into refs for each workflow input. - Otherwise try `find-test-data` — Search IWC fixtures and public sources for test data matching a data-flow shape. - **user-supplied** — if nothing above yields acceptable output, ask the user to supply it directly. 8. **freeform-summary-to-galaxy-test-plan** — invoke the `freeform-summary-to-galaxy-test-plan` skill. Synthesize a Galaxy workflow test plan from a free-form summary and the Galaxy design briefs. @@ -70,7 +83,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//` and any phases handled manually (marked MANUAL above). -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/_assembly.json b/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/_assembly.json index 0413fba..f51531a 100644 --- a/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-paper-to-galaxy/_assembly.json @@ -1,12 +1,12 @@ { "source_pipeline": "paper-to-galaxy", - "source_revision": 2, + "source_revision": 6, "harness_name": "pipeline-paper-to-galaxy", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ - { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list"] }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list", "galaxy-tool-cache summarize"] }, + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-paper", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-paper-to-workflow-brief/SKILL.md b/plugins/foundry-skills/skills/pipeline-paper-to-workflow-brief/SKILL.md new file mode 100644 index 0000000..1f2f90d --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-paper-to-workflow-brief/SKILL.md @@ -0,0 +1,76 @@ +--- +name: pipeline-paper-to-workflow-brief +description: "Summarize a paper source and produce a scoped Workflow Brief for expert review — orchestrates the Foundry skills of the PAPER → WORKFLOW BRIEF pipeline in order, in a per-run working directory." +--- + +# pipeline-paper-to-workflow-brief + +Harness for the **PAPER → WORKFLOW BRIEF** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/paper-to-workflow-brief/index.md` (revision 3) — regenerate with `foundry-build assemble-pipeline paper-to-workflow-brief` if the pipeline changes; do not hand-edit. + +## When To Use + +- Summarize a paper source and produce a scoped Workflow Brief for expert review. + +## Bootstrap (install these CLIs first) + +Install the harness CLIs every constituent skill invokes before driving the pipeline. Deduped across all phases; bioinformatics tools the constructed workflow installs are out of scope (the discovery phase pins those). + +- **`foundry`** (foundry). `npm install -g @galaxy-foundry/gxwf-foundry`. + Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. + Check: `foundry --help`. + Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md + +## Run options + +Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. + +- `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. + +## Working directory (do this first) + +Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): + +1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `paper-to-workflow-brief-run`. +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). +3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). + +Announce the chosen directory before starting. + +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: paper-to-workflow-brief`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-paper, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + +## Pipeline + +Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. + +1. **summarize-paper** — invoke the `summarize-paper` skill. Extract methods, tools, sample data, and references from a paper. +2. **freeform-summary-to-workflow-brief** — invoke the `freeform-summary-to-workflow-brief` skill. Turn a freeform source summary and selected scope into a Workflow Brief without Galaxy design assumptions. + +## Done + +Report the final artifacts in `.//`. + +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. + +## Notes + +- Do not re-implement any skill's internal logic here; this harness only sequences and routes. +- Carry unresolved assumptions forward as notes rather than inventing missing inputs. +- Brief production ends with workflow-brief.md for expert editing and review; it does not continue into Galaxy design or implementation. +- The caller supplies the desired objective and selected scope. Ambiguous selection becomes a Workflow blocker rather than an agent choice. +- Retain source evidence and run both structural validation and the static blocker check on the emitted brief. A valid blocked brief is an honest production result, not permission to implement. +- When reporting at Done, list the brief's open questions with their proposed answers, low-confidence proposals first. Tell the expert that leaving a proposal unedited accepts it and editing the answer overrides it. diff --git a/plugins/foundry-skills/skills/pipeline-paper-to-workflow-brief/_assembly.json b/plugins/foundry-skills/skills/pipeline-paper-to-workflow-brief/_assembly.json new file mode 100644 index 0000000..4e7770c --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-paper-to-workflow-brief/_assembly.json @@ -0,0 +1,13 @@ +{ + "source_pipeline": "paper-to-workflow-brief", + "source_revision": 3, + "harness_name": "pipeline-paper-to-workflow-brief", + "options": ["use-subagents", "checkpoint", "feedback"], + "required_tools": [ + { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "implied", "implied_by": ["foundry validate-workflow-brief", "foundry check-workflow-brief"] } + ], + "phases": [ + { "phase": 1, "kind": "mold", "skill": "summarize-paper", "cast_present": true, "loop": false }, + { "phase": 2, "kind": "mold", "skill": "freeform-summary-to-workflow-brief", "cast_present": true, "loop": false } + ] +} diff --git a/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/SKILL.md b/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/SKILL.md index 1258496..55f6b7c 100644 --- a/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/SKILL.md +++ b/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/SKILL.md @@ -5,7 +5,7 @@ description: "Interview-driven, edit-in-place modification of an existing Galaxy # pipeline-update-interview-to-galaxy -Harness for the **UPDATE-INTERVIEW → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/update-interview-to-galaxy/index.md` (revision 2) — regenerate with `foundry-build assemble-pipeline update-interview-to-galaxy` if the pipeline changes; do not hand-edit. +Harness for the **UPDATE-INTERVIEW → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/update-interview-to-galaxy/index.md` (revision 3) — regenerate with `foundry-build assemble-pipeline update-interview-to-galaxy` if the pipeline changes; do not hand-edit. ## When To Use @@ -19,16 +19,16 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Ephemeral run: `npx --package @galaxy-foundry/gxwf-foundry foundry`. Check: `foundry --help`. Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md -- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache`. +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. Check: `galaxy-tool-cache --help | grep -q summarize`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ @@ -37,7 +37,7 @@ Install the harness CLIs every constituent skill invokes before driving the pipe Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. - `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. -- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. - `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. ## Working directory (do this first) @@ -45,11 +45,24 @@ Optional flags, given as leading arguments. Strip any you recognize; treat the r Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): 1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `update-interview-to-galaxy-run`. -2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now (see Run options). +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). 3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). Announce the chosen directory before starting. +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: update-interview-to-galaxy`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: summarize-galaxy-workflow, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + ## Pipeline Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. @@ -68,7 +81,9 @@ Run these phases in order. After each, confirm the expected artifact exists in t Report the final artifacts in `.//`. -If the run was invoked with `--feedback`, close the ledger first: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. ## Notes diff --git a/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/_assembly.json b/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/_assembly.json index fbf9dbe..d5571a4 100644 --- a/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/_assembly.json +++ b/plugins/foundry-skills/skills/pipeline-update-interview-to-galaxy/_assembly.json @@ -1,13 +1,13 @@ { "source_pipeline": "update-interview-to-galaxy", - "source_revision": 2, + "source_revision": 3, "harness_name": "pipeline-update-interview-to-galaxy", "options": ["use-subagents", "checkpoint", "feedback"], "required_tools": [ { "tool": "foundry", "origin": "npm", "package": "@galaxy-foundry/gxwf-foundry", "invoke": "foundry", "invoke_fallback": "npx --package @galaxy-foundry/gxwf-foundry foundry", "availability_check": "foundry --help", "docs_url": "https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md", "source": "referenced" }, - { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list"] }, - { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.8.1", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf validate", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests"] }, - { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.47", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.47 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list", "galaxy-tool-cache summarize"] }, + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf validate", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } ], "phases": [ { "phase": 1, "kind": "mold", "skill": "summarize-galaxy-workflow", "cast_present": true, "loop": false }, diff --git a/plugins/foundry-skills/skills/pipeline-workflow-brief-to-galaxy/SKILL.md b/plugins/foundry-skills/skills/pipeline-workflow-brief-to-galaxy/SKILL.md new file mode 100644 index 0000000..e61ef90 --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-workflow-brief-to-galaxy/SKILL.md @@ -0,0 +1,99 @@ +--- +name: pipeline-workflow-brief-to-galaxy +description: "Consume an expert-reviewed, unchanged Workflow Brief directly through the Galaxy design and implementation chain — orchestrates the Foundry skills of the WORKFLOW BRIEF → GALAXY pipeline in order, in a per-run working directory." +--- + +# pipeline-workflow-brief-to-galaxy + +Harness for the **WORKFLOW BRIEF → GALAXY** Foundry pipeline. Runs the constituent skills in order inside a single per-run working directory. Assembled from `content/pipelines/workflow-brief-to-galaxy/index.md` (revision 6) — regenerate with `foundry-build assemble-pipeline workflow-brief-to-galaxy` if the pipeline changes; do not hand-edit. + +## When To Use + +- Consume an expert-reviewed, unchanged Workflow Brief directly through the Galaxy design and implementation chain. + +## Bootstrap (install these CLIs first) + +Install the harness CLIs every constituent skill invokes before driving the pipeline. Deduped across all phases; bioinformatics tools the constructed workflow installs are out of scope (the discovery phase pins those). + +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. + Check: `galaxy-tool-cache --help | grep -q summarize`. + Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. + Check: `gxwf --help | grep -q draft-validate`. + Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. + Check: `planemo --version`. + Docs: https://planemo.readthedocs.io/ + +## Run options + +Optional flags, given as leading arguments. Strip any you recognize; treat the remaining positional argument as the run slug. All default off and compose. + +- `--use-subagents` — run each cast phase in its own subagent to keep this orchestrator's context small. For each phase whose skill is cast, spawn a subagent, tell it the run directory and to invoke the named skill with every default filename prefixed by `.//`, and have it return a short report (artifacts written, assumptions, status) rather than its full transcript; carry only that report forward. A cast loop phase runs **one subagent per iteration** — each advances a single step and returns its done-signal, and you inspect that signal to decide whether to spawn the next iteration. Branch phases run their whole fallback chain in one subagent. MANUAL (un-cast) phases are never delegated — including MANUAL loop phases — so handle those yourself regardless of the per-iteration rule above. +- `--checkpoint` — commit after every phase so the run directory's git history is a per-step record (a data source for workflow-implementation visualizations). When set, `git init .//` during working-directory setup — this is a standalone per-run repo; do not add it to any surrounding repo you are working inside. Write `.//.gitignore` in the same step holding `dashboard.html`, `run-manifest.json`, and `.omc/`, so a later reading tool cannot land in the history it reads. Then after each phase's artifact is confirmed run `git -C .// add -A && git -C .// commit -m "phase : "`. Loop phases commit **once per iteration** (`phase step : `); for a MANUAL loop, commit once per by-hand step. Keep the subject grammar exactly — `phase : ` and `phase step : `, with any note in parentheses after it — because it is parsed. With `--use-subagents`, the subagent does the work and returns; you make the commit. +- `--feedback` — create `.//foundry-feedback.ledger.yml` before phase 1. Set `run.pipeline` to this pipeline's slug and `run.run_slug` to the chosen run slug. Copy the complete top-level roster from `_assembly.json` with `run.status: running` and every phase `pending`; set a phase `running` immediately before it starts and `done` after success. Keep one row per loop and increment `iterations`; record a branch's chosen path as `selected`. Pass the same ledger path to every skill and subagent. Before marking a phase `done`, require its feedback outcome — appended entry ids or an explicit `no feedback` — and record `feedback_checked: true` on that row; under `--use-subagents` the subagent's short report carries that line. Do the same pass yourself for a MANUAL phase, naming the pipeline source as the subject when the harness is what needs to change. On a terminating error mark the phase and run `failed`; on a graceful stop mark the run `cancelled`; after every intended phase is done mark the run `complete`. Never describe an empty incomplete ledger as a clean run. + +## Working directory (do this first) + +Every constituent skill writes fixed filenames to its working directory. To keep one run's artifacts namespaced and avoid clobbering a prior run (foundry#282): + +1. Pick a run slug — use the harness argument if given; else ask the user for a short project name up front (the directory must exist before phase 1 writes its first artifact, so don't wait for a source title); else default `workflow-brief-to-galaxy-run`. +2. Create `.//` in the current directory. If it exists, suffix `-2`, `-3`, … . If invoked with `--checkpoint`, run `git init .//` now and write its `.gitignore` (see Run options). +3. Run **every** skill invocation below with `.//` as its working directory: **prefix every default input and output filename with `.//`** when you invoke the skill. The skills preserve their declared basenames and honor a harness-supplied directory; you supply the prefix on **both reads and writes**, so each phase finds the prior phase's output and nothing lands in the repo root. +4. Create `.//foundry-run.yml` now, before phase 1 writes anything (see Run record below). + +Announce the chosen directory before starting. + +## Run record (keep this current) + +Maintain `.//foundry-run.yml` for the whole run. It is not optional and has no flag: it is how anything reading this run afterwards knows what it was. + +- **Create it during working-directory setup**, before phase 1. Set `run_record_version: 1`, `pipeline: workflow-brief-to-galaxy`, `run_slug` to the chosen slug, `harness_name` and `source_revision` from `_assembly.json`, `assembly_sha256` to the sha256 hex digest of that file, `foundry_head` to the Foundry commit you are reading (`null` when the Foundry is not a git checkout), `started_at` to now, `finished_at: null`, and `status: running`. Quote both shas: an unquoted all-digit sha reads back as a number and is discarded. +- **Copy the phase roster** into `phases`: one row per entry of `_assembly.json`'s top-level `phases`, keyed `n:` (that entry's `phase` number), with `kind`, then `skill` for a mold or `pattern` for a branch, `status: pending`, and `artifacts: []`. A loop row later gains `iterations`, a branch row `selected`. Phase 1 reads `{ n: 1, kind: mold, skill: workflow-brief-to-galaxy-interface, status: pending, artifacts: [] }`. +- **Set a phase `running`** immediately before invoking it, and `done` once you have confirmed its artifacts exist. +- **Append one `artifacts` row per file a phase writes**: `{ id, file, at }` using the artifact id and basename the skill declares. A loop phase stays one row, increments `iterations`, and appends a row for each iteration that writes, carrying `iteration: `. +- **Record a branch's outcome** as `selected: ` on that phase's row. +- **Close it in `## Done`**: set `finished_at` and the final `status` — `complete` when every intended phase is done, `failed` when a phase terminated the run, `cancelled` on a graceful stop. Leave it `running` on a hard interruption rather than inventing a recovery write. +- Under `--use-subagents` the subagent does the work and reports; **you** write the record. Do the same pass yourself for a MANUAL phase. + +## Pipeline + +Run these phases in order. After each, confirm the expected artifact exists in the run directory before advancing. + +1. **workflow-brief-to-galaxy-interface** — invoke the `workflow-brief-to-galaxy-interface` skill. Map a reviewed Workflow Brief into a Galaxy workflow interface design brief, consulting retained source evidence when available. +2. **workflow-brief-to-galaxy-data-flow** — invoke the `workflow-brief-to-galaxy-data-flow` skill. Translate a reviewed Workflow Brief and its Galaxy interface into a Galaxy data-flow design brief. +3. **compare-against-iwc-exemplar** — invoke the `compare-against-iwc-exemplar` skill. Find nearest IWC exemplar(s) and surface a structural diff against the upstream Galaxy design briefs to guide template authoring. +4. **workflow-brief-to-galaxy-template** — invoke the `workflow-brief-to-galaxy-template` skill. Build a gxformat2 skeleton from a reviewed Workflow Brief and its Galaxy design briefs, consulting retained source evidence. +5. **advance-galaxy-draft-step** (loop) — invoke the `advance-galaxy-draft-step` skill, once per step. It owns its own endstate oracle (`gxwf draft-next-step`) and concretizes one drafty step per call; re-invoke until it reports `draft: false`, then it extracts the concrete `galaxy-workflow.gxwf.yml` (via `gxwf draft-extract`) and continues. +6. **test-data-resolution** (branch) — resolve in order; stop at the first that yields acceptable output: + - Try `find-test-data` — Search IWC fixtures and public sources for test data matching a data-flow shape. + - **user-supplied** — if nothing above yields acceptable output, ask the user to supply it directly. +7. **workflow-brief-to-galaxy-test-plan** — invoke the `workflow-brief-to-galaxy-test-plan` skill. Synthesize a Galaxy workflow test plan from a reviewed Workflow Brief, its Galaxy design briefs, and available source evidence. +8. **implement-galaxy-workflow-test** — invoke the `implement-galaxy-workflow-test` skill. Assemble Galaxy workflow test fixtures and assertions. +9. **validate-galaxy-workflow** — invoke the `validate-galaxy-workflow` skill. Run terminal gxwf validation on an assembled Galaxy workflow and classify workflow-level failures. +10. **run-workflow-test** — invoke the `run-workflow-test` skill. Execute a workflow's tests via Planemo; emit structured pass/fail and outputs. +11. **debug-galaxy-workflow-output** — invoke the `debug-galaxy-workflow-output` skill. Triage failing Galaxy run outputs; classify the failure surface and capture evidence before recommending repairs. + +## Done + +Report the final artifacts in `.//` and any phases handled manually (marked MANUAL above). + +Close `.//foundry-run.yml` before anything else: set `finished_at` and the final `status`, and make sure every phase row carries the artifacts it wrote. A run whose record still says `running` is one nothing downstream can read honestly. + +If the run was invoked with `--feedback`, close the ledger next, after the run record: set the final `run.status`, then tell the user the ledger path `.//foundry-feedback.ledger.yml` and how many entries it holds. If it holds any, offer to run the `report-foundry-run-feedback` skill on it, which triages them into local drafts and files nothing without explicit confirmation. Report an empty ledger as "no feedback recorded", never as evidence the Foundry worked well. + +## Notes + +- Do not re-implement any skill's internal logic here; this harness only sequences and routes. +- Carry unresolved assumptions forward as notes rather than inventing missing inputs. +- The caller supplies workflow-brief.md; a hand-authored brief is a valid entry and requires no source-to-brief producer run. Keep referenced source evidence available, preserving originals at separate paths. +- Before phase 1, record expert review of the supplied brief and perform current environment preflight for required tooling, versions, workspace access, and usable container engines. Missing review or failed preflight stops the run. +- Before phase 1 and whenever resuming, run foundry check-workflow-brief workflow-brief.md --json. Exit 1, 3, or 4 stops before design; exit 0 establishes only structure and no declared blockers. +- Treat workflow-brief.md as unchanged input through every design, draft, implementation, test, and debug phase. Record its initial hash and check it after each phase and loop iteration and on resumption; a mismatch stops the run. +- Record durable workflow knowledge in open-requirements.ledger.yml: evidence as discoveries, authorized choices as decisions, unmet needs as obligations, and changes to expert intent as proposed brief-change recommendations. Harness progress remains harness state. Do not apply a recommendation to the brief; return a required change to the separate expert editing step. +- The selected brief governs scientific scope. Design Molds consume it directly and may consult retained freeform or Nextflow summaries only as source evidence; linked evidence does not broaden scope. +- Phase 6 resolves test data with find-test-data then caller-supplied fixtures. Phase 7 develops the test plan; the brief itself never contains final fixture/assertion declarations. +- When reporting at Done, list the Workflow Brief open questions recorded in open-requirements.ledger.yml with the current answer the run used and its basis, low-confidence answers first, followed by every proposed brief-change recommendation. These are the assumptions the expert should check. Changing one means editing the brief and rerunning. diff --git a/plugins/foundry-skills/skills/pipeline-workflow-brief-to-galaxy/_assembly.json b/plugins/foundry-skills/skills/pipeline-workflow-brief-to-galaxy/_assembly.json new file mode 100644 index 0000000..d242c54 --- /dev/null +++ b/plugins/foundry-skills/skills/pipeline-workflow-brief-to-galaxy/_assembly.json @@ -0,0 +1,24 @@ +{ + "source_pipeline": "workflow-brief-to-galaxy", + "source_revision": 6, + "harness_name": "pipeline-workflow-brief-to-galaxy", + "options": ["use-subagents", "checkpoint", "feedback"], + "required_tools": [ + { "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["galaxy-tool-cache add", "galaxy-tool-cache list", "galaxy-tool-cache summarize"] }, + { "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", "package_version": "^1.15.0", "invoke": "gxwf", "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", "implied_by": ["gxwf convert", "gxwf draft-validate", "gxwf draft-next-step", "gxwf draft-extract", "gxwf validate-tests", "gxwf validate"] }, + { "tool": "planemo", "origin": "pypi", "package": "planemo", "package_version": "0.75.48", "invoke": "planemo", "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "source": "referenced" } + ], + "phases": [ + { "phase": 1, "kind": "mold", "skill": "workflow-brief-to-galaxy-interface", "cast_present": true, "loop": false }, + { "phase": 2, "kind": "mold", "skill": "workflow-brief-to-galaxy-data-flow", "cast_present": true, "loop": false }, + { "phase": 3, "kind": "mold", "skill": "compare-against-iwc-exemplar", "cast_present": true, "loop": false }, + { "phase": 4, "kind": "mold", "skill": "workflow-brief-to-galaxy-template", "cast_present": true, "loop": false }, + { "phase": 5, "kind": "mold", "skill": "advance-galaxy-draft-step", "cast_present": true, "loop": true }, + { "phase": 6, "kind": "branch", "pattern": "test-data-resolution", "chain": ["find-test-data", "user-supplied"], "cast_present": [true, null] }, + { "phase": 7, "kind": "mold", "skill": "workflow-brief-to-galaxy-test-plan", "cast_present": true, "loop": false }, + { "phase": 8, "kind": "mold", "skill": "implement-galaxy-workflow-test", "cast_present": true, "loop": false }, + { "phase": 9, "kind": "mold", "skill": "validate-galaxy-workflow", "cast_present": true, "loop": false }, + { "phase": 10, "kind": "mold", "skill": "run-workflow-test", "cast_present": true, "loop": false }, + { "phase": 11, "kind": "mold", "skill": "debug-galaxy-workflow-output", "cast_present": true, "loop": false } + ] +} diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/SKILL.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/SKILL.md index 7df57f8..88f86c0 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/SKILL.md +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/SKILL.md @@ -1,6 +1,6 @@ --- name: repair-galaxy-draft-topology -description: "Re-wire a Galaxy draft region when a step's declared output can't be computed from its wired inputs." +description: "Make the smallest structural edit that closes an open blocking entry in a settled Galaxy draft." --- # repair-galaxy-draft-topology @@ -9,35 +9,42 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## When To Use -- Re-wire a Galaxy draft region when a step's declared output can't be computed from its wired inputs. +- Make the smallest structural edit that closes an open blocking entry in a settled Galaxy draft. ## Inputs -- Read artifact `galaxy-workflow-draft`. Schema: galaxy-workflow-draft. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Partially-realized gxformat2 draft (see galaxy-workflow-draft-format) whose topology can't support a declared step — the region to repair. -- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. +- Read artifact `galaxy-workflow-draft`. Schema: galaxy-workflow-draft. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-template`. Partially-realized gxformat2 draft (see galaxy-workflow-draft-format) with an open blocking entry. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. ## Outputs -- Write artifact `galaxy-workflow-draft` as `galaxy-workflow-draft.gxwf.yml`. Format: `yaml`. Schema: galaxy-workflow-draft. Re-wired gxformat2 draft: a producer step (or small sub-path) inserted so the previously-blocked step's declared output is computable; new steps land at draft (TODO) tier. +- Write artifact `galaxy-workflow-draft` as `galaxy-workflow-draft.gxwf.yml`. Format: `yaml`. Schema: galaxy-workflow-draft. Edited gxformat2 draft. Each blocking entry addressed is closed by the named edits, and new or replaced steps land at draft tier. - Write artifact `open-requirements-ledger` as `open-requirements.ledger.yml`. Format: `yaml`. Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact. ## Required Tools -- None declared. Procedure should not assume external CLIs are present. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. + Check: `gxwf --help | grep -q draft-validate`. + Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli ## Load Upfront -- `references/notes/galaxy-data-flow-draft-contract.md`: Research note copied verbatim into the bundle. Re-wire the affected region consistently with the data-flow design that settled the surrounding topology. -- `references/notes/galaxy-workflow-draft-format.md`: Research note copied verbatim into the bundle. Emit re-wired steps in the draft superset (TODO tool_id, _plan_* fields) so existing per-step machinery realizes them; respect the settle-vs-repair boundary. -- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Scope the repair from the open blocking entries — each names the step, the uncomputable output, and the missing evidence — then mark resolved each one repaired, and leave any it cannot close open with a surrender note for the terminal path. -- `references/schemas/galaxy-workflow-draft.schema.json`: Schema file copied verbatim into the bundle. In/out contract: the draft this Mold reads and re-wires conforms to galaxy-workflow-draft; cast bundles the JSON Schema so the re-wired draft stays inside the superset the per-step loop expects. +- `references/cli/draft-validate.json`: CLI command reference packaged as a sidecar. Check the draft contract, including dangling edges and sources, on the edited draft. Use when: after every edit. +- `references/notes/galaxy-data-flow-draft-contract.md`: Research note copied verbatim into the bundle. Edit the affected region consistently with the data-flow design that settled the surrounding topology. +- `references/notes/galaxy-workflow-draft-format.md`: Research note copied verbatim into the bundle. Emit added or replaced steps in the draft superset (TODO tool_id, _plan_* fields) so existing per-step machinery realizes them; respect the settle-vs-repair boundary. +- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Scope the repair from the open blocking entries — each names the step and what's missing — then mark resolved each one repaired, and leave any it cannot close open with a note. +- `references/schemas/galaxy-workflow-draft.schema.json`: Schema file copied verbatim into the bundle. In/out contract: the draft this Mold reads and edits conforms to galaxy-workflow-draft; cast bundles the JSON Schema so the edited draft stays inside the superset the per-step loop expects. ## Load On Demand -- `references/patterns/galaxy-collection-patterns.md`: Pattern note copied verbatim into the bundle. Choose a corpus-attested collection recipe when the missing producer is a collection construction, reshape, or bridge. Use when: the repair sub-path needs collection cleanup, reshaping, relabeling, identifier synchronization, or a collection-tabular bridge. -- `references/patterns/galaxy-tabular-patterns.md`: Pattern note copied verbatim into the bundle. Choose a corpus-attested tabular recipe when the missing evidence is a column, key, or aggregate a tabular step can produce. Use when: the repair sub-path needs a computed column, join key, filter criterion, or aggregate the blocked step depends on. -- `references/notes/galaxy-collection-semantics.md`: Research note copied verbatim into the bundle. Preserve collection typing and map-over/reduction semantics when an inserted producer step joins or reshapes collection inputs. Use when: the repair inserts or rewires steps touching collection inputs, outputs, or mapped/reduced connections. +- `references/patterns/galaxy-collection-patterns.md`: Pattern note copied verbatim into the bundle. Choose a corpus-attested collection recipe when the missing producer is a collection construction, reshape, or bridge. Use when: an added step needs collection cleanup, reshaping, relabeling, identifier synchronization, or a collection-tabular bridge. +- `references/patterns/galaxy-conditionals-patterns.md`: Pattern note copied verbatim into the bundle. Choose how to merge when a `when`-gated step's output feeds another step. Use when: an inserted or rewired step consumes a `when`-gated step's output. +- `references/patterns/galaxy-tabular-patterns.md`: Pattern note copied verbatim into the bundle. Choose a corpus-attested tabular recipe when the missing evidence is a column, key, or aggregate a tabular step can produce. Use when: an added step needs a computed column, join key, filter criterion, or aggregate the blocked step depends on. +- `references/notes/galaxy-collection-semantics.md`: Research note copied verbatim into the bundle. Preserve collection typing and map-over/reduction semantics when an edit joins or reshapes collection inputs, and quote the rule when a removal rests on it. Use when: the repair inserts or rewires steps touching collection inputs, outputs, or mapped/reduced connections, or a removal rests on collection semantics. - `references/notes/galaxy-collection-semantics.yml`: Companion file copied verbatim into the bundle. Sibling of `references/notes/galaxy-collection-semantics.md`; read it where that note directs. +- `references/notes/gxformat2-schema.md`: Research note copied verbatim into the bundle. Keep post-job actions and nested parameter keys correct when an edit moves them. Use when: an edit touches `out:` actions (`hide`, `rename`, `change_datatype`) or wires a nested parameter. +- `references/notes/gxformat2.schema.json`: Companion file copied verbatim into the bundle. Sibling of `references/notes/gxformat2-schema.md`; read it where that note directs. ## Validation @@ -45,13 +52,27 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Procedure -Topology is settled upstream by the `*-summary-to-galaxy-template` skill, and the per-step loop is wrapper-tier — it does not edit topology. This skill is the **escalation target** the loop reaches when implementation proves the settled topology cannot support a declared step: a step output that needs evidence no wired input carries (the connection graph validates fine, but nothing produces what the output requires). Wrapper-tier gaps — no Tool Shed wrapper picked yet — are not this case; those route through the discover-or-author branch. +Topology is settled upstream by the `*-summary-to-galaxy-template` skill, and the per-step loop doesn't edit it. This skill makes the structural edits a settled draft still needs. It runs on an open blocking entry in the open-requirements-ledger: the settled draft can't give a correct run, and a structural edit closes it. A missing wrapper is this case only when the discover-or-author branch is exhausted: no stock, shed or authored wrapper can produce the step's declared output from its wired inputs, or the ledger already records the wrapper as confirmed absent. A tool version missing from a local cache is not absence. When the ledger records absence, don't search again. A chosen wrapper that can't produce one of the step's declared outputs is this case too, whatever other wrappers might do: an `add-step` carries that output, and the loop resolves its wrapper later. When the entry shows a consumer's wrapper does the step's work itself, as `rna_star` builds its index from a genome in the history, fold instead: `remove-step`, `rewire` its inputs to the consumers, and `change-parameter` on the consumers to select that mode. An added producer for the same work would raise the entry again. -You are invoked with the partially-realized draft and the open-requirements-ledger. Read the open blocking entries: each names a step, the uncomputable output, and the missing evidence. Your job is **bounded topology repair** — not re-settling the workflow. Repair the named region and nothing else; the surrounding topology, already-realized steps, and workflow interface stay put. +Decide the edit from the entry, the surrounding data-flow design (galaxy-data-flow-draft-contract), IWC structure, and the pattern pages. -Decide the *shape* of the fix from the missing evidence, the surrounding data-flow design (galaxy-data-flow-draft-contract), IWC structure, and the pattern pages. It may be one producer step, or a small sub-path of a few tools, or — when the declared output is genuinely uncomputable and no producer exists — narrowing the step to what its inputs can support. Insert the new step(s) in the draft superset (galaxy-workflow-draft-format): concrete topology and edges, wrapper-tier `TODO` for `tool_id` and ports, `_plan_*` fields carrying intent. The existing discover-or-author → `summarize-galaxy-tool` → `implement-galaxy-tool-step` machinery realizes them on later loop iterations; do not resolve wrappers here. +**Edit.** Close each open blocking entry you have evidence for, not only the one that triggered the call, with the smallest set of edits, named by change-set kind: `remove-step`, `rewire`, `change-parameter`, `add-input`, `remove-input`, `add-output` / `expose-output`, `remove-output`, `add-step`, `replace-tool`. Apply all but the last two in place. `add-step` and `replace-tool` land at draft tier (galaxy-workflow-draft-format): concrete topology and edges, wrapper-tier `TODO` for `tool_id` and ports, `_plan_*` carrying intent. The discover-or-author → `summarize-galaxy-tool` → `implement-galaxy-tool-step` machinery realizes them on later loop iterations. A step duplicated from a realized step keeps its pin and `state`, and changes only the key that distinguishes it. Everything the entries don't name stays as it is. -Then update the ledger. For each blocking entry your repair addresses, mark it `resolved` with a note on how (producer added, sub-path added, output narrowed). The loop's convergence gate counts open blocking entries and requires each escalation to strictly reduce that count, under a hard cap on escalations. If you cannot close an entry — no producer is discoverable and the output cannot be honestly narrowed — **surrender it**: leave it open with a note, so the terminal path writes it into the final draft as a labelled gap rather than spinning or fabricating. Never insert a producer whose own output is uncomputable from what feeds it; that grows the DAG without reducing the open count and the loop will not converge. +**Interface.** Change the workflow interface only when an entry names that as the remedy or a narrowing requires it. Promote an input with a default equal to the literal it replaces, and remove the literal from `state`. Expose an output by removing its `hide`, unless the design wants it hidden. Removing an input or output is a narrowing: append a `kind: dropped` entry with its `units` and a cited `because`. + +**Claims.** A claim about a step with no resolved wrapper is not a fact. Attach it to that step as a plain ledger entry with `step:` set, and keep it out of `doc:`. "Computable" means the tool's semantics give what the output declares, including element count and identifiers for collections. Reading a pinned wrapper's command block or a consumer's parser is allowed. Picking a wrapper is not. + +**Removal.** When nothing can close an entry against a drafty step, narrow the step if it isn't fully redundant, and rewrite its `_plan_*` to the reduced job. Otherwise remove it, append the `kind: dropped` entry, and resolve the blocking entry with a note naming the drop. A removal's `because` quotes the primary artifact it rests on: the search run, the wrapper's ports or command block, the semantics rule. The quoted artifacts must carry the removal alone: strike everything uncited, and the removal must still follow. Naming the blocking entry is not a citation, and a tool search says nothing about stock tools. Don't remove a realized step. + +**After every edit:** + +1. Remove what existed only to feed a removed step: compute it by walking every step's `in:`, the workflow outputs, and `${label}` in `out:` `rename`, not from the entry's wording. +2. Rewrite or remove mentions of removed ids and labels in `doc`, `_plan_*` and comment frames, and any `doc` promising what a removed step supplied. +3. Move a removed step's `rename` or `change_datatype` to the step that inherits its consumers. +4. If an edit touched a step with a twin, a step duplicated from the same tool and plan, make the same edit there. +5. Run draft-validate, with `--concrete --strict-state` when an edit touched a realized step, and fix what it reports. + +**Ledger.** Mark each entry you close `resolved`, with a note naming the kinds you applied. A repair must leave fewer open blocking entries than it found, or say it could close none, which ends escalation. Never insert a producer whose own output is uncomputable from what feeds it. Leave an entry `open`, with a note saying what's unproven, when a consumer truly needs the removed output, when no other edit closes an entry on a realized step, or when the removal doesn't follow from what you can quote. An entry on a realized step reaches the final artifact as a labelled gap. A drafty step stays drafty, and the call reports it. ## Feedback Mode diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/_provenance.json b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/_provenance.json index 31c12d2..48e7a45 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/_provenance.json +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/_provenance.json @@ -4,12 +4,28 @@ "mold": { "name": "repair-galaxy-draft-topology", "path": "content/molds/repair-galaxy-draft-topology/index.md", - "revision": 3, - "content_hash": "cafba235f3bb41612216f766eb390ee6fddcb1192ddf445b6d9eb7f2c132678c", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "revision": 7, + "content_hash": "692708480037be9680eda6be70c7fa2fd93604037432f7d94b6363266a6ca25e", + "commit": "db8801f2f0a31746e35cf399dd96cdbb8aa36dcf" }, - "cast_at": "2026-09-13T20:58:29.614Z", + "cast_at": "2026-11-10T10:22:52.006Z", "refs": [ + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[draft-validate]]", + "src": "content/cli/gxwf/draft-validate.md", + "dst": "references/cli/draft-validate.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Check the draft contract, including dangling edges and sources, on the edited draft.", + "trigger": "After every edit.", + "verification": "Cast the skill, run a removal repair on a real draft, confirm the diagnostics name any source left dangling and the repaired draft exits 0.", + "src_hash": "37cded427fbccf49ff19a6fe8b03f9e90857afe556e5db0e51a01402fbf33f1e", + "dst_hash": "afa59b0815308a340780247df0548909518564d58352a0c695ceca37a07eee5b", + "source": "deterministic" + }, { "kind": "pattern", "mode": "verbatim", @@ -20,9 +36,24 @@ "load": "on-demand", "evidence": "corpus-observed", "purpose": "Choose a corpus-attested collection recipe when the missing producer is a collection construction, reshape, or bridge.", - "trigger": "When the repair sub-path needs collection cleanup, reshaping, relabeling, identifier synchronization, or a collection-tabular bridge.", - "src_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", - "dst_hash": "9840d686228a254766611196f2be03b2a66119281322c6373ce082cd08e78717", + "trigger": "When an added step needs collection cleanup, reshaping, relabeling, identifier synchronization, or a collection-tabular bridge.", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-conditionals-patterns]]", + "src": "content/patterns/galaxy-conditionals-patterns.md", + "dst": "references/patterns/galaxy-conditionals-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Choose how to merge when a `when`-gated step's output feeds another step.", + "trigger": "When an inserted or rewired step consumes a `when`-gated step's output.", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", "source": "deterministic" }, { @@ -35,9 +66,9 @@ "load": "on-demand", "evidence": "corpus-observed", "purpose": "Choose a corpus-attested tabular recipe when the missing evidence is a column, key, or aggregate a tabular step can produce.", - "trigger": "When the repair sub-path needs a computed column, join key, filter criterion, or aggregate the blocked step depends on.", - "src_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", - "dst_hash": "e580e71fabec9132fbdf3165cc092fcdb96a8b9bf7996b3b9b6a4b0a627065cb", + "trigger": "When an added step needs a computed column, join key, filter criterion, or aggregate the blocked step depends on.", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", "source": "deterministic" }, { @@ -49,8 +80,8 @@ "used_at": "runtime", "load": "on-demand", "evidence": "corpus-observed", - "purpose": "Preserve collection typing and map-over/reduction semantics when an inserted producer step joins or reshapes collection inputs.", - "trigger": "When the repair inserts or rewires steps touching collection inputs, outputs, or mapped/reduced connections.", + "purpose": "Preserve collection typing and map-over/reduction semantics when an edit joins or reshapes collection inputs, and quote the rule when a removal rests on it.", + "trigger": "When the repair inserts or rewires steps touching collection inputs, outputs, or mapped/reduced connections, or a removal rests on collection semantics.", "license": "MIT", "license_file": "LICENSES/galaxy.LICENSE", "src_hash": "4ddc7587e187cf38a3dca02d6966e2f6ea58f9faf9bd97034ad513eb4c4c638b", @@ -67,8 +98,8 @@ "used_at": "runtime", "load": "on-demand", "evidence": "corpus-observed", - "purpose": "Preserve collection typing and map-over/reduction semantics when an inserted producer step joins or reshapes collection inputs.", - "trigger": "When the repair inserts or rewires steps touching collection inputs, outputs, or mapped/reduced connections.", + "purpose": "Preserve collection typing and map-over/reduction semantics when an edit joins or reshapes collection inputs, and quote the rule when a removal rests on it.", + "trigger": "When the repair inserts or rewires steps touching collection inputs, outputs, or mapped/reduced connections, or a removal rests on collection semantics.", "companion_of": "references/notes/galaxy-collection-semantics.md", "license": "MIT", "license_file": "LICENSES/galaxy.LICENSE", @@ -86,10 +117,10 @@ "used_at": "runtime", "load": "upfront", "evidence": "hypothesis", - "purpose": "Re-wire the affected region consistently with the data-flow design that settled the surrounding topology.", + "purpose": "Edit the affected region consistently with the data-flow design that settled the surrounding topology.", "verification": "Promote after two repairs preserve the surrounding data-flow design without re-deriving it from the source.", - "src_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", - "dst_hash": "f8f75f5a4ff750bff5f682847466492dd8a9910b95b5078373567f84ce16fb54", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", "source": "deterministic" }, { @@ -101,12 +132,49 @@ "used_at": "runtime", "load": "upfront", "evidence": "hypothesis", - "purpose": "Emit re-wired steps in the draft superset (TODO tool_id, _plan_* fields) so existing per-step machinery realizes them; respect the settle-vs-repair boundary.", + "purpose": "Emit added or replaced steps in the draft superset (TODO tool_id, _plan_* fields) so existing per-step machinery realizes them; respect the settle-vs-repair boundary.", "verification": "Promote after a repair run inserts a producer the per-step loop then realizes without re-running the template Mold over the whole draft.", - "src_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", - "dst_hash": "93f3b231dd063bf059c550f18b2200c61909af0da9408e0d43351116e886bf7b", + "src_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", + "dst_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", "source": "deterministic" }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[gxformat2-schema]]", + "src": "content/research/gxformat2-schema/index.md", + "dst": "references/notes/gxformat2-schema.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Keep post-job actions and nested parameter keys correct when an edit moves them.", + "trigger": "When an edit touches `out:` actions (`hide`, `rename`, `change_datatype`) or wires a nested parameter.", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", + "dst_hash": "47a3b94dc15bb954f9193e888522b7aaff80143f2486531a21146fa4daaa54d4", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[gxformat2-schema]]", + "src": "content/research/gxformat2-schema/gxformat2.schema.json", + "dst": "references/notes/gxformat2.schema.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Keep post-job actions and nested parameter keys correct when an edit moves them.", + "trigger": "When an edit touches `out:` actions (`hide`, `rename`, `change_datatype`) or wires a nested parameter.", + "companion_of": "references/notes/gxformat2-schema.md", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "2c4b3df730b70d9c648105ced438fc08be306a41f2aff7d4bd2897f72f94d509", + "dst_hash": "2c4b3df730b70d9c648105ced438fc08be306a41f2aff7d4bd2897f72f94d509", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, { "kind": "research", "mode": "verbatim", @@ -116,10 +184,10 @@ "used_at": "runtime", "load": "upfront", "evidence": "hypothesis", - "purpose": "Scope the repair from the open blocking entries — each names the step, the uncomputable output, and the missing evidence — then mark resolved each one repaired, and leave any it cannot close open with a surrender note for the terminal path.", + "purpose": "Scope the repair from the open blocking entries — each names the step and what's missing — then mark resolved each one repaired, and leave any it cannot close open with a note.", "verification": "Promote after a run closes a blocking entry and the loop terminates on the reduced open count.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -131,11 +199,11 @@ "used_at": "runtime", "load": "upfront", "evidence": "cast-validated", - "purpose": "In/out contract: the draft this Mold reads and re-wires conforms to [[galaxy-workflow-draft]]; cast bundles the JSON Schema so the re-wired draft stays inside the superset the per-step loop expects.", + "purpose": "In/out contract: the draft this Mold reads and edits conforms to [[galaxy-workflow-draft]]; cast bundles the JSON Schema so the edited draft stays inside the superset the per-step loop expects.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", - "dst_hash": "86822b6bf4c11ba1d9eaa12d9e442db7f7d28b2734a2daddf62b5cdd2f1f75aa", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } @@ -147,7 +215,7 @@ "kind": "yaml", "default_filename": "galaxy-workflow-draft.gxwf.yml", "schema": "[[galaxy-workflow-draft]]", - "description": "Re-wired gxformat2 draft: a producer step (or small sub-path) inserted so the previously-blocked step's declared output is computable; new steps land at draft (TODO) tier." + "description": "Edited gxformat2 draft. Each blocking entry addressed is closed by the named edits, and new or replaced steps land at draft tier." }, { "id": "open-requirements-ledger", @@ -159,7 +227,7 @@ "consumes": [ { "id": "galaxy-workflow-draft", - "description": "Partially-realized gxformat2 draft (see [[galaxy-workflow-draft-format]]) whose topology can't support a declared step — the region to repair.", + "description": "Partially-realized gxformat2 draft (see [[galaxy-workflow-draft-format]]) with an open blocking entry.", "inherited_schema": "[[galaxy-workflow-draft]]", "producers": [ "advance-galaxy-draft-step", @@ -168,7 +236,8 @@ "freeform-summary-to-galaxy-template", "implement-galaxy-tool-step", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-template" ] }, { @@ -191,7 +260,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/_required_tools.json b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/_required_tools.json new file mode 100644 index 0000000..18f1089 --- /dev/null +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/_required_tools.json @@ -0,0 +1,16 @@ +[ + { + "tool": "gxwf", + "origin": "npm", + "package": "@galaxy-tool-util/cli", + "package_version": "^1.15.0", + "invoke": "gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", + "availability_check": "gxwf --help | grep -q draft-validate", + "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", + "source": "implied", + "implied_by": [ + "gxwf draft-validate" + ] + } +] diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/cli/draft-validate.json b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/cli/draft-validate.json new file mode 100644 index 0000000..58fda0f --- /dev/null +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/cli/draft-validate.json @@ -0,0 +1,140 @@ +{ + "type": "cli-command", + "tool": "gxwf", + "command": "draft-validate", + "summary": "Validate a `class: GalaxyWorkflowDraft` workflow against draft-contract rules; with --concrete, also validate the extracted concrete subset.", + "source_path": "content/cli/gxwf/draft-validate.md", + "source_revision": 1, + "package": "@galaxy-tool-util/cli", + "description": "Validate a draft Galaxy workflow (class: GalaxyWorkflowDraft)", + "synopsis": "gxwf draft-validate [options] ", + "args": [ + { + "raw": "file", + "name": "file", + "required": true, + "variadic": false, + "description": "Draft workflow file (.gxwf.yml)" + } + ], + "options": [ + { + "flags": "--format ", + "name": "format", + "description": "Force format: format2 (native is rejected for drafts)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--json", + "name": "json", + "description": "Output structured JSON report", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--report-html [file]", + "name": "reportHtml", + "description": "Write HTML report to file (or stdout if omitted)", + "takesArgument": true, + "argumentPlaceholder": "[file]", + "optionalArgument": true, + "negatable": false + }, + { + "flags": "--report-markdown [file]", + "name": "reportMarkdown", + "description": "Write Markdown report to file (or stdout if omitted)", + "takesArgument": true, + "argumentPlaceholder": "[file]", + "optionalArgument": true, + "negatable": false + }, + { + "flags": "--concrete", + "name": "concrete", + "description": "Additionally extract the concrete subset and run the regular `gxwf validate` checks on it", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--cache-dir ", + "name": "cacheDir", + "description": "Tool cache directory (for --concrete tool-state validation)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--offline", + "name": "offline", + "description": "Do not fetch uncached tools; read only the local cache (uncached tools are skipped)", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--galaxy-url ", + "name": "galaxyUrl", + "description": "Galaxy instance to try as a tool source after the ToolShed when fetching uncached tools", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--no-tool-state", + "name": "toolState", + "description": "Skip tool-state validation in the --concrete pass (default: enabled when --concrete is set)", + "takesArgument": false, + "optionalArgument": false, + "negatable": true + }, + { + "flags": "--connections", + "name": "connections", + "description": "Validate connection-type compatibility on the --concrete subset (collection algebra, map-over)", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict", + "name": "strict", + "description": "Shorthand for --strict-structure --strict-encoding --strict-state", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict-structure", + "name": "strictStructure", + "description": "Reject unknown keys at envelope/step level", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict-encoding", + "name": "strictEncoding", + "description": "Reject JSON-string tool_state and format2 field misuse", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict-state", + "name": "strictState", + "description": "Require every tool step to validate; no skips allowed", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + } + ], + "body": "# `gxwf draft-validate`\n\nValidate a draft Galaxy workflow against the **draft contract**: sentinel form, dangling edge references, top-level `_plan_*` placement, `_plan_*` on fully-resolved tool steps, and recursive draft subworkflows. Native (.ga) input is rejected — drafts are format2-only.\n\nDistinct from [[validate]], which validates a fully concrete `class: GalaxyWorkflow` and would reject the draft relaxations outright. Use `draft-validate` during the per-step authoring loop; use `validate` at the terminal pass once `promoteFullyConcreteDrafts` has flipped the class.\n\n## Output\n\nDefault output is human-readable: counted buckets for structure / topology / semantic errors and warnings, plus a one-line survey (TODO sentinel count, paths carrying `_plan_*`). `--json` emits a `SingleDraftValidationReport`; `--report-html` and `--report-markdown` write the same data as a self-contained HTML page or templated Markdown. With `--concrete`, the report carries an optional `concrete: ConcreteValidationReport` whose buckets (`structure_errors`, `strict_structure_errors`, `strict_encoding_errors`, `strict_state_errors`, `tool_state`, `connection_report`) are **absent when the corresponding check did not run** — readers should treat absence as \"not run,\" not as \"passed.\"\n\n## Flags\n\n`--concrete` runs the extract+promote pipeline (`extractConcreteSubset` → `stripPlanFields` → `promoteFullyConcreteDrafts`) and applies the full concrete `gxformat2` validation surface to the result. The following pass-through flags only take effect under `--concrete`; passing them without it prints a stderr warning and no-ops:\n\n- `--cache-dir ` — tool cache for tool-state lookups.\n- `--no-tool-state` — skip tool-state validation on the concrete pass. Combined with `--strict-state`, the strict flag warns + no-ops (there's no state to be strict about).\n- `--connections` — run connection validation on the concrete subset.\n- `--strict` — escalate every strict bucket (structure, encoding, state) to error.\n- `--strict-structure` / `--strict-encoding` / `--strict-state` — escalate one bucket only.\n\nDraft structural errors gate the concrete pass entirely — the pipeline would mutate a malformed input, so the concrete section is skipped and only the draft buckets populate.\n\n## Examples\n\n```bash\ngxwf draft-validate workflow.gxwf.yml\ngxwf draft-validate workflow.gxwf.yml --json\ngxwf draft-validate workflow.gxwf.yml --concrete --json\ngxwf draft-validate workflow.gxwf.yml --concrete --strict --cache-dir ~/.cache/gxwf\ngxwf draft-validate workflow.gxwf.yml --report-html report.html\n```\n\n## Exit codes\n\n- `0` — clean draft (no structure / topology / semantic errors; warnings allowed). Under `--concrete`, also requires the concrete pass to be clean.\n- `1` — draft validation errors (topology or semantic). With `--concrete`, also returned when the concrete projection fails — including strict-* failures that `gxwf validate` would have exited `2` on standalone. The asymmetry is deliberate: the draft itself parsed, only its projection failed.\n- `2` — parse failure, native input rejected, structural decode failure (class is not `GalaxyWorkflowDraft`), or a stdout-sink collision between `--json` and `--report-{html,markdown}=-`.\n\n## Gotchas\n\n- The draft validator is **contract-shaped**, not workflow-shaped. It will not catch tool-state errors, parameter-name mismatches, or other concerns that belong to [[validate]] on the concretized subset. Use `--concrete` (or run [[draft-extract]] | [[validate]] manually) when you need both surfaces.\n- A draft with zero implemented steps still runs the `--concrete` pipeline, but the projection is empty — the concrete buckets report `skipped` semantics (absent in the JSON) rather than a green pass. Signal grows as the loop fills steps in; do not interpret early-loop runs as evidence the workflow concretizes correctly.\n- `_plan_*` on a fully-resolved tool step is a `semanticError`, not a warning. Strip planning fields explicitly via [[draft-extract]] (or `stripPlanFields`) when a step concretizes; the per-step Mold should never carry them forward.\n- Subworkflow draft roots are validated recursively; diagnostics carry the outer step path.\n- `TODO`, `TODO_x`, `TODO-x` are sentinels; `TODONE` and `TODOLIST` are not.\n- Prefer `--json` whenever a cast skill or harness needs to classify diagnostics." +} diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-data-flow-draft-contract.md index fac3c9c..853abc2 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-data-flow-draft-contract.md +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-data-flow-draft-contract.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 4 related_notes: - "[[nextflow-to-galaxy-channel-shape-mapping]]" - "[[nextflow-operators-to-galaxy-collection-recipes]]" @@ -15,9 +15,11 @@ related_molds: - "[[nextflow-summary-to-galaxy-data-flow]]" - "[[cwl-summary-to-galaxy-data-flow]]" - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[advance-galaxy-draft-step]]" sources: diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-workflow-draft-format.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-workflow-draft-format.md index 2a66f1a..9fc6e51 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-workflow-draft-format.md +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/galaxy-workflow-draft-format.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-06 -revised: 2026-05-10 -revision: 2 +revised: 2026-11-10 +revision: 6 related_notes: - "[[gxformat2-schema]]" - "[[galaxy-data-flow-draft-contract]]" @@ -17,6 +17,7 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" - "[[implement-galaxy-tool-step]]" - "[[advance-galaxy-draft-step]]" @@ -117,14 +118,14 @@ steps: ## Topology repair escalation -The template Mold settles topology, but it can miss a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. +The template Mold settles topology, but a settled draft can still be unable to give a correct run. The common case is a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. When one surfaces, the per-step loop does not author topology to fix it. It **escalates**: -- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. -- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and re-wires the affected region: one producer step, a small sub-path, or honest narrowing of the output. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. +- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs, or that the chosen wrapper can't produce it. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. Earlier Molds and the loop raise blocking entries too. +- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and makes the smallest structural edits that close them. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. -This is **repair**, distinct from **settling**: it is narrow (one named region), reactive (only a detected computability gap triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each escalation must strictly reduce that count, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. +This is **repair**, distinct from **settling**: it is narrow (the entries bound it), reactive (an open blocking entry triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each repair must leave fewer than it found, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. ## Open work diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/gxformat2-schema.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/gxformat2-schema.md new file mode 100644 index 0000000..af5053a --- /dev/null +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/gxformat2-schema.md @@ -0,0 +1,74 @@ +--- +type: research +title: "gxformat2 structural JSON Schema" +tags: + - target/galaxy +status: draft +created: 2026-05-05 +revised: 2026-11-06 +revision: 5 +license: MIT +license_file: LICENSES/galaxy-tool-util-ts.LICENSE +related_notes: + - "[[galaxy-collection-semantics]]" + - "[[galaxy-datatypes-conf]]" + - "[[galaxy-workflow-testability-design]]" + - "[[gxformat2-workflow-inputs]]" + - "[[galaxy-workflow-draft]]" +sources: + - "https://github.com/jmchilton/galaxy-tool-util-ts/blob/7ae4ecd0ba8d492225f58a6d455c4cc5317298f0/packages/schema/src/galaxy-workflow.ts" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py" + - "https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py" +companions: + - "gxformat2.schema.json" +summary: "Vendored gxformat2 structural schema, its input, output, and step vocabulary, and the limits of schema-only validation." +--- + +# gxformat2 structural schema + +The companion `gxformat2.schema.json` is a vendored Draft-07 JSON Schema generated from the `@galaxy-tool-util/schema` Effect-TS model at `7ae4ecd`. Casting carries it with this note. Use it to inspect the structural vocabulary of a gxformat2 workflow, then check connections, tool references, and runtime behavior with gxwf and Galaxy validation. The schema does not establish that an `outputSource` exists or that a tool state is valid for its wrapper. + +The schema is regenerated by `pnpm sync:vendored --update`, which builds the upstream package and runs `gxwf structural-schema --format format2` before copying it here. Do not edit the companion by hand. + +## Root and input fields + +The root requires `class: GalaxyWorkflow`, `inputs`, `outputs`, and `steps`. Those sections accept arrays or keyed maps. The root also permits `id`, `label`, `doc`, `uuid`, `report`, `tags`, `comments`, `creator`, `license`, and `release`. + +Input entries describe the public workflow interface. The array form exposes these fields: + +| Field | Structural schema | Meaning and use | +|---|---|---| +| `id`, `label`, `doc` | Optional | Identify or describe the input. Check the effective imported name before using it as a test `job:` key. | +| `type` | Optional string, string array, or `null` | Enum members are `null`, `boolean`, `int`, `long`, `float`, `double`, `string`, `integer`, `text`, `File`, `data`, and `collection`. This is accepted vocabulary, not a list of preferred authoring spellings. | +| `format` | Optional string array or `null` | Galaxy datatype extension filter. See [[galaxy-datatypes-conf]]. | +| `collection_type` | Optional string or `null` | Collection shape such as `list:paired`. The schema does not enumerate valid shapes. See [[galaxy-collection-semantics]]. | +| `optional`, `default` | Optional boolean or `null`, and opaque value, respectively | Control whether an input can be omitted and what value is supplied when absent. See [[gxformat2-workflow-inputs]] for runtime behavior. | +| `position` | Optional `{top, left}` numeric coordinates or `null` | Workflow editor placement. | + +The generated schema does not include all fields accepted by current gxformat2 input handling, including `restrictions`, `suggestions`, and `restrictOnConnections`. [[gxformat2-workflow-inputs]] distinguishes that implementation behavior from what this vendored schema can check. + +## Output names and connections + +An output entry can carry `id`, `label`, `doc`, `outputSource`, and `type`. `outputSource` names the producing step output, written `step/output`. The output's **public name** is not always its `label`: normalized gxformat2 export places a native Galaxy workflow-output label in `id`, and the pinned IWC [SRA manifest workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs.ga) exports `id: paired_output`. Current gxformat2 import chooses `label` when present and otherwise `id`. Check the effective name after import or conversion before writing the sibling test's `outputs:` key. See [[galaxy-workflow-testability-design]]. + +The schema makes `outputSource` optional, even though a usable produced output needs a valid source. Its string type cannot prove that the referenced step and port exist. gxformat2 lint checks the step reference. Workflow validation must also check that the producing port exists. + +The step part of a `source` or `outputSource` depends on how the workflow lists its steps. In list form it is the step's `label` when one is present, otherwise its `id`. In map form, for a step with no `label`, it is the map key. + +A step's `in:` key addresses a parameter nested in a conditional or section with a `|`-joined path, and a repeat instance with `_N`. For example, `input_type|i` wires an input inside a conditional in `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences`, and `style_cond|type_cond|pick_from_0|value` wires the first instance of a repeat inside two conditionals in [[conditional-route-between-alternative-outputs]]. The key names the path, and the conditional's selector value stays in `state`. Each `_N` connection also needs its own entry in the repeat in `state`. The repeat creates the input terminal, so without the entry the connection does nothing and the job can still succeed. + +Step `out:` entries serve a different purpose. They can be output names or records with `id` and actions such as `rename`, `change_datatype`, `add_tags`, `remove_tags`, `hide`, `delete_intermediate_datasets`, and `set_columns`. These actions affect produced datasets or history presentation. They do not supply a top-level output's public name. + +A `rename` value is not a tool-XML label template. Galaxy's [`RenameDatasetAction`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/job_execution/actions/post.py#L152-L256) fills in two forms. `#{input}` becomes the name of the dataset bound to that tool input. Use the tool's input name, with `.` for a nested one (e.g. `queries_0.input2`), and optional `| basename`, `| upper` or `| lower`. `${label}` becomes the value of the workflow parameter input with that label. Write `#{gtf_to_extend} BREW3R extended` to name an output after its input. A `${name}` with no matching parameter input is an informal, untyped parameter ([`get_informal_replacement_parameters`](https://github.com/galaxyproject/galaxy/blob/v26.1.1/lib/galaxy/workflow/modules.py#L3094-L3103)). The run form asks for a free-text value, and an API or Planemo run that supplies no `replacement_params` leaves `${name}` literal in the dataset name. The tool-XML variables `${on_string}` and `${tool.name}` work only in a wrapper's ``; in a rename they fall into this case. This is what the workflow linters report as an "untyped parameter"; declare a parameter input instead. + +## Step fields and state + +The structural step `type` enum is `tool`, `subworkflow`, `pause`, and `pick_value`, with `null` also admitted. gxformat2's source schema defaults an omitted type to `tool`. A step can also carry `id`, `label`, `doc`, `position`, `tool_id`, `tool_shed_repository`, `tool_version`, `in`, `out`, `state`, `tool_state`, `run`, `runtime_inputs`, `when`, `errors`, and `uuid`. The structural schema does not require subtype-specific fields such as `tool_id` on a tool step, so validation must continue beyond JSON Schema. + +For hand-authored tool steps, use `state` for typed nested YAML, including conditionals and repeats. A converted workflow may contain `tool_state`, as the pinned IWC SRA manifest example does. Treat that as the representation received, not an authoring recommendation. Current import accepts `tool_state` as a map or JSON string, but gives `state` precedence if both fields are present. The structural schema permits both, so use only one on a step. Preserve an exported representation when reading or carrying it forward, and use `state` when authoring a new step. + +## What schema validation leaves open + +The root and array-form entries constrain their listed properties, but generated map-form `inputs` and `outputs` each include a permissive object branch. Passing this schema therefore does not prove that every map entry has only documented fields. In map form, the `type` enum and `format` shape can also escape the documented constraints. Every form still needs semantic checks for connections and tool configuration. + +The generated JSON also omits field descriptions and repeats `$id: "/schemas/unknown"` on opaque values. Some strict Ajv configurations reject those duplicate identifiers. If that occurs, compile with the project's supported non-strict configuration or remove duplicate markers in a validation copy. Keep the vendored companion unchanged. The upstream [gxformat2 source schema](https://github.com/galaxyproject/gxformat2/blob/main/schema/v19_09/workflow.yml) supplies field documentation that this generated artifact lacks. diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/gxformat2.schema.json b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/gxformat2.schema.json new file mode 100644 index 0000000..526c8d2 --- /dev/null +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/gxformat2.schema.json @@ -0,0 +1,3926 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$defs": { + "WorkflowStepSchema": { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "tool_id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "tool_shed_repository": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "name", + "changeset_revision", + "owner", + "tool_shed" + ], + "properties": { + "name": { + "type": "string" + }, + "changeset_revision": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "tool_shed": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "tool_version": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "errors": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "in": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "source": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "source": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "null" + } + ] + }, + "out": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "add_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "change_datatype": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "delete_intermediate_datasets": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "hide": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "remove_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "rename": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "set_columns": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "add_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "change_datatype": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "delete_intermediate_datasets": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "hide": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "remove_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "rename": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "set_columns": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "null" + } + ] + }, + "state": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "tool_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "tool", + "subworkflow", + "pause", + "pick_value" + ] + } + ] + }, + "run": { + "anyOf": [ + { + "type": "object", + "required": [ + "inputs", + "outputs", + "class", + "steps" + ], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "inputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "outputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "GalaxyWorkflow" + ] + }, + "steps": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/WorkflowStepSchema" + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$ref": "#/$defs/WorkflowStepSchema" + } + } + ] + }, + "report": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "markdown" + ], + "properties": { + "markdown": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "comments": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "creator": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Person" + ] + }, + "givenName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "familyName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificPrefix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificSuffix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "jobTitle": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Organization" + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "license": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "release": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "runtime_inputs": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "when": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + } + }, + "type": "object", + "required": [ + "inputs", + "outputs", + "class", + "steps" + ], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "inputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "default": { + "$id": "/schemas/unknown", + "title": "unknown" + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "type": { + "anyOf": [ + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "outputs": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection" + ] + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "string" + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$id": "/schemas/unknown", + "title": "unknown" + } + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "GalaxyWorkflow" + ] + }, + "steps": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/WorkflowStepSchema" + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$ref": "#/$defs/WorkflowStepSchema" + } + } + ] + }, + "report": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "markdown" + ], + "properties": { + "markdown": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "comments": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "text" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "markdown" + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "frame" + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + } + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "position": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "null" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "freehand" + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "line": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "creator": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Person" + ] + }, + "givenName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "familyName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificPrefix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificSuffix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "jobTitle": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "class" + ], + "properties": { + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "Organization" + ] + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, + "license": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "release": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false +} diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-collection-patterns.md index bd4ddc7..b00b1d5 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-collection-patterns.md +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-collection-patterns.md @@ -13,9 +13,9 @@ tags: - topic/collection-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 -summary: "Use this MOC to choose corpus-grounded Galaxy collection transformation patterns." +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." related_notes: - "[[iwc-transformations-survey]]" - "[[iwc-conditionals-survey]]" @@ -25,10 +25,13 @@ related_patterns: - "[[reshape-relabel-remap-by-collection-axis]]" - "[[fan-in-bundle-consume-and-flatten]]" - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" - "[[sync-collections-by-identifier]]" - "[[harmonize-by-sortlist-from-identifiers]]" - "[[regex-relabel-via-tabular]]" - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" - "[[collection-swap-nesting-with-apply-rules]]" - "[[collection-split-identifier-via-rules]]" - "[[collection-build-list-paired-with-apply-rules]]" @@ -42,45 +45,46 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: collection patterns -This is the runtime-facing map for Galaxy collection transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation and recipe pages are the actionable references. +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. -## Cleanup +## Enter or leave a collection -- [[collection-cleanup-after-mapover-failure]] — use `__FILTER_EMPTY_DATASETS__` or `__FILTER_FAILED_DATASETS__` after map-over when empty or errored elements would break downstream steps. -- [[collection-unbox-singleton]] — use `__EXTRACT_DATASET__` with `which: first` when a known one-element collection must become a dataset. +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. -## Map-over Lifecycle Recipes +## Keep mapped siblings aligned -- [[manifest-to-mapped-collection-lifecycle]] — build collection elements from manifest/table rows, map a tool over them, then relabel or reshape outputs. -- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped outputs, sync siblings from surviving identifiers, then gate final reports. -- [[reshape-relabel-remap-by-collection-axis]] — correct the mapped axis after domain fan-out using Apply Rules and deterministic relabeling. -- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection consumer, then flatten or aggregate results. +First identify what changed. Membership, order, and labels are separate decisions. -## Identifiers +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. -- [[sync-collections-by-identifier]] — membership sync: extract identifiers from one collection and filter or relabel a sibling collection. -- [[harmonize-by-sortlist-from-identifiers]] — order sync: sort one sibling collection by another collection's identifier order. -- [[regex-relabel-via-tabular]] — label rewrite: derive new element identifiers in tabular form and apply them with `__RELABEL_FROM_FILE__`. -- [[relabel-via-rules-and-find-replace]] — relabel inside a structural reshape, currently grounded in the influenza fan-out pattern. +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. -## Structural Reshape +## Change the collection axes -- [[collection-flatten-after-fanout]] — collapse nested collection output to a flat list when the outer axis no longer matters. -- [[collection-build-named-bundle]] — assemble individual outputs into a named collection bundle for publishing or downstream fan-in. -- [[collection-swap-nesting-with-apply-rules]] — use Apply Rules to swap `list:list` axes. -- [[collection-split-identifier-via-rules]] — use Apply Rules regex columns to derive nested list identifiers from one identifier string. -- [[collection-build-list-paired-with-apply-rules]] — use Apply Rules to promote identifier columns into `list:paired` structure. +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. -## Bridges +## Follow a map-over lifecycle -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. ## See also diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-conditionals-patterns.md new file mode 100644 index 0000000..2dfe890 --- /dev/null +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-conditionals-patterns.md @@ -0,0 +1,62 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: conditionals patterns" +aliases: + - "Galaxy conditional pattern MOC" + - "Galaxy when patterns" + - "conditional workflow patterns" +tags: + - target/galaxy + - topic/galaxy-transform +status: draft +created: 2026-05-02 +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." +related_notes: + - "[[iwc-conditionals-survey]]" +related_patterns: + - "[[conditional-run-optional-step]]" + - "[[conditional-route-between-alternative-outputs]]" + - "[[conditional-gate-on-nonempty-result]]" + - "[[conditional-transform-or-pass-through]]" + - "[[collection-cleanup-after-mapover-failure]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: conditionals patterns + +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. + +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. + +## Skip work without replacing its output + +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. + +## Continue with one value + +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. + +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. + +## See Also + +- [[iwc-conditionals-survey]] — conditionals survey and evidence trail. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-tabular-patterns.md index 445ed6c..591b3f5 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-tabular-patterns.md +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/patterns/galaxy-tabular-patterns.md @@ -13,8 +13,8 @@ tags: - topic/tabular-transform status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-10-15 +revision: 4 summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." related_notes: - "[[iwc-tabular-operations-survey]]" @@ -40,38 +40,39 @@ related_molds: - "[[nextflow-summary-to-galaxy-template]]" - "[[cwl-summary-to-galaxy-template]]" - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" - "[[compare-against-iwc-exemplar]]" --- # Galaxy: tabular patterns -This is the runtime-facing map for Galaxy tabular transformation choices. Use it before loading raw survey notes. The survey remains evidence backing; the operation pages are the actionable references. +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. -## Row And Column Operations +## Rows and columns in one table -- [[tabular-filter-by-column-value]] — keep/drop rows by string column value with `Filter1`. -- [[tabular-filter-by-regex]] — keep/drop rows by regex or pattern matching. -- [[tabular-cut-and-reorder-columns]] — project and reorder columns with `Cut1`. -- [[tabular-compute-new-column]] — compute row-wise values into a new column. +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. -## Joins And Aggregation +## Combine or summarize tables -- [[tabular-join-on-key]] — join two tabular datasets by key columns. -- [[tabular-group-and-aggregate-with-datamash]] — group and aggregate rows with `datamash_ops`. -- [[tabular-sql-query]] — use SQL when filtering, joining, and projection are clearer as one query. +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. -## Text-Processing Recipes +## Text-processing recipes -- [[tabular-prepend-header]] — add a header row with an awk/text-processing step. -- [[tabular-synthesize-bed-from-3col]] — build BED from three-column tabular inputs. -- [[tabular-split-taxonomy-string]] — split taxonomy-like strings into useful columns. -- [[tabular-relabel-by-row-counter]] — synthesize row labels from row order. +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. -## Bridges +## Between tables and collections -- [[tabular-to-collection-by-row]] — split a tabular manifest/list into collection elements for map-over. -- [[tabular-concatenate-collection-to-table]] — row-bind a collection of tabular outputs into one table. -- [[tabular-pivot-collection-to-wide]] — outer-join a collection of id/value tabulars into one wide table. +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. ## See also diff --git a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/schemas/galaxy-workflow-draft.schema.json index 3a39f43..dd3a94a 100644 --- a/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/schemas/galaxy-workflow-draft.schema.json +++ b/plugins/foundry-skills/skills/repair-galaxy-draft-topology/references/schemas/galaxy-workflow-draft.schema.json @@ -75,7 +75,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_version": { "anyOf": [ @@ -108,7 +108,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "out": { "anyOf": [ @@ -309,7 +309,7 @@ ] }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "release": { "anyOf": [ @@ -412,22 +412,28 @@ "items": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" } ] }, @@ -437,22 +443,28 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared11" + "$ref": "#/$defs/Shared12" }, { "$ref": "#/$defs/Shared5" }, { - "$ref": "#/$defs/Shared10" + "$ref": "#/$defs/Shared11" }, { - "$ref": "#/$defs/Shared12" + "$ref": "#/$defs/Shared13" }, { "$ref": "#/$defs/Shared8" }, { - "$ref": "#/$defs/Shared13" + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" }, { "type": "string" @@ -927,7 +939,7 @@ ] }, "in": { - "$ref": "#/$defs/Shared16" + "$ref": "#/$defs/Shared19" }, "label": { "anyOf": [ @@ -1040,7 +1052,7 @@ ] }, "tool_shed_repository": { - "$ref": "#/$defs/Shared15" + "$ref": "#/$defs/Shared18" }, "tool_state": { "anyOf": [ @@ -1856,7 +1868,9 @@ "text", "File", "data", - "collection" + "collection", + "color", + "directory_uri" ], "type": "string" } @@ -1952,10 +1966,10 @@ ] }, "restrictions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "suggestions": { - "$ref": "#/$defs/Shared17" + "$ref": "#/$defs/Shared20" }, "type": { "enum": [ @@ -1963,6 +1977,9 @@ "string" ], "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" } }, "required": [ @@ -1971,6 +1988,96 @@ "type": "object" }, "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { "additionalProperties": false, "properties": { "default": { @@ -2016,7 +2123,7 @@ "required": [], "type": "object" }, - "Shared10": { + "Shared11": { "additionalProperties": false, "properties": { "default": { @@ -2124,7 +2231,7 @@ ], "type": "object" }, - "Shared11": { + "Shared12": { "additionalProperties": false, "properties": { "default": { @@ -2230,7 +2337,7 @@ "required": [], "type": "object" }, - "Shared12": { + "Shared13": { "additionalProperties": false, "properties": { "default": { @@ -2337,7 +2444,7 @@ ], "type": "object" }, - "Shared13": { + "Shared14": { "additionalProperties": false, "properties": { "default": { @@ -2424,7 +2531,143 @@ ], "type": "object" }, - "Shared14": { + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { "anyOf": [ { "items": { @@ -2457,7 +2700,7 @@ } ] }, - "Shared15": { + "Shared18": { "anyOf": [ { "type": "null" @@ -2488,11 +2731,11 @@ } ] }, - "Shared16": { + "Shared19": { "anyOf": [ { "items": { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, "type": "array" }, @@ -2500,7 +2743,7 @@ "additionalProperties": { "anyOf": [ { - "$ref": "#/$defs/Shared9" + "$ref": "#/$defs/Shared10" }, { "type": "string" @@ -2522,7 +2765,7 @@ } ] }, - "Shared17": { + "Shared20": { "anyOf": [ { "type": "null" @@ -2610,7 +2853,7 @@ "$ref": "#/$defs/Shared1" }, "outputs": { - "$ref": "#/$defs/Shared14" + "$ref": "#/$defs/Shared17" }, "uuid": { "anyOf": [ diff --git a/plugins/foundry-skills/skills/review-galaxy-workflow/SKILL.md b/plugins/foundry-skills/skills/review-galaxy-workflow/SKILL.md index 16437fe..7aa9b75 100644 --- a/plugins/foundry-skills/skills/review-galaxy-workflow/SKILL.md +++ b/plugins/foundry-skills/skills/review-galaxy-workflow/SKILL.md @@ -22,7 +22,7 @@ Follow the procedure below and use the artifact/reference sections as the runtim - Read artifact `galaxy-workflow-pr-context`. Optional; absence is allowed and must be reported honestly. Harness-supplied pull-request context: repository, pull request number, title and body, base and head SHAs, changed-file list, base-to-head diff, and the complete relevant files. - Read artifact `iwc-dockstore-metadata`. Optional; absence is allowed and must be reported honestly. Produced by `mature-galaxy-workflow-for-iwc`. The submission's `.dockstore.yml` when present; it names the primary Galaxy descriptor under the IWC-Lab repository profile, which may be a `.gxwf.yml` file. - Read artifact `iwc-comparison-notes`. Optional; absence is allowed and must be reported honestly. Produced by `compare-against-iwc-exemplar`. Existing structural diff from compare-against-iwc-exemplar when a Foundry run already produced one. Consume it; never rerun that comparison inside this Mold. -- Read artifact `open-requirements-ledger`. Optional; absence is allowed and must be reported honestly. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`. Carried obligations ledger open-requirements-ledger when a Foundry run supplied one; read-only here, used to report unresolved intent and surrendered work. +- Read artifact `open-requirements-ledger`. Optional; absence is allowed and must be reported honestly. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger when a Foundry run supplied one; read-only here, used to report unresolved intent and surrendered work. ## Outputs diff --git a/plugins/foundry-skills/skills/review-galaxy-workflow/_provenance.json b/plugins/foundry-skills/skills/review-galaxy-workflow/_provenance.json index e42663a..06c8888 100644 --- a/plugins/foundry-skills/skills/review-galaxy-workflow/_provenance.json +++ b/plugins/foundry-skills/skills/review-galaxy-workflow/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/review-galaxy-workflow/index.md", "revision": 1, "content_hash": "6c65f653439814a9150cd24341e67a612f4e2c06c0bf2dd7ee838c9c8430bafe", - "commit": "79bf5c3ab98eec3a67948d8bdebb9dba7d8a2359" + "commit": "94ce4ea20107680b467d03e8889b63689774e325" }, - "cast_at": "2026-09-12T19:19:46.936Z", + "cast_at": "2026-11-10T10:05:12.873Z", "refs": [ { "kind": "prompt", @@ -38,8 +38,8 @@ "evidence": "corpus-observed", "purpose": "Distinguish a genuine workflow-interface or test-addressing defect from a stylistic preference when a label and a test key disagree.", "trigger": "When an input, promoted output, collection identifier, or workflow-output label does not agree with the supplied test.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" }, { @@ -53,8 +53,8 @@ "evidence": "corpus-observed", "purpose": "Recognize corpus-observed shortcuts so a finding cites an observed IWC anti-pattern rather than reviewer taste.", "trigger": "When a structural choice looks like a shortcut and the finding needs corpus grounding before it is raised as required.", - "src_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", - "dst_hash": "14cc27c6a3e01588b06151535afdcb671cadd8e10dc960a248934e9133ebc8ab", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", "source": "deterministic" }, { @@ -68,8 +68,8 @@ "evidence": "corpus-observed", "purpose": "Judge test labels, durable remote fixtures, creator identifiers, and companion-file naming against current IWC conventions.", "trigger": "When assessing the submission's test file, test-data references, creator metadata, or release entry.", - "src_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", - "dst_hash": "1921e939703444604440e0768caca6e6b834a73dcfa964f450fa081143e008d3", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", "source": "deterministic" }, { @@ -84,8 +84,8 @@ "purpose": "Read a supplied ledger's open and surrendered entries into the clearly labeled Foundry context section without treating their absence as an IWC failure.", "trigger": "When the harness supplied an open-requirements ledger and the review needs to report unresolved intent or surrendered work as Foundry context.", "verification": "Run the with-foundry-context and without-foundry-context scenarios; confirm the labeled section appears in one, is omitted or explicitly stated absent in the other, and that the IWC checklist verdicts are identical between them.", - "src_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", - "dst_hash": "e1d5e3a4d6d650af5b2efc4703fe6f79bb4d0cc12e6cd14e841c5b91df122ea0", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", "source": "deterministic" }, { @@ -202,7 +202,10 @@ "nextflow-summary-to-galaxy-interface", "nextflow-summary-to-galaxy-reference-data", "nextflow-summary-to-galaxy-template", - "repair-galaxy-draft-topology" + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" ] } ] diff --git a/plugins/foundry-skills/skills/review-galaxy-workflow/_verify.json b/plugins/foundry-skills/skills/review-galaxy-workflow/_verify.json index 8c0e832..c089863 100644 --- a/plugins/foundry-skills/skills/review-galaxy-workflow/_verify.json +++ b/plugins/foundry-skills/skills/review-galaxy-workflow/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-galaxy-workflow", + "--schema-sha256", + "55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-shortcuts-anti-patterns.md index ae98802..8dc1aa5 100644 --- a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-shortcuts-anti-patterns.md +++ b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-shortcuts-anti-patterns.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,342 +16,37 @@ related_notes: - "[[iwc-map-over-lifecycle-survey]]" - "[[iwc-tabular-operations-survey]]" - "[[iwc-transformations-survey]]" -summary: "What IWC test suites cut corners on (accepted) vs what's a code smell — existence-only probes, sim_size deltas, image dim checks, label coupling." +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." --- # IWC test-suite shortcuts and anti-patterns -## Purpose +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. -When an agent translates or authors a Galaxy workflow for IWC submission, the test suite it writes will be reviewed against IWC's *de facto* style — not against an idealized assertion ladder. That style routinely tolerates assertions that look weak in isolation. This note distinguishes the corner-cutting that is **normal and accepted** in the corpus from the patterns that an agent should treat as **smells** worth flagging. +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. -This note owns accepted-vs-smell calls. For positive workflow-structure guidance behind label stability, checkpoint promotion, and collection identifier design, use [[galaxy-workflow-testability-design]]. +## Choose by the failure the test must catch -Grounding: 115 `*-tests.yml` files under `workflow-fixtures/iwc-src/workflows/` (mirror of `galaxyproject/iwc`), prior synthesis in `galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md`. Path citations below are relative to `iwc-src/workflows/` unless absolute. +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | -## TL;DR rules of thumb +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. -1. **Default to tolerant assertions.** `compare: sim_size` + `delta:`, `has_image_*` + `delta:`, `has_text` substring, `has_h5_keys`, `has_n_lines` + `delta:` are *the IWC vocabulary*. Strict `compare: diff` or exact-`file:` is the exception, used only when the upstream tool is fully deterministic on fixed inputs. -2. **No negative tests.** `expect_failure:` does not appear in the corpus. Don't author one. -3. **No checksums.** `md5:` / `checksum:` do not appear on outputs in the corpus. SHA-1 hashes are used on **inputs** (integrity of remote fetch), never on output assertions. -4. **Preserve labels.** Inputs and outputs are referenced by label. Renaming silently breaks tests; treat label changes as breaking changes that require a sibling `-tests.yml` update. -5. **Big data goes to Zenodo.** In-repo `test-data/` is for toy fixtures and expected outputs only. +## Size and difference tolerances need a reason ---- - -## 1. Existence-only content probes - -### Accepted - -The HyPhy test files reduce JSON output validation to "starts with `{`": - -- `comparative_genomics/hyphy/hyphy-core-tests.yml:32-71` — four output collections (`meme_output`, `prime_output`, `busted_output`, `fel_output`), each gene element asserts `has_text: text: "{"` and nothing else. -- `comparative_genomics/hyphy/hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` — same pattern across the rest of the family. - -This is **accepted** because: -- HyPhy's MEME/PRIME/BUSTED/FEL/CFEL/RELAX statistical outputs embed run-dependent floats throughout (likelihoods, AIC, posterior probabilities). Substring assertions on numeric fields would fail intermittently. -- Selecting any specific gene name or category in the JSON would couple the test to internal HyPhy keying that the wrapper has changed across versions. -- The assertion does verify "the tool ran, produced JSON, did not crash, and the collection structure matches expected element identifiers" — which is genuinely useful given HyPhy's history of opaque failures. - -A quick scan finds **298 lines** matching the existence-style `has_text:` pattern across the corpus (`grep "has_text:\s*$\|text: \"{\"$"` yields 298 hits across many files). It is widespread, not a HyPhy quirk. - -Other variants in the same accepted-shortcut family: - -- **First-line-of-header probes**: `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:46-49` asserts `has_text: "# mapseq v1.2.6 (Jan 20 2023)"` — version banner only. -- **`has_n_columns` schema probes**: same file lines 50-52, 67-69 — "the table has 15 columns" / "4 columns" with no row-content check. -- **`has_h5_keys` structure probes**: `scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27-32, 159-166` — confirms AnnData has `obs/louvain`, `var/highly_variable`, `uns/rank_genes_groups`, etc., but says nothing about cluster labels or values. Also `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` for Merged anndata. - -### Smell - -Existence probes on **deterministic** outputs. If the underlying tool is deterministic on fixed inputs (alignment, simple QC stats, well-defined transformations), reducing to `has_text: "{"` is laziness — agent should at least pull a known stable substring from the expected JSON. - -Heuristic for an agent: **if the source workflow's tool is on the "stochastic / floating-point heavy / version-fragile" list (HyPhy, RepeatModeler, scanpy plots, MCMC samplers, ML inference), existence probes are accepted. Otherwise, prefer `has_text` against a stable token from a real output.** - ---- - -## 2. Size-only comparisons (`compare: sim_size` + `delta:`) - -### Accepted - -Canonical example: `repeatmasking/RepeatMasking-Workflow-tests.yml:11-46` — every output is `compare: sim_size` with `delta: 30000` (30 KB) on small outputs and `delta: 90000000` (90 MB!) on the Stockholm seed-alignment file. RepeatModeler's discovered repeat families differ run-to-run; only output magnitude is reproducible. - -`grep "compare: sim_size"` returns 9 files using this pattern: -- `repeatmasking/RepeatMasking-Workflow-tests.yml`, `repeatmasking/Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` (RepeatModeler — large delta band) -- `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `epigenetics/hic-hicup-cooler/hic-fastq-to-cool-hicup-cooler-tests.yml` (HiC matrices) -- `genome_annotation/functional-annotation/functional-annotation-of-sequences/Functional_annotation_of_sequences-tests.yml` -- `VGP-assembly-v2/kmer-profiling-hifi-trio-VGP2/kmer-profiling-hifi-trio-VGP2-tests.yml` -- `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` -- `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `scRNAseq/baredsc/baredSC-2d-logNorm-tests.yml` (Bayesian sampling — uses `delta_frac:` here) - -### Delta-magnitude survey - -From `grep "delta: [0-9]+"` distribution across the corpus: - -| Delta band | Count | Typical use | -|---|---|---| -| 4–100 (tiny) | ~20 | image pixel dimensions, line counts | -| 1K–10K | ~40 | small text/tabular outputs, plot PNGs | -| 25K–100K | ~25 | mid-size reports, multi-page plots | -| 200K–1M | ~10 | report HTML, BAM stats | -| 1M–10M | ~10 | medium BAM/BCF/cool files | -| 10M+ (up to 90M) | ~7 | RepeatModeler libraries, large alignments | - -The 90 MB delta on `RepeatMasking-Workflow-tests.yml:20` is at the extreme. It says "this output is somewhere between zero bytes and 180 MB" — effectively only catches the empty-output failure mode. Accepted because RepeatModeler's seed alignments are known to vary by tens of MB across runs. - -### `delta_frac:` - -Used in 3 files (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`). Preferred over absolute `delta:` when the expected output size scales with input. An agent translating a workflow whose output size depends on input volume should consider `delta_frac:` over `delta:`. - -### Smell - -`compare: sim_size` on outputs from a **deterministic** tool. If the tool is bwa/bowtie2/samtools-sort with fixed seeds and pinned versions, there's no excuse for size-only — `compare: diff` (with modest `lines_diff:`) or content assertions are appropriate. - -Also a smell: stacking size + `has_image_*` checks on a PNG without any content assertion when the workflow's claim is *about the data shown* (e.g., a clustering plot). The corpus does this routinely (Scanpy file below) — accepted, but a translated workflow that has a more deterministic plotter should do better. +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. ---- - -## 3. Image plot assertions - -### Accepted - -`scRNAseq/scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33-205` is the dense example. ~15 PNG outputs each get the same triple: - -```yaml -UMAP of louvain: - has_size: - size: 68416 - delta: 6000 - has_image_width: - width: 601 - delta: 30 - has_image_height: - height: 429 - delta: 25 -``` - -What this catches: -- Plot was rendered (non-zero size). -- Render dimensions are stable (matplotlib defaults didn't drift, theme didn't change). -- Approximate file size hasn't shifted by an order of magnitude (no catastrophic content change like all-white or all-noise). - -What this **misses**: cluster assignments wrong, axes mislabeled, points in wrong positions, colors swapped, the wrong subset plotted, NaN handling regression. Two visually different UMAPs can have identical width/height/size-within-10%. - -Other observed image-assertion users (`grep "has_image"`): -- `imaging/tissue-microarray-analysis/tissue-microarray-analysis/tissue-micro-array-analysis-tests.yml` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - -The TMA tests use a friendlier shorthand: `has_size: size: 181K, delta: 50K` (human-readable units). - -### Smell - -Asserting `has_image_width`/`has_image_height` with **zero delta** on a tool that re-encodes (PNG round-trips through matplotlib) is brittle. The corpus uses 5–10% deltas; an agent emitting `delta: 0` should be flagged unless the renderer is byte-stable. - -Note `has_image_channels`, `has_image_center_of_mass` are documented (galaxy XSD) but **not observed** in the sampled corpus. An agent with a deterministic mask/segmentation output could use `has_image_center_of_mass` to actually verify spatial correctness — this would be an *upgrade* over the current corpus norm, not a smell. - ---- - -## 4. Happy-path-only culture (`expect_failure:`) - -`grep -r "expect_failure"` over all 115 tests files returns **zero hits**. The IWC corpus has no negative tests. Period. - -This is a structural property of IWC: the workflows are published artifacts intended to *succeed* on canonical inputs. Adversarial / error-path testing happens in tool wrappers, not in workflow tests. - -**Implication for an agent:** Do not author `expect_failure:` cases when translating a workflow. If the source pipeline (e.g., nf-core) had a "fail on bad reference" test, drop it — it doesn't belong in IWC. If the validation logic is important, it should be in a wrapper-level tool test, not a workflow test. - ---- - -## 5. `md5:` / `checksum:` rarity - -`grep -r "md5:\|checksum:"` over `*-tests.yml`: **zero hits**. - -SHA-1 `hashes:` blocks are pervasive — but exclusively on **inputs** (`hash_function: SHA-1` paired with a remote `location:`), to guard against silent corruption of the fetched fixture. Output assertions never use them. - -Accepted. The reason is empirical: outputs of real bioinformatics tools (BAM with PG headers and timestamps, VCFs with command-line provenance, JSON with run dates) are almost never byte-stable across runs. A `checksum:` would fail intermittently. - -**Smell:** an agent emitting `checksum:` or `md5:` on a workflow output. Even for "fully deterministic" tools, embedded provenance breaks checksums. Use `compare: diff` + `lines_diff:` instead, or content assertions. - ---- +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. -## 6. Output label coupling +## Keep format and contribution rules separate -Test files key outputs by **workflow label**, with spaces, capitals, punctuation preserved verbatim. Examples: - -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:27` — `Anndata with Celltype Annotation:` (spaces, mixed case) -- `scanpy-clustering/.../Preprocessing-...-Scanpy-tests.yml:33` — `UMAP of louvain and top ranked genes:` -- `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` — `Merged anndata:`, `Spatial Scatterplot Montage:` -- `consensus-from-variation-tests.yml:30` — `multisample_consensus_fasta:` (snake-case style) - -**Both styles coexist.** Snake-case (older / SARS-CoV-2 family) and natural-language-with-spaces (newer / scanpy / TMA) are equally valid. Reviewers do not enforce a single convention. - -### Coupling consequences - -Renaming an output label in the `.ga` without updating the sibling `-tests.yml` is a silent breakage: -- Test references the old label → key not present in invocation outputs → assertion mismatch surfaces as an opaque "output not found" error in planemo. -- `planemo workflow_lint --iwc` enforces that workflow outputs are *labeled*, not that test labels match. - -**Discipline observed:** every output a test asserts on is a labeled workflow output. The corpus does not assert on positional / unlabeled outputs. - -**Smell:** a translated workflow with unlabeled outputs that later need test coverage. Agent should label every output it intends to assert on, before writing assertions. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: pick stable labels before test authoring and treat renames as test-breaking API changes. - ---- - -## 7. Intermediate-step output gap - -`-tests.yml` can only assert on workflow-level **outputs** (entries in the `.ga`'s top-level outputs). Intermediate step results are inaccessible to assertions. - -Observed workaround across the corpus: **promote the intermediate to a workflow output**. This is visible indirectly — many workflows expose what would naturally be intermediates as labeled outputs solely for testability: - -- `scanpy-clustering` exposes `Initial Anndata General Info`, `Anndata with raw attribute`, `Plot highly variable`, `Elbow plot of PCs and variance` — these are mid-pipeline checkpoints surfaced specifically to be assertable. Compare counts: 22 outputs asserted vs the 7-or-so "user-meaningful" final artifacts. -- `MAGs-generation-tests.yml` exposes a `Full MultiQC Report` even though MultiQC is logically intermediate to MAG annotation. - -**Cost:** "test-only" outputs clutter the workflow's user-facing output list. Reviewers tolerate this in exchange for testability. - -**Accepted shortcut:** promoting an intermediate to a workflow output for test purposes. Not a smell. - -**Smell:** asserting on a step output via some side-channel (e.g., relying on Galaxy collection ordering, indexing into `tool_state`). The corpus does not do this and an agent should not invent it. - -Positive design guidance now lives in [[galaxy-workflow-testability-design]]: promote assertable checkpoints deliberately, especially when final reports or plots can only support weak smoke tests. - ---- - -## 8. Remote-data fragility - -### Pattern - -Overwhelming preference for **Zenodo** as the input store. Every remote `location:` is paired with a SHA-1 `hashes:` block. Examples already cited in nearly every snippet above. - -Non-Zenodo remote sources observed (from `grep "ftp.sra.ebi\|ftp://\|figshare\|github.com"`): -- `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:27,34,48,55` — `ftp://ftp.sra.ebi.ac.uk/...` SRA fastqs -- `amplicon/amplicon-mgnify/.../mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml:21` — `ftp://ftp.ebi.ac.uk/...` reference DB -- `data-fetching/parallel-accession-download/parallel-accession-download-tests.yml`, `VGP-assembly-v2/Plot-Nx-Size/...` — accession-driven -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` — github raw URLs - -### The fragility - -- **Zenodo is a single point of failure across CI.** A Zenodo outage breaks every IWC PR concurrently. SHA-1 hashes guard against silent corruption but provide no mitigation for outages or HTTP 503s. -- **EBI/SRA FTP is even less reliable** — observed flake-prone in the broader Galaxy CI history. -- No retry / backoff configured at the test-format level; planemo-ci-action's defaults handle transient failures only via re-running the chunk job manually. - -### Accepted - -This is just life in the IWC. Don't try to "fix" it in a translated workflow by inlining large data — reviewers will push back (see §10). - -### Smell - -Inputs hosted on a contributor's personal endpoint, S3 bucket, or Dropbox. Reviewers ask for migration to Zenodo before merge. - ---- - -## 9. `compare: diff` on timestamped outputs - -`compare: diff` usage from `grep`: - -- `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml:32` — `compare: diff, lines_diff: 6` on annotated VCF -- `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml:35` — same -- `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml:16` — `lines_diff: 0` -- `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml:21,25,29,33` — `lines_diff: 6` -- `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml:39,45,55` — `lines_diff: 6` - -The `lines_diff: 6` constant is suspicious — 6 lines is the typical VCF header preamble that embeds `##fileDate=...` and `##source=...`. The use is **defensible**: tolerance window matches the known mutable header lines, content lines are diffed strictly. - -### Smell - -- `compare: diff` with `lines_diff: 0` on a file that contains *any* timestamp, command-line capture, version banner, or random tie-break (hash-ordered dictionaries in Python output, etc.). The single observed `lines_diff: 0` case (`segmentation-and-counting-tests.yml:16`) appears to be on a numeric tabular output where it's defensible — verify content type before flagging. -- `compare: diff` on a BAM. BAM headers include `@PG` lines with full command lines and Galaxy job IDs. Use `has_size` + content extracts via `has_archive_member` or `samtools view`-piped XML asserts — not byte-level diff. - -**Recommended replacement** when timestamps appear: -- For VCF: `compare: diff, lines_diff: ` (corpus convention is 6). -- For tabular reports: `has_text` against stable column headers + `has_n_columns`. -- For HTML reports (MultiQC etc.): `has_text` substring on stable section names — example `short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:25-28` asserts `"Filtered Reads"` substring on the MultiQC HTML report rather than diffing. - ---- - -## 10. Reviewer-feedback recurring asks - -Synthesized from the COMPONENT_GALAXY_WORKFLOW_TESTING.md analysis (sections 9 and the "Common PR-review feedback" subsection) plus structural observation of what every accepted workflow has and rejected drafts apparently lack: - -| Reviewer ask | Where it bites | Source | -|---|---|---| -| **Creator `identifier:` must be a full ORCID URI** (`https://orcid.org/...`), not bare ID. | `.ga` frontmatter `creator:` block. Most common lint failure. | planemo PR #1458; `consensus-from-variation.ga:4-10` shows the conformant shape. | -| **Move large inputs to Zenodo.** Inline `path: test-data/big.bam` for >1 MB inputs gets pushback. | `-tests.yml` job inputs. | `iwc/workflows/README.md` (per prior analysis). | -| **Bump `release` + add CHANGELOG.md entry in the same PR.** | `.ga` `release:` and sibling `CHANGELOG.md`. Enforced by `bump_version.py`. | `iwc/workflows/README.md:217-247`. | -| **Generate tests via `planemo workflow_test_init --from_invocation `**, not by hand. Reviewers push back on hand-authored job blocks. | Any new test contribution. | help.galaxyproject.org thread 13903. | -| **Don't use `compare: diff` on outputs that embed timestamps.** Switch to `has_text`/`has_n_lines` with `delta:`. | See §9. | Recurring review comment. | -| **Add labeled outputs for any output you assert on.** Unlabeled outputs caught by `planemo workflow_lint --iwc`. | `.ga` outputs. | §6 above. | -| **Hashes on every remote `location:`.** SHA-1 block paired with the URL. Reviewers spot-check. | `-tests.yml` job inputs. | Universal in the corpus; missing hashes get flagged. | - -### Smell to flag for an agent submission - -- Bare ORCID ID (`0000-0002-...`) in `creator:` instead of full URL. -- Test job referencing >1 MB local fixture instead of a Zenodo URL. -- PR that bumps a workflow without CHANGELOG / `release:` bump. -- Hand-authored `-tests.yml` that reads "too clean" — reviewers know `--from_invocation` output has a recognizable fingerprint. - ---- - -## Summary cheatsheet for the implement-galaxy-workflow-test mold - -**Use these freely (accepted shortcuts):** -- `has_text: text: "{"` for stochastic JSON outputs. -- `compare: sim_size, delta:` for non-deterministic file outputs; pick delta from §2 distribution by tool family. -- `has_image_width/height/has_size` triple with 5–10% delta for matplotlib plots. -- `has_h5_keys` for AnnData/HDF5 — assert structure not values. -- Promoting intermediates to workflow outputs to make them assertable. -- Labels with spaces, mixed case, punctuation as the output key. -- SHA-1 hashes on every input `location:`. - -**Avoid (smells reviewers or future-you will catch):** -- `expect_failure:` — not an IWC pattern. -- `md5:` / `checksum:` on outputs. -- `compare: diff, lines_diff: 0` on anything containing timestamps, BAM `@PG` lines, or Python dict ordering. -- `has_image_*` with zero delta. -- Existence-only `has_text: "{"` on outputs from a deterministic tool. -- Asserting on positional/unlabeled outputs. -- Inlining >1 MB binary fixtures in `test-data/`. -- Bare ORCID identifier in `.ga` frontmatter. - -**Review-time checklist before submission:** -1. Every output asserted on has a label in the `.ga`. -2. Every remote `location:` has a `hashes: SHA-1` block. -3. Inputs >1 MB live on Zenodo, not in `test-data/`. -4. `release:` bumped and `CHANGELOG.md` updated if `.ga` changed. -5. `creator.identifier:` is a full `https://orcid.org/...` URL. -6. Test was generated by `planemo workflow_test_init --from_invocation`, not hand-written. - ---- +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. -## Sources +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. -- Positive workflow-structure guidance: [[galaxy-workflow-testability-design]]. -- Prior synthesis: `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` (sections 2c, 2d, 2e, 9). -- Corpus root: `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (115 `*-tests.yml` files across 22 categories). -- Specific files cited: - - `comparative_genomics/hyphy/hyphy-core-tests.yml`, `hyphy-compare-tests.yml`, `capheine-core-and-compare-tests.yml` - - `repeatmasking/RepeatMasking-Workflow-tests.yml`, `Repeat-masking-with-RepeatModeler-and-RepeatMasker-tests.yml` - - `scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml` - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml` - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml` - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-tests.yml` - - `sars-cov-2-variant-calling/sars-cov-2-ont-artic-variant-calling/ont-artic-variation-tests.yml` - - `variant-calling/variation-reporting/Generic-variation-analysis-reporting-tests.yml` - - `variant-calling/generic-variant-calling-wgs-pe/Generic-variation-analysis-on-WGS-PE-data-tests.yml` - - `imaging/fluorescence-nuclei-segmentation-and-counting/segmentation-and-counting-tests.yml` - - `imaging/tissue-microarray-analysis/multiplex-tissue-microarray-analysis/multiplex-tma-tests.yml` - - `amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-rrna-prediction/mgnify-amplicon-pipeline-v5-rrna-prediction-tests.yml` - - `epigenetics/atacseq/atacseq-tests.yml` - - `epigenetics/hic-hicup-cooler/chic-fastq-to-cool-hicup-cooler-tests.yml`, `hic-fastq-to-cool-hicup-cooler-tests.yml` - - `microbiome/mags-building/MAGs-generation-tests.yml` - - `genome-assembly/polish-with-long-reads/Assembly-polishing-with-long-reads-tests.yml` - - `scRNAseq/baredsc/baredSC-1d-logNorm-tests.yml`, `baredSC-2d-logNorm-tests.yml` -- Corpus-wide grep tallies: - - `expect_failure`: 0 hits. - - `md5:|checksum:` on outputs: 0 hits. - - `compare: sim_size`: 9 files. - - `compare: diff`: 5 files (variant-calling and imaging). - - `has_image_*`: 3 files. - - Existence-style `has_text:` patterns: ~298 line matches. - - `delta_frac:`: 3 files. +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-test-data-conventions.md index 5f60727..842a914 100644 --- a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-test-data-conventions.md +++ b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/iwc-test-data-conventions.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-03 -revision: 3 +revised: 2026-10-21 +revision: 5 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-shortcuts-anti-patterns]]" @@ -13,296 +13,132 @@ related_notes: - "[[implement-galaxy-workflow-test]]" - "[[tests-format]]" - "[[iwc-tabular-operations-survey]]" -summary: "How IWC workflows organize and reference test data — Zenodo-first, SHA-1 integrity, collection shapes, CVMFS gotchas." +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." --- # IWC test data conventions -Reference for an agent implementing or editing a `-tests.yml` in IWC style. All evidence cited from `/Users/jxc755/projects/repositories/workflow-fixtures/iwc-src/workflows/` (raw IWC clone) and `workflows/README.md`. Authoritative spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). The companion analysis at `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` is the synthesized source for several normative claims here. +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. -This note owns **test YAML fixture shapes**. For workflow-structure choices that make those fixtures possible before the test file exists, use [[galaxy-workflow-testability-design]]. +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. -## 1. Where does test data live? Remote vs in-repo +## Fixture locations and provenance -Two storage patterns. They mix freely inside one job. +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). -**Remote `location:` (default for any non-trivial input).** Strongly preferred for anything bigger than a toy fixture. Order of preference, observed in the corpus: +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | -- **Zenodo** — overwhelming default, persistent DOI-backed URL. - - `read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:13,17` — `https://zenodo.org/records/11484215/files/paired_r1.fastq.gz` (note the modern `/records/` plural form; older entries use `/record/`). - - `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:6,17,24,...` — 11+ mzML files all hosted at `zenodo.org/record/10130758/files/...`. - - `sars-cov-2-variant-calling/sars-cov-2-consensus-from-variation/consensus-from-variation-tests.yml:5` — `https://zenodo.org/record/4555735/files/NC_045512.2_reference.fasta?download=1`. - - `repeatmasking/RepeatMasking-Workflow-tests.yml:5` — `https://zenodo.org/record/8364146/files/eco.fasta?download=1`. The `?download=1` query suffix is acceptable but optional; both forms appear. -- **EBI/ENA REST** — for canonical reference sequences. - - `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:5` — `https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true`. -- **SRA / ENA FTP** — for raw read accessions where re-uploading to Zenodo is wasteful. - - `pox-virus-half-genome-tests.yml:27,34,49,56` — `ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR151/076/SRR15145276/SRR15145276_1.fastq.gz`. - -**In-repo `path: test-data/...` (toy fixtures + small expected outputs only).** Used when: - -- File is small enough to commit (rule of thumb from corpus: low single-digit MB or smaller). -- File is a hand-crafted toy or trimmed reference (e.g. a 5-element FASTA for HyPhy). -- File is the *expected output* baseline for a `file:` exact-comparison assertion: `consensus-from-variation-tests.yml:31` — `file: test-data/masked_consensus.fa`. - -`workflows/README.md:67` makes the contract explicit: "try to generate a toy dataset … and then publish it to zenodo to have a permanent URL." `workflows/README.md:98`: "if some outputs are large, it is better to use assertions than storing the whole output file to the iwc repository." - -Subdirectory layout inside `test-data/` is free-form and used to organize related fixtures, e.g. `comparative_genomics/hyphy/test-data/unaligned_seqs/`, `test-data/codon_alignments/`, `test-data/iqtree_trees/` (referenced from `hyphy-core-tests.yml:13,17,21,...`). - -**Decision rule for an agent:** - -1. Expected-output baseline that's small — commit to `test-data/`. -2. Toy/trimmed input < ~5 MB — commit to `test-data/`. -3. Anything else — Zenodo (or EBI/SRA for canonical accessions). Add `hashes:` SHA-1. -4. If the input is a reference index resolvable from CVMFS (`hg38`, `dm6`, `mm10`, …), use the plain string form. See section 5. - -## 2. The `class: File` block — exact YAML shapes - -All snippets verbatim from the IWC corpus. - -### 2a. Single remote file (with integrity hash) - -`virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:3-9`: - -```yaml -Reference FASTA: - class: File - location: "https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true" - filetype: fasta - hashes: - - hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf -``` - -Field order is conventional but not enforced. `filetype:` is the Galaxy datatype name (`fasta`, `fastqsanger.gz`, `mzml`, `bed`, `tabular`, `csv`, `gtf`, `vcf`, …). `location:` URLs may be quoted or bare; both forms occur. - -### 2b. Single local file - -`comparative_genomics/hyphy/hyphy-core-tests.yml:3-6`: +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. ```yaml -reference cds: - class: File - path: test-data/denv1_ref_cds.fasta - filetype: fasta +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker ``` -`path:` is relative to the workflow directory (the directory containing the `-tests.yml`). Hashes are usually omitted on local files — the file is in-tree, so integrity is implicit. They are nonetheless added in some workflows: `consensus-from-variation-tests.yml:15-18` shows `path: test-data/aligned_reads_for_coverage.bam` paired with a SHA-1 hash. The pattern appears to be: hashes added when the `-tests.yml` was generated by `planemo workflow_test_init --from_invocation`, which mints them automatically. - -A dataset attribute can be set on a single file to force a `dbkey`, e.g. `epigenetics/consensus-peaks/consensus-peaks-chip-pe-tests.yml:7-10`: - -```yaml -- class: File - identifier: rep1 - path: test-data/rep1.bam - dbkey: mm10 -``` +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. -`dbkey:` and `decompress: true` (`scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:30`) are documented per-file attributes that ride alongside `class: File`. +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. -### 2c. List collection +## Collection inputs -`hyphy-core-tests.yml:7-30`: +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. ```yaml -unaligned sequences: - class: Collection - collection_type: list - elements: - - identifier: AB178040.1|2002 - class: File - path: test-data/unaligned_seqs/AB178040.1|2002.fasta - filetype: fasta - - identifier: AB195673.1|2003 - class: File - path: test-data/unaligned_seqs/AB195673.1|2003.fasta - filetype: fasta - ... -``` - -Each element is a single-file dict with an `identifier:`. A larger remote-only example with hashes per element: `metabolomics/lcms-preprocessing/Mass_spectrometry__LC-MS_preprocessing_with_XCMS-tests.yml:11-22`: - -```yaml -Mass-spectrometry Dataset Collection: - class: Collection - collection_type: list - elements: - - class: File - identifier: Blanc05.mzML - location: https://zenodo.org/record/10130758/files/Blanc05.mzML - filetype: mzml - hashes: - - hash_function: SHA-1 - hash_value: e98d5a4cd8849a145a032bdc31c348ad97cb59c3 +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz ``` -### 2d. Paired collection (single pair) - -A `paired` collection is a Collection whose two elements have identifiers `forward` and `reverse`. In IWC it is rarely used at top level — it is almost always wrapped as a single-element `list:paired` (see 2e). When it appears bare, the shape is the inner block of 2e without the outer `list:paired` wrapper. - -### 2e. `list:paired` collection - -`read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml:3-18`: +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: ```yaml -Raw reads: - class: Collection - collection_type: list:paired - elements: - - class: Collection - type: paired - identifier: pair +job: + Raw reads: + class: Collection + collection_type: list:paired elements: - - class: File - identifier: forward - location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz - filetype: fastqsanger.gz - - class: File - identifier: reverse - location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz - filetype: fastqsanger.gz + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz ``` -Note: outer `collection_type: list:paired`, inner `type: paired` (not `collection_type:`). Inner element identifiers are the literal strings `forward` and `reverse` — these are reserved. +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. -A multi-pair list:paired with full hashes: `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml:17-38` or `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:3-24`. +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. -### 2f. `list:list:paired` and deeper +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. -Not directly observed in sampled IWC inputs. The shape is recursive — wrap a `list:paired` element list inside another `class: Collection, collection_type: list:list:paired`, where each outer element is `class: Collection, type: list:paired` (analogous to the `list:paired` → `paired` nesting in 2e). For deeply-nested collections planemo's [`_writing_collections.rst`](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst) is the de-facto reference. +## Other documented input forms -Output side: nested-collection assertions DO exist in the corpus. `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-98`: +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. ```yaml -Seurat input for gene expression (filtered): - attributes: {collection_type: list:list} - element_tests: - subsample: - elements: - matrix: - asserts: - has_line: - line: "23932 3 1171" - barcodes: - asserts: - has_line: - line: "CACATGATCATAAGGA" +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd ``` -Pattern: outer `element_tests:` keyed by outer identifier; inner `elements:` (note plural, not `element_tests:` here) keyed by inner identifier. Optional `attributes: {collection_type: ...}` asserts the shape of the produced collection. - -### 2g. `composite_data:` - -**Not observed** in the sampled IWC corpus (`grep -rln "composite\|imzml\|primary_file\|extra_files" workflows/ --include="*-tests.yml"` returns nothing). Per the planemo spec the shape is a `composite_data:` mapping listing each component file path, but an agent emitting one in IWC style has no in-corpus reference and should treat it as unproven territory — fall back to the planemo docs and verify against `galaxy.tool_util.parser.yaml`. - -### 2h. CWL-style shorthand (list of File dicts without identifiers) +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. -Documented in planemo; not observed in sampled IWC. IWC always supplies explicit identifiers. +## Data-table values and built-in indexes -### 2i. `tags:` on elements (Galaxy 20.09+) - -Documented in planemo; not surfaced in sampled IWC `-tests.yml` files (`grep " tags:" --include="*-tests.yml"` empty across the sampled set). Practical absence — IWC tests rely on identifier matching, not tags. - -## 3. SHA-1 integrity hashes — when, when not, why - -**Format.** Always a list under `hashes:`, even for one entry. Each entry is `{hash_function: SHA-1, hash_value: <40-hex>}`. Verbatim from `pox-virus-half-genome-tests.yml:7-9`: +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: ```yaml -hashes: -- hash_function: SHA-1 - hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf +job: + reference genome: hg38 ``` -Only `SHA-1` observed in the IWC corpus. The planemo spec also accepts `MD5`, `SHA-256`, `SHA-512` as `hash_function:` values; IWC does not exercise these. - -**When present:** - -- Every remote `location:`-fetched input in the sampled corpus carries a SHA-1 (Zenodo, EBI REST, SRA FTP). The hash guards against silent upstream corruption — Zenodo records are immutable but bit-rot and CDN-edge errors do happen; ENA FASTA endpoints can quietly change line-wrap. -- Many local `path:`-loaded inputs also carry a SHA-1. This is `--from_invocation` autogeneration leaking through (planemo emits hashes for every input it captured). -- `consensus-from-variation-tests.yml:6-8,17-18,27-28` — every input in the test, local and remote, has a hash. - -**When omitted:** - -- Hand-written local-file fixtures often skip hashes — `hyphy-core-tests.yml:3-30` has zero `hashes:` blocks across six local-file inputs. -- Output assertions: `file:`-comparison outputs and `location:`-comparison outputs (`RepeatMasking-Workflow-tests.yml:13`) **do not** carry `hashes:`. The integrity story for outputs is handled by `compare:` / `asserts:` instead. - -**Why bother on local files.** Catches accidental corruption when collaborators hand-edit `test-data/` fixtures. - -## 4. Element identifier conventions - -Identifiers are arbitrary strings, used both as the YAML key (`element_tests:`) and as the produced Galaxy collection element name. Conventions surfaced from the corpus: - -- **Pipes are legal** and survive YAML round-trip without quoting on the input side: `hyphy-core-tests.yml:11` — `identifier: AB178040.1|2002`. On the *output* side they're quoted because they appear as YAML mapping keys: `hyphy-core-tests.yml:34` — `"NC_001477.1|capsid_protein_C|95-394_DENV1":`. Quote when the identifier contains characters that would otherwise start a YAML flow node (`{`, `[`, `&`, `*`, `!`, `|`, `>`, `'`, `"`, `%`, `@`, `` ` ``) or look like a number/bool. -- **Dots, hyphens, underscores** — common, no quoting needed: `Blanc05.mzML`, `HU_neg_048.mzML`, `SRR11578257`, `Rep1`, `subsample`, `pair`. -- **Reserved inside `paired` collections:** `forward` and `reverse` (lowercase). Never use these names for outer identifiers in `list:paired`. -- **Inline comments after identifier** are legal and used for provenance: `pox-virus-half-genome-tests.yml:23` — `identifier: 20L70 # SRR15145276`. -- **Whitespace and special chars in workflow input *labels*** survive too — they are job-mapping keys, not collection identifiers, but the same YAML quoting rules apply. Example from the analysis doc: `Manually annotate celltypes?: true` as a job key in `Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:23`. - -### 4a. Workflow-interface implications - -Fixture shape should feed back into workflow input design. Tests supply job inputs by workflow input label, and collection fixtures must match the workflow's declared collection type. - -- The 10x CellPlex test uses `fastq PE collection GEX`, `fastq PE collection CMO`, and `sample name and CMO sequence collection` as job keys with `list:paired` and `list` collection fixtures; the workflow declares matching collection inputs at `$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`. -- CVMFS/data-table shortcuts like `reference genome: dm6` are workflow-interface choices as much as test YAML choices: they require the workflow input to be a string/data-table-backed parameter, not a file input. -- If a fixture needs stable sample names, decide whether those names should enter through collection element identifiers, a mapping file, or a workflow parameter before writing assertions. - -Rule for an agent: when a new workflow input is being designed and the test fixture is already known, check [[galaxy-workflow-testability-design]] before locking the input label/type. - -## 5. CVMFS / `.loc` / built-in-index references - -When a workflow input is a Galaxy data-table-backed reference (built-in genome index, dbsnp file, kraken DB, …), the input takes a **plain string** matching the `value` column of a `.loc` file. No `class: File`, no `location:`. Examples: - -- `scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:25` — `reference genome: dm6 # To test on usegalaxy.eu dm6full` -- `epigenetics/cutandrun/cutandrun-tests.yml:27` — `reference_genome: 'hg38canon'` -- `epigenetics/atacseq/atacseq-tests.yml:25` — `reference_genome: hg19` -- `microbiome/host-contamination-removal/host-contamination-removal-short-reads/host-or-contamination-removal-on-short-reads-tests.yml:19` — `Host/Contaminant Reference Genome: hg38` -- `microbiome/pathogen-identification/nanopore-pre-processing/Nanopore-Pre-Processing-tests.yml:22` — `host_reference_genome: apiMel3` -- `transcriptomics/goseq/goseq-go-kegg-enrichment-analsis-tests.yml:24` — `Select genome to use: mm10` - -The string is the first column of the `.loc` entry. `workflows/README.md:169-182` shows the lookup procedure: browse `http://datacache.galaxyproject.org/indexes/location/
.loc`, find the `.loc` row, take the value column (which column varies by tool — check the matching `tool_data_table_conf.xml.sample`). - -**Portability implication:** these tests **only** run on a Galaxy with CVMFS mounted at `/cvmfs/data.galaxyproject.org`. IWC CI mounts CVMFS in-runner (`setup-cvmfs: true` in `.github/workflows/test_workflows.yml`). On a plain developer laptop running `planemo test`, CVMFS-string inputs fail unresolved. An agent generating tests should either: - -1. Stay URL-driven: replace the CVMFS string with a `class: File` + `location:` to a Zenodo-hosted reference (portable, slower CI, larger inputs). -2. Use the CVMFS string and document "CI-only" — the user cannot reproduce locally without a CVMFS mount. - -There is no `.loc`-resolution shorthand in the YAML — it is just a bare string parameter. Agents must not invent `class: DataTable` or similar. - -## 6. Reviewer-feedback patterns (PR-review) - -Recurring asks from IWC reviewers, distilled from the contribution README and community help threads: - -- **Move large data to Zenodo before merge.** Anything bigger than a toy fixture committed in `test-data/` will be flagged. `workflows/README.md:67`. -- **Use assertions, not exact-file comparison, for large outputs.** `workflows/README.md:98` — "if some outputs are large, it is better to use assertions than storing the whole output file." -- **Generate tests with `planemo workflow_test_init --from_invocation`, don't hand-write.** `workflows/README.md:88-94`. The autogenerated test will mint correct hashes, correct labels, and a working `test-data/` snapshot. -- **Set creator `identifier:` to a full ORCID URL** (e.g. `https://orcid.org/0000-0002-1825-0097`) — `workflows/README.md:53`. The most common lint failure; enforced in [planemo#1458](https://github.com/galaxyproject/planemo/pull/1458). -- **Don't use `compare: diff` on outputs that embed timestamps or non-deterministic ordering.** Switch to `has_text` / `has_n_lines` (with `delta:`) or `compare: sim_size`+`delta:`. -- **Bump `release:` in the `.ga` and add a `CHANGELOG.md` entry** in the same PR — enforced by the IWC PR template; reviewers verify via `bump_version.py`. -- **Lower-case + `-` only** in directory names — `workflows/README.md:11,72`. -- **Element identifiers must round-trip** — quote in YAML when they contain pipes/colons/whitespace. +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. -## 7. What is *not* covered by current convention +## Authoring and checking an IWC test -Gaps an agent should be aware of: +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). -- **`composite_data:` is unrepresented in the IWC corpus.** No working examples to imitate for imzml+ibd or other multi-file datatypes. Treat as off-trail; cite planemo docs and verify by parser. -- **`tags:` on elements unused.** Galaxy 20.09+ supports them; IWC does not exercise them. -- **CWL-style shorthand File lists unused.** IWC always names elements. -- **No negative tests.** Zero `expect_failure:` across the sampled corpus. There is no IWC convention for asserting a workflow *should* fail on bad input. -- **No checksum-based output assertion in the wild.** `checksum: "sha1$..."` is documented and used at the input side as `hashes:`, but output-side `checksum:` was not surfaced in sampled tests. The corpus prefers `file:` (exact), `compare: sim_size`+`delta:` (size only), or `asserts:` (content probes). -- **No data-cache layer beyond pip/planemo.** Every test re-pulls remote inputs from Zenodo / EBI / SRA on every CI run. A Zenodo outage breaks many PRs simultaneously. There is no IWC-side mirror or cache. -- **No per-test timeout / retry convention.** A hung tool blocks the whole chunk until the GitHub Actions 6-hour limit. -- **No portable story for CVMFS-only tests.** The same `-tests.yml` either works locally (URL-driven) or works in CI (CVMFS-string-driven), not both. There is no documented switch. -- **Hash function is de-facto SHA-1.** The spec accepts SHA-256/SHA-512/MD5; IWC does not use them. Agents that emit a stronger hash will pass planemo but break corpus convention. -- **Local-file `hashes:` are inconsistently present.** `--from_invocation`-generated tests have them, hand-written tests usually don't. Either is accepted; don't expect uniformity. -- **`decompress: true` and `dbkey:` are file-level attributes** but undocumented at the planemo test_format page in the same place as `class:`/`location:`/`hashes:`. Cross-reference against the `gxformat2` Schema-Salad bindings if unsure. -- **`?download=1` query string on Zenodo URLs** is optional. Both forms appear (`/record/4555735/files/X?download=1` vs `/records/11484215/files/Y`). The newer `/records/` plural is the modern Zenodo URL form; both work. +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. -## Cross-references +## Related references -- [[galaxy-workflow-testability-design]] — workflow input/output structure choices that make these fixture shapes testable. -- `/Users/jxc755/projects/repositories/galaxy-brain/vault/projects/workflow_state/skills/COMPONENT_GALAXY_WORKFLOW_TESTING.md` — full synthesis with assertion-vocabulary table, CI internals, and tooling rundown. Sections 2b, 3, 4, 5, 9 are the direct upstream of this note. -- planemo test format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- planemo best practices: [planemo.readthedocs.io/en/latest/best_practices_workflows.html](https://planemo.readthedocs.io/en/latest/best_practices_workflows.html). -- planemo collections reference: [github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst](https://github.com/galaxyproject/planemo/blob/master/docs/_writing_collections.rst). -- IWC contribution contract: `workflows/README.md` in [github.com/galaxyproject/iwc](https://github.com/galaxyproject/iwc). -- Galaxy datacache (CVMFS `.loc` browsing): [datacache.galaxyproject.org/indexes/location/](http://datacache.galaxyproject.org/indexes/location/). +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/open-requirements-ledger.md index 49a9ff4..761183e 100644 --- a/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/open-requirements-ledger.md +++ b/plugins/foundry-skills/skills/review-galaxy-workflow/references/notes/open-requirements-ledger.md @@ -5,20 +5,22 @@ tags: - target/galaxy status: draft created: 2026-06-16 -revised: 2026-08-29 -revision: 3 +revised: 2026-11-10 +revision: 7 related_notes: - "[[galaxy-workflow-draft-format]]" related_molds: - "[[advance-galaxy-draft-step]]" - "[[repair-galaxy-draft-topology]]" - "[[implement-galaxy-tool-step]]" -summary: "Carried unresolved-requirements artifact the source→Galaxy pipeline discharges or explicitly surrenders, autonomously." +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." --- # Open-requirements ledger -The `open-requirements-ledger` is a single artifact threaded through the source→Galaxy pipeline that records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. Each Mold that surfaces one **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. ## Framing: obligations the pipeline discharges, not questions a human answers @@ -34,7 +36,7 @@ Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries ar - id: sccmec-evidence-missing status: open # open | resolved | surrendered kind: gap # gap | dropped — see below; omit for gap - blocking: true # true only for a computability gap; the convergence gate counts these + blocking: true # true only when a structural edit closes it; the convergence gate counts these raised_by: implement-galaxy-tool-step # Mold that appended it step: classify_context # draft step the obligation attaches to (if any) unmet: "SCCmec-region candidate output category" @@ -84,17 +86,72 @@ finding the run may reason from afterwards. It is a debt, and it should read as Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. -`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. ## How to use the ledger This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. -**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries. Ones bearing on those decisions are the ones you may be able to close. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. **Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. -**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only for a computability gap. +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. **Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. @@ -120,7 +177,7 @@ surrendered gap whose stated reason is known to be false is worse than an unexpl Reopening does not oblige *you* to do the dropped work — it obliges the run to stop claiming a settled reason it no longer has. -**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either — no producer is discoverable, the output cannot be honestly narrowed — leave it `open` and say so in `note`. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. **Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. @@ -128,21 +185,22 @@ claiming a settled reason it no longer has. In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): -- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs. Rather than fabricate, it appends a blocking entry and falls through to repair. -- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, re-wires the affected region (template-tier authoring — one step or a small sub-path), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes the new steps. -- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair escalation must strictly reduce that count, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. So the ledger is not a convenience here; it is the countable state the termination guard depends on. ## Escalation budget (loop-level state) -The decreasing-blocker invariant needs more than the entries themselves: "strictly reduce the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: ```yaml topology_repair: escalations: 2 # repairs invoked this run cap: 5 # hard ceiling; at cap, still-open entries are surrendered - open_history: [4, 3, 2] # open blocking-entry count after each escalation — must be strictly decreasing + open_history: [4, 3, 2] # open blocking-entry count after each escalation entries: - id: sccmec-evidence-missing status: open @@ -150,9 +208,9 @@ entries: # … per-entry shape above ``` -- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator increments it on each escalation. +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. - `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. -- `open_history` — the open blocking-entry count recorded after each escalation. The convergence gate requires it to be strictly decreasing: a repair that fails to lower the open count — or raises it by inserting a producer that is itself uncomputable — is non-convergent and trips the gate immediately, without waiting for the cap. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. @@ -167,7 +225,7 @@ The template Mold's computability review pass is the notable appender ahead of t - Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. - Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. - Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. -- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. Every design Mold still emits both, with no rule for which destination an unresolved choice belongs in. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. - Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. - Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. - Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/run-workflow-test/SKILL.md b/plugins/foundry-skills/skills/run-workflow-test/SKILL.md index 2fa1148..8b09628 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/SKILL.md +++ b/plugins/foundry-skills/skills/run-workflow-test/SKILL.md @@ -21,12 +21,12 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli -- **`planemo`** (planemo). `uv tool install planemo==0.75.47` (or `pip install planemo==0.75.47`). - Ephemeral run: `uvx --from planemo==0.75.47 planemo`. +- **`planemo`** (planemo). `uv tool install planemo==0.75.48` (or `pip install planemo==0.75.48`). + Ephemeral run: `uvx --from planemo==0.75.48 planemo`. Check: `planemo --version`. Docs: https://planemo.readthedocs.io/ Bundled reference: `references/cli/planemo.md`. @@ -53,12 +53,12 @@ Execute an assembled workflow's test file via planemo and emit a structured pass ### Sequence -0. **Establish the test definition.** Locate the workflow's test file before anything else. If none can be found, do not abort: emit `workflow-test-result.json` with `status: test-definition-missing`, the paths searched, and the workflow it was searched for, then stop. Emit `status: not-run` plus a `not_run_reason` instead when a test exists but cannot be executed — a static validate-tests failure, missing test data, an unavailable Galaxy, an uninstallable tool, a timeout, or a checkout the caller has not confirmed as trusted. A downstream consumer needs to know that nothing ran; it must never have to infer it from a missing file. +0. **Establish the test definition.** Locate the workflow's test file before anything else. If none can be found, do not abort: emit `workflow-test-result.json` with `status: test-definition-missing`, the paths searched, and the workflow it was searched for, then stop. Emit `status: not-run` plus a `not_run_reason` instead when a test exists but cannot be executed — a static validate-tests failure, missing test data, an unavailable Galaxy, an uninstallable tool, a timeout, or a checkout the caller has not confirmed as trusted. An environment that prevented the attempt is `not-run`, including an import refused for a missing permission or an uninstallable tool. A workflow Galaxy refused for its own content or through a server defect is `fail` (step 5). A downstream consumer needs to know that nothing ran; it must never have to infer it from a missing file. 1. **Validate before running.** When a test file is present, run validate-tests for the static schema and workflow-label checks first. A run is expensive; do not spend one on a test file that fails static validation. 2. **Pick the Galaxy mode.** Run against a Planemo-managed Galaxy or an existing/external Galaxy. **Planemo-managed** is the default and needs no pre-provisioned server: `planemo test` bootstraps its own Galaxy and installs the workflow's tools from the Tool Shed/conda — so the absence of a running Galaxy is not a reason to skip the run. Use an existing/external Galaxy only when you deliberately want to target one (shared instance, pre-installed heavy tools/reference data, specific credentials). The real cost of the managed path is install/runtime weight (large tools or multi-GB reference databases), which is a deliberate deferral, not an impossibility. Record which mode was used, how tools, workflows, and test data were staged, and the URLs or API credentials a follow-up inspection would need. The choice and its consequences are guided by planemo-workflow-test-architecture. 3. **Do not pin a Galaxy version.** Leave `--galaxy_branch` off the command unless the user or the harness supplied a specific branch, and never guess a `release_*` value — Planemo's default targets the newest Galaxy, which is the only version guaranteed to understand every construct a freshly authored workflow can contain. If a branch was supplied, record it in the output so a later failure can be read against it. -4. **Run and capture.** Drive `planemo test` with structured output enabled. Preserve the invocation id, history id, workflow id, the Planemo structured result, and any test-output artifact paths — these are the inputs the debug skill consumes. -5. **Classify on failure.** When the run exits non-zero or reports failed assertions, failed jobs, a failed invocation, missing outputs, or upload/staging problems, identify the observed failure modality and the single next reference surface to open: Planemo result, Galaxy job API, Galaxy invocation API, history contents, or the test assertion report. Use planemo-asserts-idioms to read assertion failures and galaxy-workflow-invocation-failure-reference to preserve invocation identifiers and state. +4. **Run and capture.** Drive `planemo test` with structured output enabled. Preserve the invocation id, history id, workflow id, the Planemo structured result, and any test-output artifact paths — these are the inputs the debug skill consumes. When Galaxy rejects the workflow import, none of these exist: Planemo stops before writing structured output (planemo-workflow-test-architecture). Record the ids as absent and preserve the Planemo terminal log, the Galaxy server log, and the import request's HTTP status and response body instead. +5. **Classify on failure.** When the run exits non-zero or reports failed assertions, failed jobs, a failed invocation, missing outputs, or upload/staging problems, identify the observed failure modality and the single next reference surface to open: Planemo result, Galaxy job API, Galaxy invocation API, history contents, the test assertion report, the workflow import response, or the Galaxy server log. A workflow that Galaxy refused to import for its content or through a server defect is modality `workflow-import-rejected` with `status: fail`, not `not-run`—Galaxy evaluated the workflow itself, so the response and the server log are runtime evidence about it. Name the import response as the next surface for a 4xx that carries a message, and the server log for a 500. Use planemo-asserts-idioms to read assertion failures and galaxy-workflow-invocation-failure-reference to preserve invocation identifiers and state. 6. **Hand off.** Emit the structured summary carrying a `status` from the closed set `pass | fail | test-definition-missing | not-run` — green, or red with modality + captured artifacts + the named next surface — for debug-galaxy-workflow-output. A `test-definition-missing` or `not-run` result is never reported as a pass and never as a Planemo failure: neither carries a failure modality or a next reference surface, because nothing ran. A consumer must read those two states as *no runtime evidence*, not as a debug target. Keep this skill's output a faithful record of what happened, not a diagnosis. Mislabeling a staging failure as an assertion failure here sends the debug pass to the wrong reference surface. diff --git a/plugins/foundry-skills/skills/run-workflow-test/_provenance.json b/plugins/foundry-skills/skills/run-workflow-test/_provenance.json index ebb7e7f..d81cbc9 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/_provenance.json +++ b/plugins/foundry-skills/skills/run-workflow-test/_provenance.json @@ -4,11 +4,11 @@ "mold": { "name": "run-workflow-test", "path": "content/molds/run-workflow-test/index.md", - "revision": 7, - "content_hash": "259675e41c65f7622049cc9ef5f64b7d9f7ac5edb005555a09045e747708218c", - "commit": "79bf5c3ab98eec3a67948d8bdebb9dba7d8a2359" + "revision": 8, + "content_hash": "6e312c6ee27ef6e45a6ace74087cd28a6509ade11f788a51ee478efce58971b6", + "commit": "4c747d65e84f0100611458b5e869e9641bc50a21" }, - "cast_at": "2026-09-12T19:19:46.957Z", + "cast_at": "2026-11-09T16:55:03.332Z", "cast_date": "2026-07-20", "cast_revision": 1, "cast_history": [ @@ -44,8 +44,8 @@ "load": "upfront", "evidence": "corpus-observed", "purpose": "Runtime that executes the Galaxy or CWL workflow test; install before driving the harness.", - "src_hash": "12e8cac3ffb4ab7f007ced7a8d4f01e6b3785d06a55926496bbacc0a8f3fc5c5", - "dst_hash": "12e8cac3ffb4ab7f007ced7a8d4f01e6b3785d06a55926496bbacc0a8f3fc5c5", + "src_hash": "7a149ff6f1cc396dd72bbdd7d0b93acecb12e1b86663c6119d4c0815b411fe94", + "dst_hash": "7a149ff6f1cc396dd72bbdd7d0b93acecb12e1b86663c6119d4c0815b411fe94", "source": "deterministic" }, { @@ -59,8 +59,8 @@ "evidence": "corpus-observed", "purpose": "Preserve invocation identifiers and state summaries needed to inspect workflow-level runtime failures after Planemo returns.", "trigger": "When Planemo reports a failed, cancelled, missing-output, or ambiguous workflow invocation result.", - "src_hash": "e8115df6ddca447d47d77cb33becda847cdd19b2cbe9924782bfc180b94bb451", - "dst_hash": "e8115df6ddca447d47d77cb33becda847cdd19b2cbe9924782bfc180b94bb451", + "src_hash": "e86b17c0b39b88076f1a1f177cce9ef1f868de0b36437a624e4b630e2c126559", + "dst_hash": "e86b17c0b39b88076f1a1f177cce9ef1f868de0b36437a624e4b630e2c126559", "source": "deterministic" }, { @@ -74,8 +74,8 @@ "evidence": "corpus-observed", "purpose": "Interpret assertion failures and choose the right fast inner-loop command before full reruns.", "trigger": "When a workflow test file exists and the task is to run, iterate, or classify its test assertions.", - "src_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", - "dst_hash": "81b8feeccd891642572e587a494d1e4d3a1b369a19d92947626230494fd92beb", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", "source": "deterministic" }, { @@ -89,8 +89,8 @@ "evidence": "corpus-observed", "purpose": "Run workflow tests while preserving Planemo structured artifacts, Galaxy mode, invocation id, history id, and API follow-up context.", "trigger": "When choosing between managed Galaxy, external Galaxy, full test runs, existing invocation checks, or direct workflow runs.", - "src_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", - "dst_hash": "d6ac347c8361d6c312c64c1197426673e53aa1c5577e1c72f1c80a20ccd1cce3", + "src_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "dst_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", "source": "deterministic" }, { @@ -105,8 +105,8 @@ "purpose": "Validate Galaxy workflow test files before starting a Planemo or Galaxy-backed execution.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", - "dst_hash": "ff0de5041f4c3de0ab7abe9e7555c5a7120665b077f42661461301b0fb26f372", + "src_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "dst_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } diff --git a/plugins/foundry-skills/skills/run-workflow-test/_required_tools.json b/plugins/foundry-skills/skills/run-workflow-test/_required_tools.json index d93d84d..de85976 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/_required_tools.json +++ b/plugins/foundry-skills/skills/run-workflow-test/_required_tools.json @@ -3,9 +3,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", @@ -17,9 +17,9 @@ "tool": "planemo", "origin": "pypi", "package": "planemo", - "package_version": "0.75.47", + "package_version": "0.75.48", "invoke": "planemo", - "invoke_fallback": "uvx --from planemo==0.75.47 planemo", + "invoke_fallback": "uvx --from planemo==0.75.48 planemo", "availability_check": "planemo --version", "docs_url": "https://planemo.readthedocs.io/", "reference": "references/cli/planemo.md", diff --git a/plugins/foundry-skills/skills/run-workflow-test/references/cli/planemo.md b/plugins/foundry-skills/skills/run-workflow-test/references/cli/planemo.md index 10eeb14..d5fa285 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/references/cli/planemo.md +++ b/plugins/foundry-skills/skills/run-workflow-test/references/cli/planemo.md @@ -3,17 +3,17 @@ type: cli-tool tool: planemo origin: pypi package: planemo -package_version: "0.75.47" +package_version: "0.75.48" invoke: planemo -invoke_fallback: "uvx --from planemo==0.75.47 planemo" +invoke_fallback: "uvx --from planemo==0.75.48 planemo" availability_check: "planemo --version" docs_url: "https://planemo.readthedocs.io/" tags: - cli/planemo status: draft created: 2026-05-10 -revised: 2026-09-13 -revision: 6 +revised: 2026-11-09 +revision: 7 summary: "Galaxy tool/workflow runtime testing CLI; used by run-workflow-test and friends." --- @@ -23,7 +23,7 @@ Galaxy's runtime testing and authoring CLI. Foundry Molds invoke `planemo test`, ## Pin -`package_version` pins to the released `planemo==0.75.47` from PyPI — base upstream planemo, no fork. It is the single source of truth: the sync scripts carry no pin of their own and stamp whichever planemo they invoked, and `make check-planemo-pin` fails if any recorded copy of the version disagrees with this field. +`package_version` pins to the released `planemo==0.75.48` from PyPI — base upstream planemo, no fork. It is the single source of truth: the sync scripts carry no pin of their own and stamp whichever planemo they invoked, and `make check-planemo-pin` fails if any recorded copy of the version disagrees with this field. [galaxyproject/planemo#1636](https://github.com/galaxyproject/planemo/pull/1636) merged 2026-05-14 and first shipped in released **0.75.42**, so `planemo cli_metadata` and `planemo output_schema` are available in every release at or above this pin. Every Foundry consumer runs off this base pin: the workflow-test phases ([[run-workflow-test]], [[implement-galaxy-workflow-test]]) which need only `planemo test --test_output_json`, the convergence loop in [[convert-nfcore-module-to-galaxy-tool]], and the vendored-artifact regeneration story (`packages/planemo-cli-meta/`, `packages/planemo-test-report-schema/`). No fork pin is required. @@ -32,13 +32,13 @@ The floor is now 0.75.46 for a packaging reason, not a feature one. `galaxy-tool ## Install ```sh -uvx --from planemo==0.75.47 planemo --version +uvx --from planemo==0.75.48 planemo --version ``` For a persistent install: ```sh -uv tool install planemo==0.75.47 +uv tool install planemo==0.75.48 ``` Contributor laptops only need planemo when **regenerating** vendored artifacts (the schema JSON in `packages/planemo-test-report-schema/`, the planemo CLI manual pages under `content/cli/planemo/`). Normal Foundry development — `npm run validate`, `npm run test`, `npm run packages-test` — builds against the vendored JSON and does not invoke planemo. diff --git a/plugins/foundry-skills/skills/run-workflow-test/references/notes/galaxy-workflow-invocation-failure-reference.md b/plugins/foundry-skills/skills/run-workflow-test/references/notes/galaxy-workflow-invocation-failure-reference.md index 73d45d3..05128f3 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/references/notes/galaxy-workflow-invocation-failure-reference.md +++ b/plugins/foundry-skills/skills/run-workflow-test/references/notes/galaxy-workflow-invocation-failure-reference.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-tool-job-failure-reference]]" - "[[planemo-workflow-test-architecture]]" @@ -16,123 +16,113 @@ related_molds: - "[[debug-galaxy-workflow-output]]" - "[[validate-galaxy-workflow]]" sources: - - "~/projects/repositories/galaxy/lib/galaxy/schema/invocation.py" - - "~/projects/repositories/galaxy/lib/galaxy/workflow/run.py" - - "~/projects/repositories/galaxy/lib/galaxy/workflow/modules.py" - - "~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/workflows.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/invocation.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/workflow/run.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/workflow/modules.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/model/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/api/workflows.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/services/invocations.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/jobs.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/galaxy/activity.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/runnable.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/reports/report_markdown.tpl" summary: "Reference for Galaxy workflow invocation states, messages, failure reasons, and invocation API surfaces." --- -# Galaxy Workflow Invocation Failure Reference +# Galaxy workflow invocation failure reference -This note describes workflow-level failure surfaces in Galaxy. It is separate from [[galaxy-tool-job-failure-reference]] because invocation state answers whether Galaxy could schedule and drive the workflow, while job state answers whether individual tool jobs succeeded. +A workflow invocation records Galaxy's progress in scheduling a workflow. Its jobs record execution of the tools. A `scheduled` invocation has finished scheduling, and a `completed` invocation has terminal jobs, but neither state proves that the jobs succeeded or the outputs passed their tests. -## Invocation Versus Job Failure +For a failed workflow test, preserve the invocation state and messages alongside job states and output problems. Tool exit codes, streams, and structured job messages are covered in [[galaxy-tool-job-failure-reference]]. [[planemo-workflow-test-architecture]] describes how Planemo collects the run evidence, and [[debug-galaxy-workflow-output]] uses it to classify failures before recommending repairs. -Important distinction: +## Invocation evidence -- Invocation state says whether Galaxy scheduled, cancelled, failed, or completed the workflow invocation. -- Job state says whether jobs produced by invocation steps succeeded or failed. -- Invocation messages explain scheduler/evaluation/cancellation problems. -- Step states usually describe scheduling progress, not actual job success, unless a legacy serialization mode substitutes job state. +Invocation records expose scheduling state, structured messages, step jobs, and registered outputs. Job summaries aggregate execution states, while full job records contain exit codes, streams, and structured job messages. These records describe different parts of a workflow failure. -A robust workflow test reference should inspect both invocation APIs and job APIs. +A run can fail during staging or an API request before Galaxy creates an invocation. An absent invocation ID is not an invocation failure reason. -## Invocation States +## Invocation and step states -Galaxy invocation states are defined in `~/projects/repositories/galaxy/lib/galaxy/schema/invocation.py` and related model code. - -| State | Meaning | +| Invocation state | Meaning for diagnosis | |---|---| -| `new` | Newly created invocation. | -| `requires_materialization` | Deferred or undeferred inputs must materialize before scheduling can continue. | -| `ready` | Ready for another scheduler iteration. | -| `scheduled` | Workflow has been scheduled. Older clients may treat this as successful terminal state. | -| `cancelling` | Scheduler will cancel jobs and subworkflow invocations. | -| `cancelled` | Cancellation terminal state. | -| `failed` | Scheduler/evaluation/materialization failure terminal state. | -| `completed` | Newer completion state after all jobs reach terminal states. | +| `new` | Newly created invocation awaiting scheduling. | +| `requires_materialization` | Inputs need materialization before scheduling can continue. This is a waiting state, not itself a failure. | +| `ready` | Ready for another scheduling iteration. | +| `scheduled` | All steps have been scheduled. Tool jobs can still be running or have failed. | +| `cancelling` | Cancellation requested. The scheduler will cancel jobs and subworkflow invocations. | +| `cancelled` | Invocation cancelled. Inspect cancellation messages. | +| `failed` | Invocation failed during scheduling or evaluation. Inspect messages and upstream jobs or inputs. | +| `completed` | All jobs reached terminal states. Those states include `ok`, `error`, `deleted`, `skipped`, `paused`, and `stopped`. | -Version caveat: `completed` may not be uniformly available across all Galaxy/Planemo combinations. Planemo accounts for both `scheduled` and `completed` as success-like invocation states. +Completion is distinct from successful execution. Inspect the job-state counts even when the invocation is `completed`, then check the expected outputs and assertions. Older servers may expose `scheduled` without the newer completion tracking. -## Invocation Message Reasons +Ordinary invocation step states are `new`, `ready`, and `scheduled`. They describe scheduling, not tool execution. The deprecated `legacy_job_state=true` option substitutes job state and can return one step entry per mapped job. It can also show incomplete information during partial scheduling. The default serialization retains workflow step identity, while each step's `jobs` exposes execution states. -Structured invocation messages are the main durable surface for workflow-level failures. +## Structured invocation messages -| Reason | Typical meaning | -|---|---| -| `dataset_failed` | Dataset input or upstream dataset was failed/unusable. | -| `collection_failed` | Collection was failed, incompletely populated, or unusable. | -| `job_failed` | Upstream dependent job failed. | -| `output_not_found` | Expected step/subworkflow output could not be resolved. | -| `expression_evaluation_failed` | Workflow expression or condition evaluation failed. | -| `when_not_boolean` | Conditional `when` expression did not return a boolean. | -| `unexpected_failure` | Scheduler or evaluation failure without a more specific safe reason. | -| `workflow_parameter_invalid` | Workflow parameter validation failed. | -| `step_input_deleted` | Step input dataset/collection was deleted. | -| `history_deleted` | Invocation history was deleted, causing cancellation. | -| `user_request` | User/API cancellation. | -| `cancelled_on_review` | Pause/review step rejected continuation. | -| `workflow_output_not_found` | Warning: declared workflow output was not found. | - -Message records can include affected workflow step id, dependent step id, job id, HDA/HDCA ids, output name, and nested workflow step index path. These fields matter more than free-text messages for reference-quality diagnosis. - -## Common Workflow-Level Failure Surfaces - -| Surface | Reference signal | -|---|---| -| Request-time validation | API returns an error before useful invocation state exists. | -| Input materialization | Invocation enters `requires_materialization`, then may fail with `dataset_failed`. | -| Conditional evaluation | Invocation messages include `expression_evaluation_failed` or `when_not_boolean`. | -| Missing output graph edge | Invocation messages include `output_not_found`. | -| Deleted/purged inputs | Invocation messages include `step_input_deleted` or `dataset_failed`. | -| Upstream job failure | Invocation messages or summaries point to `job_failed`; job API has details. | -| Collection population failure | Invocation messages include `collection_failed`; inspect HDCA and mapped jobs. | -| Pause/review rejection | Cancellation reason `cancelled_on_review`. | -| User/history cancellation | Cancellation messages `user_request` or `history_deleted`. | -| Warning-only missing output | `workflow_output_not_found` warning; may later become Planemo assertion failure. | - -## API Surfaces - -Useful invocation APIs from `~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/workflows.py`: - -| API | Use | +The `messages` list contains failure, cancellation, and warning records. The `reason` and associated IDs identify the failure and affected resources. A missing free-text explanation does not mean there is no diagnosable failure. + +| Reason | Category | Evidence to inspect | +|---|---|---| +| `dataset_failed` | Failure | `hda_id`, affected step, and optional dependent step. Inspect dataset state and the producing job or materialization failure. | +| `collection_failed` | Failure | `hdca_id` and dependent step. Inspect collection population and member jobs. | +| `job_failed` | Failure | `job_id` and dependent step. Full job detail contains execution evidence. This reason identifies a dependency failure, not every failed job in a workflow. | +| `output_not_found` | Failure | `output_name` and dependent step. A required step output could not be resolved. Check the connection and producer. | +| `expression_evaluation_failed` | Failure | Affected step. Check expressions, conditions, or `pick_value` selection requirements. Detailed expression errors can be withheld. | +| `when_not_boolean` | Failure | Affected step and `details`. The `when` result was not a boolean. | +| `unexpected_failure` | Failure | Optional affected step and `details`. If the record lacks detail, an administrator may need scheduler logs. | +| `workflow_parameter_invalid` | Failure | Parameter input step and validator `details`. Compare the supplied value with the workflow input constraints. | +| `step_input_deleted` | Failure | `hda_id` or `hdca_id`, affected step, and `details`. A referenced dataset or collection was deleted. | +| `history_deleted` | Cancellation | `history_id`. The invocation history was deleted. | +| `user_request` | Cancellation | Cancellation was requested by the user or API. | +| `cancelled_on_review` | Cancellation | Affected pause step. Review rejected continuation. | +| `workflow_output_not_found` | Warning | Affected step and `output_name`. A declared workflow output was not registered. Check its producer and output declaration. | + +`output_not_found` is an evaluation failure. `workflow_output_not_found` is a warning about a declared workflow output. A warning alone does not fail the invocation, but a test requiring that output can still fail. Check optional or conditional output behavior before assuming a tool failed. + +Galaxy deliberately omits potentially sensitive exception or expression text on some `unexpected_failure` and `expression_evaluation_failed` paths. Other paths provide safe `details`. Preserve the structured record and request server logs when needed instead of inferring the missing explanation. + +### Match message steps to invocation steps + +Message `workflow_step_id` and `dependent_workflow_step_id` identify integer workflow step indices. Match them to the invocation step's `order_index` within the relevant workflow. They are different from the encoded workflow database ID also named `workflow_step_id` in a step response. + +An invocation step's encoded `id` is the value accepted by the step API. Its `job_id` identifies a single job when present, while `jobs` includes mapped jobs with `step_details=true`. For a subworkflow, `subworkflow_invocation_id` identifies the nested invocation. The message's `workflow_step_index_path` supplies enclosing subworkflow context and excludes the failing step itself. Preserve the path along with the message instead of treating its step index as a top-level step. + +## Invocation API reference + +Paths below are relative to the Galaxy server URL. IDs in paths are encoded API IDs. + +| Method and path | Result or action | |---|---| -| `GET /api/invocations/{invocation_id}` | Invocation state, messages, inputs, outputs, and steps. | -| `GET /api/invocations/{invocation_id}?step_details=true` | Richer step detail, jobs, outputs, and output collections. | +| `GET /api/invocations/{invocation_id}` | Invocation state, messages, inputs, registered outputs, output values, and steps. | +| `GET /api/invocations/{invocation_id}?step_details=true` | Adds step jobs, outputs, and output collections. | | `GET /api/invocations/steps/{step_id}` | Detail for one invocation step. | -| `GET /api/invocations/{invocation_id}/steps/{step_id}` | Step detail scoped to invocation. | -| `GET /api/invocations/{invocation_id}/jobs_summary` | Job state summary across the invocation. | -| `GET /api/invocations/{invocation_id}/step_jobs_summary` | Job state summary per workflow invocation step. | -| `GET /api/invocations/{invocation_id}/completion` | Completion record, if available. | -| `DELETE /api/invocations/{invocation_id}` | Cancel invocation. | -| `PUT /api/invocations/{invocation_id}/steps/{step_id}` | Continue or cancel pause/review step. | -| `GET /api/invocations/{invocation_id}/report` | Invocation report surface. | +| `GET /api/invocations/{invocation_id}/steps/{step_id}` | Alias of the preceding endpoint. It ignores `invocation_id`, so the path does not establish step membership. | +| `GET /api/invocations/{invocation_id}/jobs_summary` | Aggregate job counts in `states` and a `populated_state`. | +| `GET /api/invocations/{invocation_id}/step_jobs_summary` | Job-source summaries, including mapped job groups and recursive subworkflow jobs. | +| `GET /api/invocations/{invocation_id}/completion` | Completion time, final job-state summary, and executed completion hooks. Returns JSON `null` when there is no completion record. | +| `GET /api/invocations/{invocation_id}/report` | Generated invocation report, separate from raw invocation messages and job records. | +| `DELETE /api/invocations/{invocation_id}` | Requests cancellation. This changes the run. | +| `PUT /api/invocations/{invocation_id}/steps/{step_id}` | Sets pause/review action using `{"action": true}` to proceed or `{"action": false}` to reject continuation. This changes the run. | Workflow aliases under `/api/workflows/{workflow_id}/invocations/...` also exist. -## Planemo Caveats - -Planemo polls invocation and job state, but its user-facing output may not expose all structured invocation messages. Treat Planemo terminal output as a summary; use raw invocation and job APIs when the failure class matters. - -Likely noisy boundaries: +Despite its name, `step_jobs_summary` is not a dictionary keyed by workflow step index. Each entry's `model` identifies the kind of object summarized, and `id` belongs to that kind. Step details associate a `Job` or `ImplicitCollectionJobs` summary with the workflow step. A mapped step can have many jobs, and an aggregate state cannot identify which collection member failed. -- Planemo may show failed job details but omit structured invocation messages. -- `step_jobs_summary` is useful in Galaxy but not necessarily surfaced directly by Planemo output. -- Subworkflow paths can be represented in Galaxy message fields but may not be displayed by Planemo. -- A workflow can be `completed` or `scheduled` while one or more jobs failed; job summaries must still be inspected. +## Diagnostic pitfalls -## Durable Reference Use +- **A downstream message may describe an upstream defect.** `dataset_failed`, `collection_failed`, and `job_failed` can identify the consuming step. Follow the dependent step or resource ID back to the producer before changing that consumer. +- **A waiting invocation is not automatically broken.** For `ready` or `requires_materialization`, inspect inputs, jobs, and pause actions. Preserve timeout duration if polling ends without a terminal result. +- **Successful scheduling does not validate output content.** Inspect job states, collection population, registered output labels, and assertion failures independently. See [[galaxy-collection-semantics]] for collection shape and member diagnostics. +- **One step is not necessarily one job.** Preserve failed member jobs for mapped collections and nested invocation IDs for subworkflows. Do not collapse them into a single top-level step state. +- **Missing completion data does not prove failure.** A `null` completion response means no record exists. An older server may not implement the endpoint at all. Use invocation and job states to diagnose the run. -Use this note when a workflow run fails before or around job scheduling, when outputs are missing without an obvious tool stderr cause, or when Planemo only reports a generic invocation failure. +## Planemo evidence and version limits -The goal is not to prescribe a single repair loop. The goal is to preserve which Galaxy API surface proves the failure mode so later Molds can decide whether the defect belongs to data-flow, template, concrete step implementation, workflow test assertions, or runtime environment. +Planemo 0.75.47 collects invocation messages in `invocation_details`. In its JSON test report, the path is `tests[].data.invocation_details.details.messages`. Its Markdown report template also renders those messages. -## Verification Gaps +Preserve `tool_test_output.json`, other reports when generated, and the Galaxy and Planemo versions. Start with the test's `invocation_details`, `execution_problem`, and `output_problems` rather than a terminal summary alone. Planemo's success check combines history state with invocation state and accepts both `scheduled` and `completed` as successful scheduling states. It does not make either invocation state proof of passing output assertions. -Actual runs should verify: +A staging or execution exception can still leave incomplete invocation details. Missing-output errors can come from label drift, a missing workflow output declaration, conditional absence, failed jobs, or a download problem. Job, collection, and nested invocation records can distinguish these causes when the collected artifacts are incomplete. -- Whether Planemo surfaces invocation `messages` for each reason. -- Whether `completed` and `/completion` are populated in the target Galaxy used by tests. -- How mapped collections and subworkflow failures appear in `step_jobs_summary`. -- How warning-only `workflow_output_not_found` interacts with Planemo assertions. +The state and endpoint definitions above describe the pinned Galaxy source. No live run was used to verify every message reason, collection mapping, or subworkflow representation. Check the deployed server's responses before relying on completion tracking or a particular report layout. diff --git a/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-asserts-idioms.md index 6d0fef7..bd1160e 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-asserts-idioms.md +++ b/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-asserts-idioms.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-04-30 -revised: 2026-05-11 -revision: 6 +revised: 2026-10-21 +revision: 10 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[iwc-test-data-conventions]]" @@ -16,231 +16,109 @@ related_notes: - "[[validate-tests]]" - "[[iwc-tabular-operations-survey]]" - "[[galaxy-discover-datasets]]" -summary: "Decision and idiom guide for picking planemo workflow-test assertions: which family per output type, how to size tolerances, when to validate." +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." --- -# Planemo asserts: idiom and decision guide +# Choosing Planemo workflow-test assertions -Companion to [[iwc-test-data-conventions]] (input shapes), [[galaxy-workflow-testability-design]] (workflow structure before test YAML exists), and [[iwc-shortcuts-anti-patterns]] (what's accepted vs smell). This note is forward-looking: when authoring a new `-tests.yml`, which assertion family fits which output, and what the recommended tolerances and operators are. +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. -The **vocabulary itself is not restated here** — every assertion's parameter list, types, defaults, required fields, and Python docstring is rendered from the test-format JSON Schema at [[tests-format]]. Assertion names below deep-link into that page (e.g. [[tests-format#has_text_model|has_text]] jumps straight to that `$def`). +## 1. Choose by the property, then the output type -## 1. Choose by output type +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | -The single most useful decision table. Pick the row that matches the file format the workflow emits; default to the recommended assertion family. +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. -| Output type | Default assertion family | Why | Fallback | -|---|---|---|---| -| **Plain text reports / logs** (FastQC summary, MultiQC text section) | [[tests-format#has_text_model|has_text]] (substring on a known stable token) + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Substring is robust across versions; line count catches truncation | [[tests-format#has_text_matching_model|has_text_matching]] (regex) when token text changes per run | -| **HTML reports** (MultiQC HTML, custom dashboards) | [[tests-format#has_text_model|has_text]] against stable section names | HTML embeds timestamps and asset hashes; byte-diff is hopeless | [[tests-format#has_size_model|has_size]] + large `delta` as last resort | -| **Tabular** (TSV, CSV, BED-like) | [[tests-format#has_n_columns_model|has_n_columns]] + [[tests-format#has_text_model|has_text]] for headers + [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | Schema-shape + content probes; tolerant to row reordering | [[tests-format#has_line_matching_model|has_line_matching]] for a known canonical row | -| **VCF** | `compare: diff` with `lines_diff: 6` | The `lines_diff: 6` constant matches the typical VCF header preamble that embeds `##fileDate=` and `##source=` | [[tests-format#has_text_matching_model|has_text_matching]] for known variants | -| **BAM** | [[tests-format#has_size_model|has_size]] + [[tests-format#has_archive_member_model|has_archive_member]] (BAM is a gzipped block format) | Headers contain `@PG` lines with command lines and Galaxy job IDs; never byte-diff | Convert to SAM in a downstream step and assert on text | -| **FASTA** (deterministic — assemblies, consensus) | `file:` exact comparison or [[tests-format#has_text_model|has_text]] for known sequence | Output is byte-stable when the upstream tool is deterministic | [[tests-format#has_n_lines_model|has_n_lines]] if line wrapping varies | -| **FASTA** (non-deterministic — RepeatModeler libraries) | `compare: sim_size` with large `delta:` | Family content varies run-to-run | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` | -| **FASTQ** (rare as workflow output) | [[tests-format#has_n_lines_model|has_n_lines]] (must be multiple of 4) | Quality scores are read-id-dependent | `compare: sim_size` | -| **JSON** (deterministic — config dumps, params) | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] / [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] | Surgical: assert on the property you care about, ignore the rest | `compare: diff` if the JSON is fully canonical | -| **JSON** (stochastic — HyPhy stats, MCMC results) | `has_text: text: "{"` (existence-only) | Embedded floats break any structural assertion; see [[iwc-shortcuts-anti-patterns]] §1 | [[tests-format#has_h5_keys_model|has_h5_keys]]-equivalent for top-level structural keys | -| **HDF5 / AnnData** | [[tests-format#has_h5_keys_model|has_h5_keys]] + [[tests-format#has_h5_attribute_model|has_h5_attribute]] for known structure | Asserts shape, not values — values are float-noisy | [[tests-format#has_size_model|has_size]] with `delta:` | -| **XML** | [[tests-format#is_valid_xml_model|is_valid_xml]] + [[tests-format#has_element_with_path_model|has_element_with_path]] + `element_text_is`/`element_text_matches` | Surgical structural probes; byte-diff fails on whitespace/attribute-order | [[tests-format#has_n_elements_with_path_model|has_n_elements_with_path]] for cardinality | -| **PNG / image plots** | [[tests-format#has_image_width_model|has_image_width]] + [[tests-format#has_image_height_model|has_image_height]] + [[tests-format#has_size_model|has_size]] with `delta:` (5–10%) | Catches "rendered something with the right dimensions"; corpus norm | [[tests-format#has_image_center_of_mass_model|has_image_center_of_mass]] if mask/segmentation outputs are deterministic | -| **TIFF / multipage images** | [[tests-format#has_image_frames_model|has_image_frames]] + [[tests-format#has_image_channels_model|has_image_channels]] + [[tests-format#has_size_model|has_size]] with `delta:` | Same logic, channel/frame structure matters | — | -| **Archives** (zip, tar.gz) | `has_archive_member: path: "regex"` with nested `asserts:` | Asserts on a specific member; archive timestamps never byte-stable | [[tests-format#has_size_model|has_size]] with `delta:` | -| **GFF / GTF** | [[tests-format#has_n_lines_model|has_n_lines]] with `delta:` + [[tests-format#has_text_model|has_text]] for known feature | Tools reorder freely; hard to byte-diff | `has_n_columns: 9` (GFF spec) | -| **Cool / HiC matrices** | `compare: sim_size` with multi-MB `delta:` | Binary, run-to-run variance | [[tests-format#has_archive_member_model|has_archive_member]] for HDF5 component | +## 2. Compare operators and expected artifacts -When in doubt: start with [[tests-format#has_size_model|has_size]] + `delta_frac: 0.1`. It catches the catastrophic failure mode (empty / 10x bigger output). Then add a content probe. +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. -### 1a. If the assertion is too weak, revisit the workflow output +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. -Assertion choice sometimes reveals a workflow-design problem. If the only available output can support only a size check or image-dimension smoke test, check whether the workflow should expose a stronger checkpoint before settling for the weak assertion. +## 3. Set tolerances from the failure boundary -- Scanpy's plot outputs use size and image-dimension assertions, but the same workflow also exposes AnnData HDF5 checkpoints and a cluster-count table (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end uses coarse size bands for coverage/mapped-read outputs, but expression/count tables get stronger regex and exact-line checks (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. -Rule: if a translated workflow exposes only weakly assertable final reports, consult [[galaxy-workflow-testability-design]] and consider promoting a table/text/HDF5 checkpoint before writing the final assertions. +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. -## 2. The `compare:` operators +## 4. Text assertions and repeated checks -Top-level on a `file:` output assertion. Vocabulary, in decreasing strictness: - -- **`diff`** (default). Byte-for-byte equality with optional `lines_diff:` tolerance for a fixed number of header lines. Use only when the upstream tool is deterministic on fixed inputs *and* the output has no embedded timestamps, command lines, version banners, or hash-ordered Python-dict-style keys. -- **`re_match` / `re_match_multiline`**. Each line of the expected fixture is a regex that must match the corresponding output line. Useful when a few fields per row are timestamped but the rest is canonical. Rare in the corpus. -- **`contains`**. The expected fixture is a substring of the output. Cheap; weak. Prefer `asserts: has_text` for new code unless you genuinely have a multi-line block to assert as a whole. -- **`sim_size`**. Output file size matches the fixture's size within `delta:` (bytes) or `delta_frac:` (fraction). Use when the output is necessarily non-deterministic but its rough size is reproducible (RepeatModeler libraries, HiC matrices, Bayesian sampler outputs). - -Picking `lines_diff:`: count the mutable header lines in the output format. VCF: ~6 (`##fileformat`, `##fileDate`, `##source`, `##reference`, contig/info lines vary). SAM/text headers: count `@HD`/`@PG`/`@CO` lines. Set `lines_diff:` to the count exactly — looser values mask real diffs. - -## 3. Tolerance picking - -**`delta:`** is bytes (for [[tests-format#has_size_model|has_size]] and `compare: sim_size`) or absolute count (for [[tests-format#has_n_lines_model|has_n_lines]], [[tests-format#has_n_columns_model|has_n_columns]], [[tests-format#has_image_width_model|has_image_width]], etc.). Suffix multipliers documented in the schema — `1K`, `1M`, `1G` work. - -**`delta_frac:`** is a fraction (0.1 = 10%). Use when expected size scales with input volume. Three IWC tests use it (`scRNAseq/baredsc/*`, `genome-assembly/polish-with-long-reads/*`); the rest use absolute `delta:`. - -**Picking magnitudes** (from corpus survey in [[iwc-shortcuts-anti-patterns]] §2): - -- Image dimensions: `delta: 25–30` pixels (5% of typical matplotlib defaults). -- Image file size: `delta: 5K–60K` (5–10% of file size). -- Small text reports: `delta: 1K–10K`. -- HTML reports: `delta: 25K–100K`. -- BAM files: `delta: 1M–10M`. -- RepeatModeler / Bayesian sampler outputs: `delta: 30K–90M` (extreme, but justified by the underlying nondeterminism). - -**Heuristic for new outputs:** `delta_frac: 0.1` is a defensible default. Tighten if the output proves more deterministic than expected. - -## 4. Text family — [[tests-format#has_text_model|has_text]] vs [[tests-format#has_text_matching_model|has_text_matching]] vs [[tests-format#has_line_model|has_line]] vs [[tests-format#has_line_matching_model|has_line_matching]] vs [[tests-format#has_n_lines_model|has_n_lines]] - -All five are common; choose by what you're verifying. - -- **[[tests-format#has_text_model|has_text]]** — output contains the substring `text:`. Anywhere in the output, any number of times. Add `n:` / `min:` / `max:` to constrain occurrence count. Add `delta:` to allow slack on the count. -- **[[tests-format#has_text_matching_model|has_text_matching]]** — output matches the regex `expression:`. Use sparingly; prefer literal [[tests-format#has_text_model|has_text]] when you can. -- **[[tests-format#has_line_model|has_line]]** — output has at least one *line* matching `line:` exactly. Use when line boundaries matter (e.g. asserting on a specific row in a table). -- **[[tests-format#has_line_matching_model|has_line_matching]]** — same but with regex. -- **[[tests-format#has_n_lines_model|has_n_lines]]** — assert the line count is `n:` ± `delta:`. - -A common combination in IWC: `has_n_lines: n: 100, delta: 5` + `has_text: text: "expected_token"` — line-count sanity-check plus a content marker. This catches both truncation and content drift in one assertion pair. - -`negate: true` is supported on every assertion. Used for the "this output should NOT contain X" case. - -## 5. Collection output assertions (`element_tests:`) - -For Galaxy collection outputs, the test format keys element assertions by element identifier: +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. ```yaml -my_collection_output: - element_tests: - sample_1: +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: asserts: - has_text: - text: "expected" - sample_2: - file: test-data/expected_sample_2.txt -``` - -Optional `attributes:` at the collection level can assert on the produced collection shape: - -```yaml -my_collection_output: - attributes: {collection_type: list:list} - element_tests: - ... + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 ``` -Nested collections: outer `element_tests:` keyed by outer identifier; inner uses `elements:` (note plural, no `_tests` suffix on the inner). See [[iwc-test-data-conventions]] §2f for the live example. - -For a list-of-files where every element should pass the same minimal check, the existence-probe pattern (`has_text: "{"` for JSON; `has_size: min: 100` for any non-empty binary) is widely used and accepted in IWC. - -## 6. The validate-against-workflow inner loop - -A `-tests.yml` file can be **structurally invalid** in two distinct ways: - -1. **Schema-invalid** — wrong field names, wrong nesting, wrong types. Caught by the test-format JSON Schema. -2. **Workflow-incoherent** — schema-valid YAML, but the input/output labels don't match the actual workflow. Renaming an output in the `.ga` and forgetting to update its sibling `-tests.yml` produces this case. Planemo will surface it as an "output not found" error at test-runtime, but only after a full workflow run. - -The `@galaxy-tool-util/schema` npm package ships **two** validators that catch both cases statically — no Galaxy or Planemo invocation needed: +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. -- **`validateTestsFile(yaml)`** — runs the file against `tests.schema.json` (AJV). Reports schema violations with paths. -- **`checkTestsAgainstWorkflow(workflow, tests)`** — cross-checks a `.ga` / format2 workflow against a tests file: missing input labels, missing output labels, type incompatibilities (e.g. test supplies a `File` for a parameter typed `int`). +## 5. Collection and structured-output examples -Both are pure-JS, take milliseconds, and have no Galaxy dependency. Wire them into the inner authoring loop: +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" ``` -edit -tests.yml - → validateTestsFile() # schema gate - → checkTestsAgainstWorkflow(.ga, tests) # coherence gate - → planemo workflow_test_on_invocation # assertion gate (no full re-run) - → planemo test # full integration (slow) -``` - -The first two gates short-circuit cheap mistakes before a slow planemo run. They are the static-validation equivalent of `gxwf` for tests, and the `implement-galaxy-workflow-test` mold should reference them as its primary inner-loop tooling. Source: galaxy-tool-util-ts package, `src/test-format/index.ts` exports. - -## 7. Authoring loop — generation, then refinement - -Reviewer convention is to **generate** the initial `-tests.yml` rather than hand-write it. Two planemo subcommands cover this: - -- **`planemo workflow_test_init --from_invocation `** ([[planemo-workflow_test_init]]) — given a successful Galaxy invocation ID, emit a `-tests.yml` with a `job:` block that captures all inputs (with SHA-1 hashes) and an `outputs:` block with `file:` references to the actual outputs (downloaded into `test-data/`). Hand-tighten the assertions afterward. -- **`planemo workflow_test_on_invocation `** ([[planemo-workflow_test_on_invocation]]) — re-evaluate an edited `-tests.yml` against a saved invocation without re-running the workflow. The fast inner loop for assertion iteration; complements the static gates in §6. - -Together these cut the assertion-iteration cost dramatically. An agent should: - -1. Run the workflow once on usegalaxy.* (or local) to get a known-good invocation. -2. `--from_invocation` to bootstrap the test file. -3. Replace the autogenerated `file:` exact-comparison assertions with assertion-family-appropriate alternatives per §1. -4. [[planemo-workflow_test_on_invocation]] after each edit; full [[planemo-test]] at the end. - -## 8. What the schema gives you for free - -When the test-format schema lands as a Foundry-rendered note, the agent can consult any assertion's `$def` directly for: parameter types, defaults, required fields, the `that` discriminator constant, and the original Python docstring (carried through as `description`). This note does **not** restate that vocabulary — it complements it with the corpus-grounded *which-and-when*. - -What's still missing from the schema and worth keeping in research notes: - -- This decision table (§1) — output-type → assertion family. -- Tolerance magnitudes (§3) — corpus-derived defaults. -- The `validateTestsFile` / `checkTestsAgainstWorkflow` integration story (§6). -- Anti-pattern flags — see [[iwc-shortcuts-anti-patterns]]. -## 9. Common combinations (recipes) +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. -Six recipes worth memorizing. +## 6. Validate labels and schema before execution -**Stable text report (FastQC summary, simple stats).** -```yaml -my_report: - asserts: - has_n_lines: { n: 12, delta: 2 } - has_text: { text: "Total Sequences" } -``` - -**MultiQC HTML report.** -```yaml -multiqc_report: - asserts: - has_text: { text: "Filtered Reads" } - has_text: { text: "FastQC" } -``` +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: -**VCF (pinned tool, fixed reference).** -```yaml -called_variants: - file: test-data/expected.vcf - compare: diff - lines_diff: 6 +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga ``` -**Stochastic JSON (HyPhy-style).** -```yaml -hyphy_meme: - element_tests: - geneA: { asserts: { has_text: { text: "{" } } } - geneB: { asserts: { has_text: { text: "{" } } } -``` +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. -**Matplotlib plot.** -```yaml -umap_plot: - asserts: - has_size: { size: 68416, delta: 6000 } - has_image_width: { width: 601, delta: 30 } - has_image_height: { height: 429, delta: 25 } -``` +## 7. Create and refine the test -**AnnData (HDF5).** -```yaml -clustered_anndata: - asserts: - has_h5_keys: { keys: "obs/louvain" } - has_h5_keys: { keys: "var/highly_variable" } - has_h5_keys: { keys: "uns/rank_genes_groups" } - has_size: { size: 12000000, delta: 1500000 } -``` +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. -## 10. Cross-references +## 8. Source of the exact syntax -- [[galaxy-workflow-testability-design]] — decide which workflow outputs and checkpoints to expose before choosing assertions. -- [[iwc-test-data-conventions]] — input-side conventions (job inputs, collection shapes, `hashes:`, CVMFS). -- [[iwc-shortcuts-anti-patterns]] — accepted-vs-smell catalog and corpus prevalence; this note's mirror image. -- Test-format schema (`@galaxy-tool-util/schema` npm package) — authoritative vocabulary; will be vendored into a Foundry-rendered schema note. See `content/meta/casting.md` for the casting story. -- Planemo test-format spec: [planemo.readthedocs.io/en/latest/test_format.html](https://planemo.readthedocs.io/en/latest/test_format.html). -- [[galaxy-xsd]] — Galaxy XSD assertion source of truth, vendored from upstream. -- Tightening of the schema and Pydantic source: [galaxyproject/galaxy#22566](https://github.com/galaxyproject/galaxy/pull/22566). -- TS schema sync into npm: [jmchilton/galaxy-tool-util-ts#75](https://github.com/jmchilton/galaxy-tool-util-ts/pull/75). +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-workflow-test-architecture.md b/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-workflow-test-architecture.md index 59b3c25..72583a5 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-workflow-test-architecture.md +++ b/plugins/foundry-skills/skills/run-workflow-test/references/notes/planemo-workflow-test-architecture.md @@ -6,8 +6,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-11 -revision: 3 +revised: 2026-10-26 +revision: 4 related_notes: - "[[galaxy-workflow-testability-design]]" - "[[galaxy-tool-job-failure-reference]]" @@ -23,6 +23,8 @@ sources: - "~/projects/repositories/planemo/planemo/galaxy/activity.py" - "~/projects/repositories/planemo/planemo/galaxy/invocations" - "~/projects/repositories/planemo/planemo/galaxy/config.py" + - "~/projects/repositories/planemo/planemo/galaxy/serve.py" + - "~/projects/repositories/planemo/planemo/engine/galaxy.py" summary: "Reference for Planemo workflow test/run architecture, Galaxy modes, API polling, and noisy failure boundaries." --- @@ -94,6 +96,20 @@ Important operations: For workflows, Planemo invokes Galaxy using input labels/names. Stable generated labels are therefore important for both test execution and debugging. +### Workflow Import Failures + +Planemo imports the workflow while it brings Galaxy up for the run (`config.install_workflows()`, called from `_serve()` in `planemo/galaxy/serve.py` for managed Galaxy and from `ExternalGalaxyEngine.ensure_runnables_served` in `planemo/engine/galaxy.py` for external Galaxy, as of planemo 0.75.47). For gxformat2 it converts the workflow to a native dict client-side first, so Galaxy's own gxformat2 conversion never runs. That is outside the per-test `execute()` wrapper that turns exceptions into an `ErrorRunResponse`. When Galaxy rejects the import request, BioBlend raises `ConnectionError`, the test command exits non-zero, and Planemo writes neither the `--test_output_json` report nor the HTML report. No workflow, history, or invocation id exists. + +The evidence then lives in three places: + +| Evidence | What it holds | +|---|---| +| Upload response | HTTP status and body. A 400 usually names the failing field or validation rule. A 500 body rarely explains the cause. | +| Galaxy server log | The traceback for the failing request, the only place a 500's cause appears. Managed Galaxy streams its log into the Planemo terminal output and keeps it on disk. External Galaxy keeps it on the server. | +| Planemo terminal log | The BioBlend exception and the point in the run where it was raised. | + +Re-importing the same converted dict as the same user on the same server reproduces the failure without the rest of the Planemo run, which makes it the cheap loop for isolating a trigger. + Workflow testability design guidance lives in [[galaxy-workflow-testability-design]]. This architecture note only records why Planemo makes labels and workflow-level outputs operationally important. ## Structured Artifacts @@ -121,6 +137,7 @@ Planemo transforms and summarizes Galaxy failure information. These are likely i - Polling may summarize “at least one job is in error” while detailed job evidence is printed elsewhere or stored separately. - Missing outputs become assertion/output problems, which can conflate label mismatch, workflow output omission, optional output absence, failed invocation, or output download issue. - Output download failures may be logged but not always propagated as the most specific failure. +- A rejected workflow import aborts the run before any structured report is written, so terminal and server logs are the only record. - External Galaxy without an admin key may defer missing-tool problems until runtime. - `allow_tool_state_corrections=True` can let Galaxy adjust tool state during invocation, which is useful but can mask definition drift. - `--no_wait` is not suitable for debug conclusions because invocation creation can succeed before jobs fail. @@ -135,7 +152,7 @@ For [[run-workflow-test]]: For [[debug-galaxy-workflow-output]]: -- Start from structured Planemo output. +- Start from structured Planemo output. When there is none because the workflow import failed, start from the upload response and the Galaxy server log. - For missing-output failures, check label drift and omitted workflow-level outputs before assuming tool failure. - If the failure is job-level, inspect Galaxy job APIs described in [[galaxy-tool-job-failure-reference]]. - If the failure is invocation-level, inspect Galaxy invocation APIs described in [[galaxy-workflow-invocation-failure-reference]]. diff --git a/plugins/foundry-skills/skills/run-workflow-test/references/schemas/tests-format.schema.json b/plugins/foundry-skills/skills/run-workflow-test/references/schemas/tests-format.schema.json index 8de90e7..99d551f 100644 --- a/plugins/foundry-skills/skills/run-workflow-test/references/schemas/tests-format.schema.json +++ b/plugins/foundry-skills/skills/run-workflow-test/references/schemas/tests-format.schema.json @@ -1110,6 +1110,18 @@ "default": "Collection", "title": "Class" }, + "count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Count" + }, "element_tests": { "anyOf": [ { @@ -1171,6 +1183,30 @@ ], "default": null, "title": "Elements" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Maximum Count" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum Count" } }, "title": "TestCollectionCollectionElementAssertions", @@ -1366,6 +1402,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestCollectionDatasetElementAssertions", @@ -1679,6 +1728,19 @@ "default": null, "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" } }, "title": "TestDataOutputAssertions", diff --git a/plugins/foundry-skills/skills/summarize-cwl/SKILL.md b/plugins/foundry-skills/skills/summarize-cwl/SKILL.md index af46ec2..9f23290 100644 --- a/plugins/foundry-skills/skills/summarize-cwl/SKILL.md +++ b/plugins/foundry-skills/skills/summarize-cwl/SKILL.md @@ -58,7 +58,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `summary-cwl.json` before returning it: run `foundry validate-summary-cwl summary-cwl.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-summary-cwl summary-cwl.json`. This checks artifact `summary-cwl` against the summary-cwl schema. +- Before starting, check that the installed validator carries the summary-cwl schema this bundle was cast with: run `foundry validate-summary-cwl --schema-sha256 6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `summary-cwl.json` before returning it: run `foundry validate-summary-cwl --schema-sha256 6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef summary-cwl.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-summary-cwl --schema-sha256 6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef summary-cwl.json`. This checks artifact `summary-cwl` against the summary-cwl schema. ## Procedure diff --git a/plugins/foundry-skills/skills/summarize-cwl/_verify.json b/plugins/foundry-skills/skills/summarize-cwl/_verify.json index d99b524..d4ef6fb 100644 --- a/plugins/foundry-skills/skills/summarize-cwl/_verify.json +++ b/plugins/foundry-skills/skills/summarize-cwl/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-cwl", + "--schema-sha256", + "6ba695858bcc32ef1bb1b0c9f61d8a3c1074848819b6077c50bec3c7b5f6c6ef", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/summarize-galaxy-tool/SKILL.md b/plugins/foundry-skills/skills/summarize-galaxy-tool/SKILL.md index fa85f64..85cde2c 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-tool/SKILL.md +++ b/plugins/foundry-skills/skills/summarize-galaxy-tool/SKILL.md @@ -21,8 +21,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache`. +- **`galaxy-tool-cache`** (galaxy-tool-cache). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache`. Check: `galaxy-tool-cache --help | grep -q summarize`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli Bundled reference: `references/cli/galaxy-tool-cache.md`. @@ -43,7 +43,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `galaxy-tool-summary.json` before returning it: run `foundry validate-galaxy-tool-summary galaxy-tool-summary.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-tool-summary galaxy-tool-summary.json`. This checks artifact `galaxy-tool-summary` against the galaxy-tool-summary schema. +- Before starting, check that the installed validator carries the galaxy-tool-summary schema this bundle was cast with: run `foundry validate-galaxy-tool-summary --schema-sha256 bf77d0735410fa0028bc9bea08edaff3016580508bf27851fdcae65d038f2d94` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `galaxy-tool-summary.json` before returning it: run `foundry validate-galaxy-tool-summary --schema-sha256 bf77d0735410fa0028bc9bea08edaff3016580508bf27851fdcae65d038f2d94 galaxy-tool-summary.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-tool-summary --schema-sha256 bf77d0735410fa0028bc9bea08edaff3016580508bf27851fdcae65d038f2d94 galaxy-tool-summary.json`. This checks artifact `galaxy-tool-summary` against the galaxy-tool-summary schema. ## Procedure diff --git a/plugins/foundry-skills/skills/summarize-galaxy-tool/_provenance.json b/plugins/foundry-skills/skills/summarize-galaxy-tool/_provenance.json index 52c9eb3..82d2224 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-tool/_provenance.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-tool/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/summarize-galaxy-tool/index.md", "revision": 8, "content_hash": "a53a303bff1d7ace3b44fb12af329b12695bfcd165933634b8fb6484c0c306f7", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "f0515353aec7d57e89f60cac678c80907cf91e4c" }, - "cast_at": "2026-09-13T20:58:29.970Z", + "cast_at": "2026-10-27T20:17:22.994Z", "refs": [ { "kind": "cli-command", @@ -68,8 +68,8 @@ "load": "upfront", "evidence": "corpus-observed", "purpose": "Runtime that emits the tool summary; the Mold's entire job is invoking it against a populated cache. Install before driving the skill.", - "src_hash": "89f82f52d1a3051eb7ceca9ee87e5e925fcc5f2de9c557e336acc230edf7c2f8", - "dst_hash": "89f82f52d1a3051eb7ceca9ee87e5e925fcc5f2de9c557e336acc230edf7c2f8", + "src_hash": "2c9ed476362147b1e82fd007173e37eb5501458e1f9342bc66e28f064edbaf93", + "dst_hash": "2c9ed476362147b1e82fd007173e37eb5501458e1f9342bc66e28f064edbaf93", "source": "deterministic" }, { @@ -129,8 +129,8 @@ "trigger": "When a downstream Mold needs a specific input/output/help/citation field from the embedded `ParsedTool`.", "license": "MIT", "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", - "src_hash": "0f61d64b9f05f01513d25cddafbb05d3e39ecd3f3416d77f989246d52fffa665", - "dst_hash": "0f61d64b9f05f01513d25cddafbb05d3e39ecd3f3416d77f989246d52fffa665", + "src_hash": "3966d0c425e561891113f5c71a2d36a42c00a78cb744bb8970c577b46abcff9b", + "dst_hash": "3966d0c425e561891113f5c71a2d36a42c00a78cb744bb8970c577b46abcff9b", "source": "deterministic", "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" } diff --git a/plugins/foundry-skills/skills/summarize-galaxy-tool/_required_tools.json b/plugins/foundry-skills/skills/summarize-galaxy-tool/_required_tools.json index 18fbe4f..76702d0 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-tool/_required_tools.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-tool/_required_tools.json @@ -3,9 +3,9 @@ "tool": "galaxy-tool-cache", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "galaxy-tool-cache", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache", "availability_check": "galaxy-tool-cache --help | grep -q summarize", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "reference": "references/cli/galaxy-tool-cache.md", diff --git a/plugins/foundry-skills/skills/summarize-galaxy-tool/_verify.json b/plugins/foundry-skills/skills/summarize-galaxy-tool/_verify.json index 56c5c5e..3c0964d 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-tool/_verify.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-tool/_verify.json @@ -22,6 +22,8 @@ "validator_bin": "foundry", "args": [ "validate-galaxy-tool-summary", + "--schema-sha256", + "bf77d0735410fa0028bc9bea08edaff3016580508bf27851fdcae65d038f2d94", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/summarize-galaxy-tool/references/cli/galaxy-tool-cache.md b/plugins/foundry-skills/skills/summarize-galaxy-tool/references/cli/galaxy-tool-cache.md index 3d2b25e..4c94f7c 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-tool/references/cli/galaxy-tool-cache.md +++ b/plugins/foundry-skills/skills/summarize-galaxy-tool/references/cli/galaxy-tool-cache.md @@ -3,17 +3,17 @@ type: cli-tool tool: galaxy-tool-cache origin: npm package: "@galaxy-tool-util/cli" -package_version: "^1.8.1" +package_version: "^1.15.0" invoke: galaxy-tool-cache -invoke_fallback: "npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache" +invoke_fallback: "npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache" availability_check: "galaxy-tool-cache --help | grep -q summarize" docs_url: "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli" tags: - cli/galaxy-tool-cache status: draft created: 2026-06-16 -revised: 2026-06-18 -revision: 2 +revised: 2026-09-26 +revision: 3 summary: "Cache and inspect Galaxy tool metadata (fetch from ToolShed, summarize ParsedTool, export input JSON Schema)." --- @@ -23,4 +23,4 @@ Fetches Galaxy tool metadata from the Tool Shed, caches the parsed wrapper local ## Install -`npx --yes --package @galaxy-tool-util/cli@1.8.1 galaxy-tool-cache ` runs without a global install. For repeat use, `npm install -g @galaxy-tool-util/cli@1.8.1`. The `@1.8.1` pin matches [[gxwf]] (same npm package), tracks the `^1.8.1` devDependency in the repo `package.json` / `pnpm-lock.yaml`, and is the floor for stock/built-in tool resolution (bare ids). +`npx --yes --package @galaxy-tool-util/cli@1.15.0 galaxy-tool-cache ` runs without a global install. For repeat use, `npm install -g @galaxy-tool-util/cli@1.15.0`. The `@1.15.0` pin matches [[gxwf]] (same npm package), tracks the repository dependency floor, and includes stock/built-in tool resolution (bare ids). diff --git a/plugins/foundry-skills/skills/summarize-galaxy-tool/references/schemas/parsed-tool.schema.json b/plugins/foundry-skills/skills/summarize-galaxy-tool/references/schemas/parsed-tool.schema.json index 24e607f..52bda10 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-tool/references/schemas/parsed-tool.schema.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-tool/references/schemas/parsed-tool.schema.json @@ -280,8 +280,7 @@ "name", "label", "hidden", - "type", - "structure" + "type" ], "properties": { "name": { @@ -306,6 +305,192 @@ "collection" ] }, + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "collection_type_from_rules": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "structured_like": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "discover_datasets": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path", + "sort_key", + "sort_comp", + "sort_reverse", + "pattern" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "pattern" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + }, + "sort_key": { + "type": "string", + "enum": [ + "filename", + "name", + "designation", + "dbkey" + ] + }, + "sort_comp": { + "type": "string", + "enum": [ + "lexical", + "numeric" + ] + }, + "sort_reverse": { + "type": "boolean" + }, + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "discover_via", + "format", + "visible", + "assign_primary_output", + "directory", + "recurse", + "match_relative_path" + ], + "properties": { + "discover_via": { + "type": "string", + "enum": [ + "tool_provided_metadata" + ] + }, + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "visible": { + "type": "boolean" + }, + "assign_primary_output": { + "type": "boolean" + }, + "directory": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "recurse": { + "type": "boolean" + }, + "match_relative_path": { + "type": "boolean" + } + }, + "additionalProperties": false + } + ] + } + }, + { + "type": "null" + } + ] + }, "structure": { "type": "object", "required": [ diff --git a/plugins/foundry-skills/skills/summarize-galaxy-workflow/SKILL.md b/plugins/foundry-skills/skills/summarize-galaxy-workflow/SKILL.md index 769a458..d173a9d 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-workflow/SKILL.md +++ b/plugins/foundry-skills/skills/summarize-galaxy-workflow/SKILL.md @@ -27,8 +27,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim Check: `foundry --help`. Docs: https://github.com/galaxyproject/foundry/blob/main/packages/gxwf-foundry/README.md Bundled reference: `references/cli/foundry.md`. -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli @@ -44,7 +44,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `summary-galaxy-workflow.json` before returning it: run `foundry validate-summary-galaxy-workflow summary-galaxy-workflow.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-summary-galaxy-workflow summary-galaxy-workflow.json`. This checks artifact `summary-galaxy-workflow` against the summary-galaxy-workflow schema. +- Before starting, check that the installed validator carries the summary-galaxy-workflow schema this bundle was cast with: run `foundry validate-summary-galaxy-workflow --schema-sha256 55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `summary-galaxy-workflow.json` before returning it: run `foundry validate-summary-galaxy-workflow --schema-sha256 55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3 summary-galaxy-workflow.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-summary-galaxy-workflow --schema-sha256 55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3 summary-galaxy-workflow.json`. This checks artifact `summary-galaxy-workflow` against the summary-galaxy-workflow schema. ## Procedure diff --git a/plugins/foundry-skills/skills/summarize-galaxy-workflow/_provenance.json b/plugins/foundry-skills/skills/summarize-galaxy-workflow/_provenance.json index 94a4b03..ad12ca8 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-workflow/_provenance.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-workflow/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/summarize-galaxy-workflow/index.md", "revision": 2, "content_hash": "23b770bbbd5409a37a7a0ea457a10b2cfba21f36a6e7e892125735035ac36078", - "commit": "63a3f9cf9c97a637fe1628d298e524f35709e289" + "commit": "6af41ff36779319b209a23a56479fda1056f495c" }, - "cast_at": "2026-09-13T20:58:30.021Z", + "cast_at": "2026-09-26T14:10:15.560Z", "refs": [ { "kind": "cli-command", @@ -22,8 +22,8 @@ "purpose": "Convert a legacy `.ga` workflow to gxformat2 before summarizing, and record the conversion under documents.converted_path.", "trigger": "When the supplied workflow is native `.ga` rather than gxformat2.", "verification": "Cast the skill, run on one `.ga` and one gxformat2 fixture; confirm the `.ga` case populates documents.converted_path and both summaries validate.", - "src_hash": "1bb19d20b4a39b1708d8f5346a0294616cdb9df83ba5255c4a90d97030d13b83", - "dst_hash": "1cdf49f10f77039af6efc5b82fe79720bb18e50d1f57d653670697f7262c7640", + "src_hash": "bfbf46667e803c9ef8b176e2f01a2dbf5b0cd013bfdbf5182280803e736b3fc7", + "dst_hash": "ff19b97f01a45938a25ac68c6865903c313c94d939a5d5084250cb2191f9355c", "source": "deterministic" }, { diff --git a/plugins/foundry-skills/skills/summarize-galaxy-workflow/_required_tools.json b/plugins/foundry-skills/skills/summarize-galaxy-workflow/_required_tools.json index 9a5b721..f8bca6d 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-workflow/_required_tools.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-workflow/_required_tools.json @@ -14,9 +14,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", diff --git a/plugins/foundry-skills/skills/summarize-galaxy-workflow/_verify.json b/plugins/foundry-skills/skills/summarize-galaxy-workflow/_verify.json index 61f8d37..f34a0f4 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-workflow/_verify.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-workflow/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-galaxy-workflow", + "--schema-sha256", + "55831dd043243e137cae595badc7f1ecc9a84a44936c0e036c7c3f5d33978bb3", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/summarize-galaxy-workflow/references/cli/convert.json b/plugins/foundry-skills/skills/summarize-galaxy-workflow/references/cli/convert.json index 0f4e597..8769ba9 100644 --- a/plugins/foundry-skills/skills/summarize-galaxy-workflow/references/cli/convert.json +++ b/plugins/foundry-skills/skills/summarize-galaxy-workflow/references/cli/convert.json @@ -4,7 +4,7 @@ "command": "convert", "summary": "Convert a Galaxy workflow between native (.ga) and format2 (.gxwf.yml) representations.", "source_path": "content/cli/gxwf/convert.md", - "source_revision": 2, + "source_revision": 3, "package": "@galaxy-tool-util/cli", "description": "Convert between native (.ga) and format2 (.gxwf.yml) formats", "synopsis": "gxwf convert [options] ", @@ -72,15 +72,23 @@ { "flags": "--stateful", "name": "stateful", - "description": "Use cached tool definitions for schema-aware state re-encoding", + "description": "Force schema-aware state re-encoding even if the tool cache is empty", "takesArgument": false, "optionalArgument": false, "negatable": false }, + { + "flags": "--no-stateful", + "name": "stateful", + "description": "Skip schema-aware state re-encoding; copy tool_state through verbatim", + "takesArgument": false, + "optionalArgument": false, + "negatable": true + }, { "flags": "--cache-dir ", "name": "cacheDir", - "description": "Tool cache directory (for --stateful)", + "description": "Tool cache directory (default: schema-aware re-encoding when it is populated)", "takesArgument": true, "argumentPlaceholder": "", "optionalArgument": false, @@ -119,5 +127,5 @@ "negatable": false } ], - "body": "# `gxwf convert`\n\nConvert a Galaxy workflow between the native `.ga` JSON and the `.gxwf.yml` format2 representation. Use this to normalize fetched IWC workflows into a consistent comparison representation.\n\n## Output\n\nDefault output is the converted workflow on stdout. With `--output`, the result is written to a file. JSON output is selected with `--json` (or `--to native`); YAML output is selected with `--yaml` (or `--to format2`).\n\n## Examples\n\n```bash\ngxwf convert workflow.ga --to format2 --output workflow.gxwf.yml\ngxwf convert workflow.ga --to format2 --compact\ngxwf convert workflow.gxwf.yml --to native --output workflow.ga\n```\n\n## Gotchas\n\n- Default output is stdout; pipe or pass `--output` when persisting.\n- `--compact` drops node position metadata; useful for structural diffs and skeleton generation.\n- `--stateful` requires a populated tool cache (see `galaxy-tool-cache`); without it, tool-state stays as fetched." + "body": "# `gxwf convert`\n\nConvert a Galaxy workflow between the native `.ga` JSON and the `.gxwf.yml` format2 representation. Use this to normalize fetched IWC workflows into a consistent comparison representation.\n\n## Output\n\nDefault output is the converted workflow on stdout. With `--output`, the result is written to a file. JSON output is selected with `--json` (or `--to native`); YAML output is selected with `--yaml` (or `--to format2`).\n\n## Examples\n\n```bash\ngxwf convert workflow.ga --to format2 --output workflow.gxwf.yml\ngxwf convert workflow.ga --to format2 --compact\ngxwf convert workflow.gxwf.yml --to native --output workflow.ga\n```\n\n## Gotchas\n\n- Default output is stdout; pipe or pass `--output` when persisting.\n- `--compact` drops node position metadata; useful for structural diffs and skeleton generation.\n- With a populated tool cache, conversion re-encodes tool state using cached definitions by default. Use `--no-stateful` to copy tool state through verbatim, or `--stateful` to force re-encoding even when the cache is empty. See [[galaxy-tool-cache]].\n- Version `1.13.1` preserves numeric input `min` and `max` as native Galaxy range validators and recovers them on export." } diff --git a/plugins/foundry-skills/skills/summarize-nextflow/SKILL.md b/plugins/foundry-skills/skills/summarize-nextflow/SKILL.md index 77e6eed..b0188dc 100644 --- a/plugins/foundry-skills/skills/summarize-nextflow/SKILL.md +++ b/plugins/foundry-skills/skills/summarize-nextflow/SKILL.md @@ -46,7 +46,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Validation -- Validate `summary-nextflow.json` before returning it: run `foundry validate-summary-nextflow summary-nextflow.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-summary-nextflow summary-nextflow.json`. This checks artifact `summary-nextflow` against the summary-nextflow schema. +- Before starting, check that the installed validator carries the summary-nextflow schema this bundle was cast with: run `foundry validate-summary-nextflow --schema-sha256 1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `summary-nextflow.json` before returning it: run `foundry validate-summary-nextflow --schema-sha256 1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d summary-nextflow.json` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-summary-nextflow --schema-sha256 1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d summary-nextflow.json`. This checks artifact `summary-nextflow` against the summary-nextflow schema. ## Procedure @@ -133,7 +134,18 @@ A single JSON document conforming to summary-nextflow (`packages/summarize-nextf { "name": "versions", "shape": "path(\"versions.yml\")", "description": "tool versions YAML", "topic": null } ], "when": null, "script_excerpt": "\"\"\"\n$args\nminimap2 -t $task.cpus $reference $reads > ${prefix}.paf\n\"\"\"", // verbatim script body; script_summary is the LLM pass's job - "publish_dir": null } + "publish_dir": "[ path: { \"${params.outdir}/align\" }, mode: 'copy' ]", + "publish_dir_matches": [ + { "expression": "[ path: { \"${params.outdir}/align\" }, mode: 'copy' ]", + "selector": "withName:MINIMAP2_ALIGN", + "config": "modules/nf-core/minimap2/align/nextflow.config" } + ], + "resolved_args": [ + { "key": "args", "expression": "'-x sr'", + "selector": "withName:MINIMAP2_ALIGN", + "config": "modules/nf-core/minimap2/align/nextflow.config" } + ], + "template": null } ], "subworkflows": [ { "name": "FASTQ_TRIM_FASTP_FASTQC", @@ -272,12 +284,13 @@ This step does not reshape onto any target idiom (Galaxy `sample_sheet:paired` v #### 4. Enumerate processes For each `process { ... }` in `main.nf`, `workflows/`, `modules/**`, `subworkflows/**`: -- Pull `container`, `conda`, `publishDir`, `when:` directives **verbatim** into `processes[].container` / `processes[].conda`. Modern nf-core directives are ternary expressions (`workflow.containerEngine == 'singularity' ? : `) and file references (`${moduleDir}/environment.yml`); keep the directive text intact and resolve into `tools[]` separately (§5). +- Pull `container`, `conda`, `publishDir`, `when:` directives **verbatim** into `processes[].container` / `processes[].conda`. Modern nf-core directives are ternary expressions (`workflow.containerEngine == 'singularity' ? : `) and file references (`${moduleDir}/environment.yml`); keep the directive text intact and resolve into `tools[]` separately (§5). An in-body `publishDir` is kept in `processes[].publish_dir`. +- Follow the root `includeConfig` chain into workflow configs, per-module `nextflow.config` files, and `conf/modules/`. Skip `tests/` and `.github/`. Leave `includeConfig` statements and `withName:` / `withLabel:` selectors that sit inside a `profiles { }` block out of that walk. When `--profile` names a profile, follow only that profile's includes and selectors, and set `profile` on the `publish_dir_matches` and `resolved_args` entries they produce. Match selectors, including regular expressions and subworkflow-qualified names, against the process name, its aliases, and its labels. Record each matched `publishDir` in `publish_dir_matches` and, when the process body has none, in `publish_dir`. Record `ext.args`, `ext.args2`, and `ext.prefix` in `resolved_args` with the selector and the config path. Keep closure text verbatim so a downstream skill can see which `params` and `meta` fields it reads. - Tokenize the `input:` and `output:` blocks for declared channel names and shapes — typed channels (`tuple val(meta), path(reads)`) become shape strings (`"tuple(meta, [path])"`); arity is preserved as a string, not structured. - Sweep live `include { ... }` statements across the pipeline (`main.nf`, `workflows/`, `subworkflows/**`) to populate both `processes[].aliases` and `subworkflows[].aliases`. `include { MINIMAP2_ALIGN as MINIMAP2_CONSENSUS }` adds `MINIMAP2_CONSENSUS` to the canonical process; the same rule maps aliased subworkflow calls back to their canonical declaration. Calls and edges retain the alias spelling used by the source. Ignore includes inside line or block comments. - Detect `topic: ` annotations on outputs (Nextflow 24+ channel topics — nf-core templates emit `tuple(val("${task.process}"), val('toolname'), eval(...)) topic: versions` for version aggregation). Record the topic name in `ChannelIO.topic`. - Where `meta.yml` exists, **use it** for `description` and IO documentation rather than parsing the `script:` block. -- Carry the `script:`/`shell:`/`exec:` body verbatim into `script_excerpt` (dedented, truncated at 4000 characters, `stub:` dropped). For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. +- Carry the `script:`/`shell:`/`exec:` body verbatim into `script_excerpt` (dedented, truncated at 4000 characters, `stub:` dropped). For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When that body is only `template ''`, leave the directive in `script_excerpt` and read `templates/` beside the module into `processes[].template` (`path`, `line_count`, and the file body). The template is the program a tool wrapper has to reproduce. - LLM call (one per process, batchable): summarize `script_excerpt` in one line into `script_summary`. Ask only for what the tool *does*. Deterministic runs omit `script_summary` rather than emitting a placeholder. #### 5. Build the tool registry diff --git a/plugins/foundry-skills/skills/summarize-nextflow/_provenance.json b/plugins/foundry-skills/skills/summarize-nextflow/_provenance.json index 6ecd783..cdd907c 100644 --- a/plugins/foundry-skills/skills/summarize-nextflow/_provenance.json +++ b/plugins/foundry-skills/skills/summarize-nextflow/_provenance.json @@ -4,13 +4,13 @@ "mold": { "name": "summarize-nextflow", "path": "content/molds/summarize-nextflow/index.md", - "revision": 18, - "content_hash": "fb19fff9162cc36fae916a456cfbbe31a33c4565063a522846c48222d02f7c37", - "commit": "d1b4cabede5bc91a5ab8603d5c2d589ad7f0d100" + "revision": 20, + "content_hash": "d8e6759095994ad000fb8b78018da070761a5abfb8cf0a80c6c59b550a272501", + "commit": "871b5e6deadfd56aeb5e2ba92eb8f6f50c8449ee" }, "cast_method": "hand-cast", "cast_agent": "claude (manual transcription, no LLM cast prompt)", - "cast_at": "2026-09-15T23:16:27.772Z", + "cast_at": "2026-11-10T14:45:10.085Z", "cast_date": "2026-05-08", "cast_revision": 11, "cast_history": [ @@ -128,8 +128,8 @@ "purpose": "Interpret DSL2 layout, includes, workflow/subworkflow/module boundaries, and channel/process topology.", "trigger": "When walking pipeline structure or resolving process aliases and channel flow.", "verification": "Run the generated summarize-nextflow skill against nf-core/rnaseq and confirm this reference improves process/channel topology extraction.", - "src_hash": "30fa555739dbe40fc74f6bad780a2ec62432efb8650114c92858abe0eb76e4ab", - "dst_hash": "30fa555739dbe40fc74f6bad780a2ec62432efb8650114c92858abe0eb76e4ab", + "src_hash": "0cd0e02c9af76a8b195ce7918ed62f03f83dd8299703ec2f2d92926d8477daff", + "dst_hash": "0cd0e02c9af76a8b195ce7918ed62f03f83dd8299703ec2f2d92926d8477daff", "source": "deterministic" }, { @@ -241,8 +241,8 @@ "evidence": "cast-validated", "purpose": "Validate the emitted Nextflow summary JSON and provide downstream consumers the output contract.", "license": "MIT", - "src_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", - "dst_hash": "839e11b1892af74df3158377027e7d340a67dafdb9b5965d79e9715a1a109d54", + "src_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", + "dst_hash": "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "source": "deterministic" } ], diff --git a/plugins/foundry-skills/skills/summarize-nextflow/_verify.json b/plugins/foundry-skills/skills/summarize-nextflow/_verify.json index f1ccb79..7147067 100644 --- a/plugins/foundry-skills/skills/summarize-nextflow/_verify.json +++ b/plugins/foundry-skills/skills/summarize-nextflow/_verify.json @@ -10,6 +10,8 @@ "validator_bin": "foundry", "args": [ "validate-summary-nextflow", + "--schema-sha256", + "1b8872152de9be50634b3ec73cfd5963ae0ec19a95ccb99f6fe734f42656d80d", "{artifact_path}" ] } diff --git a/plugins/foundry-skills/skills/summarize-nextflow/references/notes/component-nextflow-pipeline-anatomy.md b/plugins/foundry-skills/skills/summarize-nextflow/references/notes/component-nextflow-pipeline-anatomy.md index 4c77cce..a181d5c 100644 --- a/plugins/foundry-skills/skills/summarize-nextflow/references/notes/component-nextflow-pipeline-anatomy.md +++ b/plugins/foundry-skills/skills/summarize-nextflow/references/notes/component-nextflow-pipeline-anatomy.md @@ -5,8 +5,8 @@ tags: component: "Nextflow Pipeline Anatomy (DSL2)" status: draft created: 2026-05-01 -revised: 2026-05-01 -revision: 1 +revised: 2026-11-10 +revision: 3 related_notes: - "[[nextflow-workflow-io-semantics]]" - "[[iwc-map-over-lifecycle-survey]]" @@ -26,6 +26,10 @@ Stub. Grown from cast contact, not pre-emptively (see issue #17). - nf-core pipeline structure: https://nf-co.re/docs/contributing/pipelines - nf-core module spec: https://nf-co.re/docs/contributing/modules +## Directives outside the process body + +nf-core pipelines keep publication paths and command-line arguments out of the `process` body. The root `nextflow.config` pulls them in with `includeConfig`, often through a workflow config into per-module `nextflow.config` files or `conf/modules/`. Each assignment sits in a `withName:` or `withLabel:` selector. A selector may be a plain process name, a regular expression, or a subworkflow-qualified path such as `NFCORE_RNASEQ:RNASEQ:FASTQC`. Closures stay as written, because evaluating them would hide which `params` and `meta` fields they read. A `profiles { }` block is separate from that chain. `includeConfig` and selectors inside it apply only when that profile is selected, and [[summarize-nextflow]] records the profile name on those entries. A script block that is only `template 'file'` points at `templates/` next to the module. The Mold copies the three into `publish_dir`, `resolved_args`, and `template`. + ## Open gaps _Paragraphs land here when the runtime cast hits a pipeline shape SKILL.md does not cover. Each entry names the motivating target._ diff --git a/plugins/foundry-skills/skills/summarize-nextflow/references/schemas/summary-nextflow.schema.json b/plugins/foundry-skills/skills/summarize-nextflow/references/schemas/summary-nextflow.schema.json index bada96d..d1b42ad 100644 --- a/plugins/foundry-skills/skills/summarize-nextflow/references/schemas/summary-nextflow.schema.json +++ b/plugins/foundry-skills/skills/summarize-nextflow/references/schemas/summary-nextflow.schema.json @@ -581,9 +581,101 @@ } } }, + "ResolvedArg": { + "title": "ResolvedArg", + "description": "One `ext.args`, `ext.args2`, or `ext.prefix` assignment copied verbatim from a `withName:` or `withLabel:` selector. Closures stay as Groovy text so a downstream Mold can see which `params` and `meta` fields they read.", + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "expression", + "selector", + "config" + ], + "properties": { + "key": { + "type": "string", + "enum": [ + "args", + "args2", + "prefix" + ] + }, + "expression": { + "type": "string", + "description": "Right-hand side of the assignment, verbatim, including a closure or list when that is how the config wrote it." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "PublishDirMatch": { + "title": "PublishDirMatch", + "description": "One `publishDir` assignment copied verbatim from a `withName:` or `withLabel:` selector.", + "type": "object", + "additionalProperties": false, + "required": [ + "expression", + "selector", + "config" + ], + "properties": { + "expression": { + "type": "string", + "description": "Right-hand side of `publishDir =`, verbatim." + }, + "selector": { + "type": "string", + "description": "The selector that matched, as `withName:` or `withLabel:`." + }, + "config": { + "type": "string", + "description": "Pipeline-relative config file that declared the selector." + }, + "profile": { + "type": "string", + "description": "Profile that contributed this assignment. Present only when the selector sits in that profile, or in a config the profile includes. Absent for selectors on the root includeConfig chain." + } + } + }, + "ProcessTemplate": { + "title": "ProcessTemplate", + "description": "The file named by a script block whose only statement is `template ''`. `script_excerpt` stays the directive. This object is the program.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "line_count", + "body" + ], + "properties": { + "path": { + "type": "string", + "description": "Pipeline-relative path of the template file. nf-core modules keep it at `templates/` beside `main.nf`." + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "body": { + "type": "string", + "description": "Template file contents. Empty when the directive names a file the resolver cannot find. A warning names that miss." + } + } + }, "Process": { "title": "Process", - "description": "One Nextflow `process { ... }` block.", + "description": "One Nextflow `process { ... }` block. `script_excerpt` is the script, shell, or exec body. When that body is only a `template` directive, `script_excerpt` is the indirection and `template` carries the file it names.", "type": "object", "additionalProperties": false, "required": [ @@ -684,14 +776,39 @@ "string", "null" ], - "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does." + "description": "Verbatim `script:`/`shell:`/`exec:` body, dedented and truncated at 4000 characters, or null when the process declares none. Deterministic. For ad-hoc pipelines with no `meta.yml` this is the only evidence of what a process does. When the body is only `template ''`, this stays that directive and `template` carries the file." }, "publish_dir": { "type": [ "string", "null" ], - "description": "`publishDir` value when the process publishes outputs (or null)." + "description": "`publishDir` value when the process publishes outputs, or null. An in-body directive is kept as written. Otherwise this is the verbatim `publishDir` assignment from each matching `withName:` / `withLabel:` selector on the root `includeConfig` chain, outside `profiles` blocks, joined by newlines when several match. Selectors from a profile named with `--profile` are included too." + }, + "publish_dir_matches": { + "type": "array", + "items": { + "$ref": "#/$defs/PublishDirMatch" + }, + "description": "Every config-selector `publishDir` that matched this process. Empty when publication is only declared in the process body, or not at all. `publish_dir` still carries an in-body directive when one exists. Entries from a selected profile carry `profile`." + }, + "resolved_args": { + "type": "array", + "items": { + "$ref": "#/$defs/ResolvedArg" + }, + "description": "`ext.args`, `ext.args2`, and `ext.prefix` assignments from matching config selectors. Expressions are verbatim, including closures. Empty when the chain declares none. Entries from a selected profile carry `profile`." + }, + "template": { + "anyOf": [ + { + "$ref": "#/$defs/ProcessTemplate" + }, + { + "type": "null" + } + ], + "description": "Template file loaded when `script_excerpt` is only a `template` directive. Null when the process script is not a template indirection." } } }, diff --git a/plugins/foundry-skills/skills/validate-galaxy-workflow/SKILL.md b/plugins/foundry-skills/skills/validate-galaxy-workflow/SKILL.md index e76364e..1c9b21d 100644 --- a/plugins/foundry-skills/skills/validate-galaxy-workflow/SKILL.md +++ b/plugins/foundry-skills/skills/validate-galaxy-workflow/SKILL.md @@ -21,8 +21,8 @@ Follow the procedure below and use the artifact/reference sections as the runtim ## Required Tools -- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.8.1'`. - Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf`. +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. Check: `gxwf --help | grep -q draft-validate`. Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli diff --git a/plugins/foundry-skills/skills/validate-galaxy-workflow/_provenance.json b/plugins/foundry-skills/skills/validate-galaxy-workflow/_provenance.json index b483e6c..59840ff 100644 --- a/plugins/foundry-skills/skills/validate-galaxy-workflow/_provenance.json +++ b/plugins/foundry-skills/skills/validate-galaxy-workflow/_provenance.json @@ -6,9 +6,9 @@ "path": "content/molds/validate-galaxy-workflow/index.md", "revision": 5, "content_hash": "74e3743fc376f33c1eb37832ab4869c327d48f87e015c7a38597cf4c4ca939cd", - "commit": "79bf5c3ab98eec3a67948d8bdebb9dba7d8a2359" + "commit": "88eb241e0b7226af678c1a938ab2bfc89a31aa1e" }, - "cast_at": "2026-09-12T19:19:47.237Z", + "cast_at": "2026-09-26T18:05:02.508Z", "refs": [ { "kind": "cli-command", @@ -37,8 +37,8 @@ "evidence": "corpus-observed", "purpose": "Keep static workflow validation findings distinct from Galaxy invocation/runtime failure surfaces.", "trigger": "When a workflow passes gxwf validation but still has likely runtime risks around invocation scheduling, outputs, conditionals, or collection population.", - "src_hash": "e8115df6ddca447d47d77cb33becda847cdd19b2cbe9924782bfc180b94bb451", - "dst_hash": "e8115df6ddca447d47d77cb33becda847cdd19b2cbe9924782bfc180b94bb451", + "src_hash": "e86b17c0b39b88076f1a1f177cce9ef1f868de0b36437a624e4b630e2c126559", + "dst_hash": "e86b17c0b39b88076f1a1f177cce9ef1f868de0b36437a624e4b630e2c126559", "source": "deterministic" }, { @@ -52,8 +52,8 @@ "evidence": "corpus-observed", "purpose": "Classify validation or pre-test findings that indicate missing labels, omitted workflow outputs, or untestable checkpoint structure.", "trigger": "When terminal validation passes but workflow-level outputs, labels, or collection shapes look likely to break future workflow tests.", - "src_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", - "dst_hash": "81b01801ec3d14810d570dd2d7815da2bb11e41a562af9182c29fd2974f4b855", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", "source": "deterministic" } ], diff --git a/plugins/foundry-skills/skills/validate-galaxy-workflow/_required_tools.json b/plugins/foundry-skills/skills/validate-galaxy-workflow/_required_tools.json index a8e3002..88cdd4d 100644 --- a/plugins/foundry-skills/skills/validate-galaxy-workflow/_required_tools.json +++ b/plugins/foundry-skills/skills/validate-galaxy-workflow/_required_tools.json @@ -3,9 +3,9 @@ "tool": "gxwf", "origin": "npm", "package": "@galaxy-tool-util/cli", - "package_version": "^1.8.1", + "package_version": "^1.15.0", "invoke": "gxwf", - "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.8.1 gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", "availability_check": "gxwf --help | grep -q draft-validate", "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", "source": "implied", diff --git a/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-invocation-failure-reference.md b/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-invocation-failure-reference.md index 73d45d3..05128f3 100644 --- a/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-invocation-failure-reference.md +++ b/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-invocation-failure-reference.md @@ -5,8 +5,8 @@ tags: - target/galaxy status: draft created: 2026-05-02 -revised: 2026-05-02 -revision: 1 +revised: 2026-09-26 +revision: 3 related_notes: - "[[galaxy-tool-job-failure-reference]]" - "[[planemo-workflow-test-architecture]]" @@ -16,123 +16,113 @@ related_molds: - "[[debug-galaxy-workflow-output]]" - "[[validate-galaxy-workflow]]" sources: - - "~/projects/repositories/galaxy/lib/galaxy/schema/invocation.py" - - "~/projects/repositories/galaxy/lib/galaxy/workflow/run.py" - - "~/projects/repositories/galaxy/lib/galaxy/workflow/modules.py" - - "~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/workflows.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/schema/invocation.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/workflow/run.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/workflow/modules.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/model/__init__.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/api/workflows.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/webapps/galaxy/services/invocations.py" + - "https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/managers/jobs.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/galaxy/activity.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/runnable.py" + - "https://github.com/galaxyproject/planemo/blob/eacf90419af1189b6b73d55a09a2651933a09b41/planemo/reports/report_markdown.tpl" summary: "Reference for Galaxy workflow invocation states, messages, failure reasons, and invocation API surfaces." --- -# Galaxy Workflow Invocation Failure Reference +# Galaxy workflow invocation failure reference -This note describes workflow-level failure surfaces in Galaxy. It is separate from [[galaxy-tool-job-failure-reference]] because invocation state answers whether Galaxy could schedule and drive the workflow, while job state answers whether individual tool jobs succeeded. +A workflow invocation records Galaxy's progress in scheduling a workflow. Its jobs record execution of the tools. A `scheduled` invocation has finished scheduling, and a `completed` invocation has terminal jobs, but neither state proves that the jobs succeeded or the outputs passed their tests. -## Invocation Versus Job Failure +For a failed workflow test, preserve the invocation state and messages alongside job states and output problems. Tool exit codes, streams, and structured job messages are covered in [[galaxy-tool-job-failure-reference]]. [[planemo-workflow-test-architecture]] describes how Planemo collects the run evidence, and [[debug-galaxy-workflow-output]] uses it to classify failures before recommending repairs. -Important distinction: +## Invocation evidence -- Invocation state says whether Galaxy scheduled, cancelled, failed, or completed the workflow invocation. -- Job state says whether jobs produced by invocation steps succeeded or failed. -- Invocation messages explain scheduler/evaluation/cancellation problems. -- Step states usually describe scheduling progress, not actual job success, unless a legacy serialization mode substitutes job state. +Invocation records expose scheduling state, structured messages, step jobs, and registered outputs. Job summaries aggregate execution states, while full job records contain exit codes, streams, and structured job messages. These records describe different parts of a workflow failure. -A robust workflow test reference should inspect both invocation APIs and job APIs. +A run can fail during staging or an API request before Galaxy creates an invocation. An absent invocation ID is not an invocation failure reason. -## Invocation States +## Invocation and step states -Galaxy invocation states are defined in `~/projects/repositories/galaxy/lib/galaxy/schema/invocation.py` and related model code. - -| State | Meaning | +| Invocation state | Meaning for diagnosis | |---|---| -| `new` | Newly created invocation. | -| `requires_materialization` | Deferred or undeferred inputs must materialize before scheduling can continue. | -| `ready` | Ready for another scheduler iteration. | -| `scheduled` | Workflow has been scheduled. Older clients may treat this as successful terminal state. | -| `cancelling` | Scheduler will cancel jobs and subworkflow invocations. | -| `cancelled` | Cancellation terminal state. | -| `failed` | Scheduler/evaluation/materialization failure terminal state. | -| `completed` | Newer completion state after all jobs reach terminal states. | +| `new` | Newly created invocation awaiting scheduling. | +| `requires_materialization` | Inputs need materialization before scheduling can continue. This is a waiting state, not itself a failure. | +| `ready` | Ready for another scheduling iteration. | +| `scheduled` | All steps have been scheduled. Tool jobs can still be running or have failed. | +| `cancelling` | Cancellation requested. The scheduler will cancel jobs and subworkflow invocations. | +| `cancelled` | Invocation cancelled. Inspect cancellation messages. | +| `failed` | Invocation failed during scheduling or evaluation. Inspect messages and upstream jobs or inputs. | +| `completed` | All jobs reached terminal states. Those states include `ok`, `error`, `deleted`, `skipped`, `paused`, and `stopped`. | -Version caveat: `completed` may not be uniformly available across all Galaxy/Planemo combinations. Planemo accounts for both `scheduled` and `completed` as success-like invocation states. +Completion is distinct from successful execution. Inspect the job-state counts even when the invocation is `completed`, then check the expected outputs and assertions. Older servers may expose `scheduled` without the newer completion tracking. -## Invocation Message Reasons +Ordinary invocation step states are `new`, `ready`, and `scheduled`. They describe scheduling, not tool execution. The deprecated `legacy_job_state=true` option substitutes job state and can return one step entry per mapped job. It can also show incomplete information during partial scheduling. The default serialization retains workflow step identity, while each step's `jobs` exposes execution states. -Structured invocation messages are the main durable surface for workflow-level failures. +## Structured invocation messages -| Reason | Typical meaning | -|---|---| -| `dataset_failed` | Dataset input or upstream dataset was failed/unusable. | -| `collection_failed` | Collection was failed, incompletely populated, or unusable. | -| `job_failed` | Upstream dependent job failed. | -| `output_not_found` | Expected step/subworkflow output could not be resolved. | -| `expression_evaluation_failed` | Workflow expression or condition evaluation failed. | -| `when_not_boolean` | Conditional `when` expression did not return a boolean. | -| `unexpected_failure` | Scheduler or evaluation failure without a more specific safe reason. | -| `workflow_parameter_invalid` | Workflow parameter validation failed. | -| `step_input_deleted` | Step input dataset/collection was deleted. | -| `history_deleted` | Invocation history was deleted, causing cancellation. | -| `user_request` | User/API cancellation. | -| `cancelled_on_review` | Pause/review step rejected continuation. | -| `workflow_output_not_found` | Warning: declared workflow output was not found. | - -Message records can include affected workflow step id, dependent step id, job id, HDA/HDCA ids, output name, and nested workflow step index path. These fields matter more than free-text messages for reference-quality diagnosis. - -## Common Workflow-Level Failure Surfaces - -| Surface | Reference signal | -|---|---| -| Request-time validation | API returns an error before useful invocation state exists. | -| Input materialization | Invocation enters `requires_materialization`, then may fail with `dataset_failed`. | -| Conditional evaluation | Invocation messages include `expression_evaluation_failed` or `when_not_boolean`. | -| Missing output graph edge | Invocation messages include `output_not_found`. | -| Deleted/purged inputs | Invocation messages include `step_input_deleted` or `dataset_failed`. | -| Upstream job failure | Invocation messages or summaries point to `job_failed`; job API has details. | -| Collection population failure | Invocation messages include `collection_failed`; inspect HDCA and mapped jobs. | -| Pause/review rejection | Cancellation reason `cancelled_on_review`. | -| User/history cancellation | Cancellation messages `user_request` or `history_deleted`. | -| Warning-only missing output | `workflow_output_not_found` warning; may later become Planemo assertion failure. | - -## API Surfaces - -Useful invocation APIs from `~/projects/repositories/galaxy/lib/galaxy/webapps/galaxy/api/workflows.py`: - -| API | Use | +The `messages` list contains failure, cancellation, and warning records. The `reason` and associated IDs identify the failure and affected resources. A missing free-text explanation does not mean there is no diagnosable failure. + +| Reason | Category | Evidence to inspect | +|---|---|---| +| `dataset_failed` | Failure | `hda_id`, affected step, and optional dependent step. Inspect dataset state and the producing job or materialization failure. | +| `collection_failed` | Failure | `hdca_id` and dependent step. Inspect collection population and member jobs. | +| `job_failed` | Failure | `job_id` and dependent step. Full job detail contains execution evidence. This reason identifies a dependency failure, not every failed job in a workflow. | +| `output_not_found` | Failure | `output_name` and dependent step. A required step output could not be resolved. Check the connection and producer. | +| `expression_evaluation_failed` | Failure | Affected step. Check expressions, conditions, or `pick_value` selection requirements. Detailed expression errors can be withheld. | +| `when_not_boolean` | Failure | Affected step and `details`. The `when` result was not a boolean. | +| `unexpected_failure` | Failure | Optional affected step and `details`. If the record lacks detail, an administrator may need scheduler logs. | +| `workflow_parameter_invalid` | Failure | Parameter input step and validator `details`. Compare the supplied value with the workflow input constraints. | +| `step_input_deleted` | Failure | `hda_id` or `hdca_id`, affected step, and `details`. A referenced dataset or collection was deleted. | +| `history_deleted` | Cancellation | `history_id`. The invocation history was deleted. | +| `user_request` | Cancellation | Cancellation was requested by the user or API. | +| `cancelled_on_review` | Cancellation | Affected pause step. Review rejected continuation. | +| `workflow_output_not_found` | Warning | Affected step and `output_name`. A declared workflow output was not registered. Check its producer and output declaration. | + +`output_not_found` is an evaluation failure. `workflow_output_not_found` is a warning about a declared workflow output. A warning alone does not fail the invocation, but a test requiring that output can still fail. Check optional or conditional output behavior before assuming a tool failed. + +Galaxy deliberately omits potentially sensitive exception or expression text on some `unexpected_failure` and `expression_evaluation_failed` paths. Other paths provide safe `details`. Preserve the structured record and request server logs when needed instead of inferring the missing explanation. + +### Match message steps to invocation steps + +Message `workflow_step_id` and `dependent_workflow_step_id` identify integer workflow step indices. Match them to the invocation step's `order_index` within the relevant workflow. They are different from the encoded workflow database ID also named `workflow_step_id` in a step response. + +An invocation step's encoded `id` is the value accepted by the step API. Its `job_id` identifies a single job when present, while `jobs` includes mapped jobs with `step_details=true`. For a subworkflow, `subworkflow_invocation_id` identifies the nested invocation. The message's `workflow_step_index_path` supplies enclosing subworkflow context and excludes the failing step itself. Preserve the path along with the message instead of treating its step index as a top-level step. + +## Invocation API reference + +Paths below are relative to the Galaxy server URL. IDs in paths are encoded API IDs. + +| Method and path | Result or action | |---|---| -| `GET /api/invocations/{invocation_id}` | Invocation state, messages, inputs, outputs, and steps. | -| `GET /api/invocations/{invocation_id}?step_details=true` | Richer step detail, jobs, outputs, and output collections. | +| `GET /api/invocations/{invocation_id}` | Invocation state, messages, inputs, registered outputs, output values, and steps. | +| `GET /api/invocations/{invocation_id}?step_details=true` | Adds step jobs, outputs, and output collections. | | `GET /api/invocations/steps/{step_id}` | Detail for one invocation step. | -| `GET /api/invocations/{invocation_id}/steps/{step_id}` | Step detail scoped to invocation. | -| `GET /api/invocations/{invocation_id}/jobs_summary` | Job state summary across the invocation. | -| `GET /api/invocations/{invocation_id}/step_jobs_summary` | Job state summary per workflow invocation step. | -| `GET /api/invocations/{invocation_id}/completion` | Completion record, if available. | -| `DELETE /api/invocations/{invocation_id}` | Cancel invocation. | -| `PUT /api/invocations/{invocation_id}/steps/{step_id}` | Continue or cancel pause/review step. | -| `GET /api/invocations/{invocation_id}/report` | Invocation report surface. | +| `GET /api/invocations/{invocation_id}/steps/{step_id}` | Alias of the preceding endpoint. It ignores `invocation_id`, so the path does not establish step membership. | +| `GET /api/invocations/{invocation_id}/jobs_summary` | Aggregate job counts in `states` and a `populated_state`. | +| `GET /api/invocations/{invocation_id}/step_jobs_summary` | Job-source summaries, including mapped job groups and recursive subworkflow jobs. | +| `GET /api/invocations/{invocation_id}/completion` | Completion time, final job-state summary, and executed completion hooks. Returns JSON `null` when there is no completion record. | +| `GET /api/invocations/{invocation_id}/report` | Generated invocation report, separate from raw invocation messages and job records. | +| `DELETE /api/invocations/{invocation_id}` | Requests cancellation. This changes the run. | +| `PUT /api/invocations/{invocation_id}/steps/{step_id}` | Sets pause/review action using `{"action": true}` to proceed or `{"action": false}` to reject continuation. This changes the run. | Workflow aliases under `/api/workflows/{workflow_id}/invocations/...` also exist. -## Planemo Caveats - -Planemo polls invocation and job state, but its user-facing output may not expose all structured invocation messages. Treat Planemo terminal output as a summary; use raw invocation and job APIs when the failure class matters. - -Likely noisy boundaries: +Despite its name, `step_jobs_summary` is not a dictionary keyed by workflow step index. Each entry's `model` identifies the kind of object summarized, and `id` belongs to that kind. Step details associate a `Job` or `ImplicitCollectionJobs` summary with the workflow step. A mapped step can have many jobs, and an aggregate state cannot identify which collection member failed. -- Planemo may show failed job details but omit structured invocation messages. -- `step_jobs_summary` is useful in Galaxy but not necessarily surfaced directly by Planemo output. -- Subworkflow paths can be represented in Galaxy message fields but may not be displayed by Planemo. -- A workflow can be `completed` or `scheduled` while one or more jobs failed; job summaries must still be inspected. +## Diagnostic pitfalls -## Durable Reference Use +- **A downstream message may describe an upstream defect.** `dataset_failed`, `collection_failed`, and `job_failed` can identify the consuming step. Follow the dependent step or resource ID back to the producer before changing that consumer. +- **A waiting invocation is not automatically broken.** For `ready` or `requires_materialization`, inspect inputs, jobs, and pause actions. Preserve timeout duration if polling ends without a terminal result. +- **Successful scheduling does not validate output content.** Inspect job states, collection population, registered output labels, and assertion failures independently. See [[galaxy-collection-semantics]] for collection shape and member diagnostics. +- **One step is not necessarily one job.** Preserve failed member jobs for mapped collections and nested invocation IDs for subworkflows. Do not collapse them into a single top-level step state. +- **Missing completion data does not prove failure.** A `null` completion response means no record exists. An older server may not implement the endpoint at all. Use invocation and job states to diagnose the run. -Use this note when a workflow run fails before or around job scheduling, when outputs are missing without an obvious tool stderr cause, or when Planemo only reports a generic invocation failure. +## Planemo evidence and version limits -The goal is not to prescribe a single repair loop. The goal is to preserve which Galaxy API surface proves the failure mode so later Molds can decide whether the defect belongs to data-flow, template, concrete step implementation, workflow test assertions, or runtime environment. +Planemo 0.75.47 collects invocation messages in `invocation_details`. In its JSON test report, the path is `tests[].data.invocation_details.details.messages`. Its Markdown report template also renders those messages. -## Verification Gaps +Preserve `tool_test_output.json`, other reports when generated, and the Galaxy and Planemo versions. Start with the test's `invocation_details`, `execution_problem`, and `output_problems` rather than a terminal summary alone. Planemo's success check combines history state with invocation state and accepts both `scheduled` and `completed` as successful scheduling states. It does not make either invocation state proof of passing output assertions. -Actual runs should verify: +A staging or execution exception can still leave incomplete invocation details. Missing-output errors can come from label drift, a missing workflow output declaration, conditional absence, failed jobs, or a download problem. Job, collection, and nested invocation records can distinguish these causes when the collected artifacts are incomplete. -- Whether Planemo surfaces invocation `messages` for each reason. -- Whether `completed` and `/completion` are populated in the target Galaxy used by tests. -- How mapped collections and subworkflow failures appear in `step_jobs_summary`. -- How warning-only `workflow_output_not_found` interacts with Planemo assertions. +The state and endpoint definitions above describe the pinned Galaxy source. No live run was used to verify every message reason, collection mapping, or subworkflow representation. Check the deployed server's responses before relying on completion tracking or a particular report layout. diff --git a/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md index 5018e34..3ba5b73 100644 --- a/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md +++ b/plugins/foundry-skills/skills/validate-galaxy-workflow/references/notes/galaxy-workflow-testability-design.md @@ -4,8 +4,8 @@ tags: - target/galaxy status: draft created: 2026-05-03 -revised: 2026-05-06 -revision: 2 +revised: 2026-09-23 +revision: 3 related_notes: - "[[iwc-workflow-testability-survey]]" - "[[iwc-test-data-conventions]]" @@ -19,118 +19,44 @@ related_notes: summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." --- -# Galaxy workflow testability design +# Design a Galaxy workflow that can be tested -Use this note when authoring or translating a Galaxy workflow **before** the `-tests.yml` file exists. It covers workflow structure choices that make later IWC-style tests meaningful: labels, promoted checkpoints, collection identifiers, and fixture-compatible inputs. +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. -This is not a `content/patterns/` page. It is cross-cutting design guidance for Molds that need testable Galaxy workflows. Assertion syntax lives in [[planemo-asserts-idioms]]. Test YAML fixture shapes live in [[iwc-test-data-conventions]]. Accepted shortcut vs smell calls live in [[iwc-shortcuts-anti-patterns]]. Corpus evidence trail lives in [[iwc-workflow-testability-survey]]. +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. -## 1. Treat labels as API +## Name the public interface before writing tests -Workflow input and output labels are not cosmetic. Planemo and IWC tests address workflow inputs and outputs by label, and the survey found exact label matches for every asserted output across 114 matched workflow/test pairs. A generated workflow should therefore pick stable, descriptive labels before test authoring starts. +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. -Rules: +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. -- Label every output that may need a test assertion. -- Treat input/output renames as breaking changes requiring sibling `-tests.yml` updates. -- Prefer stable domain names over tool-step defaults or positional names. -- Do not rely on unlabeled or positional outputs for tests. +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. -Evidence: +## Expose a checkpoint that can catch the wrong result -- Scanpy exposes outputs such as `Initial Anndata General Info`, `UMAP of louvain`, `Ranked genes with Wilcoxon test`, and `Dotplot of top genes on clusters` (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`). The sibling test keys assertions by those exact labels (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- VGP scaffolding uses punctuation-heavy labels such as `Hi-C duplication stats on scaffolds: Raw`, `Hi-C duplication stats on scaffolds: MultiQc`, and `Merged Alignment stats` (`$IWC_FORMAT2/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8.gxwf.yml:170-196`). The test asserts those exact labels (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:218-245`). +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. -## 2. Promote assertable checkpoints +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. -IWC workflow tests assert workflow-level outputs. Intermediate step results are invisible unless promoted to top-level workflow outputs. When final reports are weakly assertable, expose intermediate checkpoints that carry deterministic content or structure. +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. -Rules: +## Make collection members addressable -- Promote intermediate outputs when they are the best deterministic or structural checkpoint. -- Prefer a checkpoint table/text/HDF5 object that can prove content over a final plot/report that can only prove existence. -- Accept some output-list clutter when it buys meaningful tests. -- Do not promote every intermediate by default; expose checkpoints that map to concrete assertion intent. +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. -Evidence: +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. -- Scanpy exposes 21 workflow outputs and the sibling test asserts all 21. These include initial AnnData summaries, intermediate plots, ranked-gene tables, final AnnData, cluster-count tables, and final plots (`$IWC_FORMAT2/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.gxwf.yml:105-147`; `$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:27-205`). -- RNA-seq paired-end exposes mapped reads, stranded/unstranded coverage, abundance estimates, expression tables, counts tables, and MultiQC reports (`$IWC_FORMAT2/transcriptomics/rnaseq-pe/rnaseq-pe.gxwf.yml:90-112`). The sibling test asserts sizes for coverage/read outputs and stronger regex/line checks for expression/count outputs (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- MGnify complete exposes 83 workflow outputs; 38 are asserted in the sibling test, including MultiQC reports, FASTA collections, taxonomic classifications, OTU tables, and HDF5/JSON outputs (`$IWC_FORMAT2/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete.gxwf.yml:163-329`; `$IWC/workflows/amplicon/amplicon-mgnify/mgnify-amplicon-pipeline-v5-complete/mgnify-amplicon-pipeline-v5-complete-tests.yml:15-120`). +## Choose inputs that real fixtures can supply -## 3. Stabilize collection output identifiers +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. -Collection tests key assertions by element identifier. If a workflow emits collections with unstable or opaque identifiers, the test cannot target elements cleanly. +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. -Rules: +## Express the choice in gxformat2 -- Preserve biologically or sample-meaningful identifiers through map-over and collection reshaping. -- When generating or relabeling collections, make the identifier derivation deterministic and visible in workflow structure. -- For nested collections, ensure each axis has predictable identifiers. -- Quote special identifiers in tests when YAML requires it, but do not simplify identifiers merely for YAML convenience. +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. -Evidence: +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. -- 59 of 115 IWC test files use `element_tests:`, with 227 `element_tests:` blocks in the corpus survey. -- 10x CellPlex tests nested collection outputs by `subsample`, then inner `matrix`, `barcodes`, and `genes` elements (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:82-128`). The workflow exposes the corresponding collection outputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:73-91`). -- HyPhy collection outputs are tested by generated gene identifiers such as `NC_001477.1|capsid_protein_C|95-394_DENV1` (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:31-71`). The workflow exposes collection outputs for MEME, PRIME, BUSTED, and FEL (`$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:26-38`). - -## 4. Choose checkpoints by assertion strength - -Assertion choice is not only a test-file decision. It should feed back into workflow output design. If the only exposed output is a stochastic plot or binary file, the best possible test may be a weak size check. Exposing a sibling table, report, HDF5 structure, or summary line can make the same workflow much more testable. - -Rules: - -- For image-heavy workflows, expose data or summary outputs behind the plot when possible. -- For stochastic statistical outputs, expose structural checkpoints and stable summary tokens. -- For binary outputs, expose a text/table report or stats file when the tool can produce one. -- Use [[planemo-asserts-idioms]] to select the assertion family after choosing the checkpoint. - -Evidence: - -- Scanpy image outputs are mostly smoke-tested with `has_size`, `has_image_width`, and `has_image_height`, but the same workflow also exposes AnnData HDF5 keys and cluster-count table checks (`$IWC/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml:33-205`). -- RNA-seq paired-end pairs coarse size checks for coverage/mapped-read outputs with stronger regex and exact-line checks for expression/count tables (`$IWC/workflows/transcriptomics/rnaseq-pe/rnaseq-pe-tests.yml:48-97`). -- VGP scaffolding tests combine stable text checks for scaffold/report stats with size checks for map/alignment artifacts (`$IWC/workflows/VGP-assembly-v2/Scaffolding-HiC-VGP8/Scaffolding-HiC-VGP8-tests.yml:173-245`). - -## 5. Design inputs with fixtures in mind - -Workflow input labels and types constrain the eventual `job:` block. Fixture planning is not only a test-file activity: it should influence whether the workflow exposes a file input, a collection input, a string data-table input, or a typed parameter. - -Rules: - -- Choose input labels that will be readable as test `job:` keys. -- Match workflow input collection types to realistic fixture shapes. -- Decide early whether reference data should be a portable remote file or a CVMFS/data-table string. -- Keep typed parameters explicit when tests need to set them (`int`, `boolean`, `string`) rather than burying them in step defaults. - -Evidence: - -- 10x CellPlex job inputs include `fastq PE collection GEX`, `reference genome`, `gtf`, `cellranger_barcodes_3M-february-2018.txt`, `fastq PE collection CMO`, `sample name and CMO sequence collection`, and `Number of expected cells` (`$IWC/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml:2-75`). The workflow declares matching collection, data, string, boolean, and int inputs (`$IWC_FORMAT2/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex.gxwf.yml:4-72`). -- HyPhy accepts a `list` collection of unaligned sequences and preserves accession-like fixture identifiers through to output element assertions (`$IWC/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml:7-30`; `$IWC_FORMAT2/comparative_genomics/hyphy/hyphy-core.gxwf.yml:14-25`). - -## 6. Know what a gxformat2 output entry contains - -Top-level gxformat2 `outputs:` is the public workflow-output surface. It is separate from per-step `out:` declarations and from step post-job actions such as `change_datatype` or `rename`. - -Authoring rules: - -- Use `label` as the stable public name tests and users will address. -- Use `outputSource` to point at the producing step output; do not rely on positional output order. -- Use `doc` for short user-facing context when the label is not self-explanatory. -- Keep `type` aligned with the exposed value (`data`, `collection`, or scalar vocabulary from [[gxformat2-workflow-inputs]]) when the schema needs it. -- Apply `change_datatype` at the producing step output when Galaxy needs a stronger datatype than the tool reports; choose values from [[galaxy-datatypes-conf]]. -- Use `rename` only for generated dataset names inside Galaxy histories. It is not a substitute for stable workflow-output `label`. -- Treat `add_tags` and `remove_tags` as metadata helpers, not as the test API. IWC tests key by labels and collection element identifiers, not tags. -- Avoid `hide` or `delete_intermediate_datasets` on outputs that are promoted as test checkpoints. - -Design inference: a workflow-output promotion decision should pick both the public `outputs:` entry and any producer-side post-job action needed to make that output useful. For example, a synthesized BED checkpoint needs a stable output `label` plus a producer-side `change_datatype: bed`; one without the other is incomplete for a testable workflow. - -## Cross-references - -- [[iwc-workflow-testability-survey]] — corpus survey and distribution rationale. -- [[iwc-test-data-conventions]] — job/input YAML shapes, remote fixtures, hashes, collection fixture syntax. -- [[planemo-asserts-idioms]] — assertion-family choice after an output is exposed. -- [[iwc-shortcuts-anti-patterns]] — accepted shortcut vs smell calls for weak assertions and label coupling. -- [[planemo-workflow-test-architecture]] — Planemo execution, output-problem ambiguity, and structured artifacts. -- [[gxformat2-schema]] — structural vocabulary for top-level workflow outputs and step post-job actions. -- [[galaxy-datatypes-conf]] — valid Galaxy datatype extensions for `format` and `change_datatype` choices. +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/SKILL.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/SKILL.md new file mode 100644 index 0000000..f558b8f --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/SKILL.md @@ -0,0 +1,71 @@ +--- +name: workflow-brief-to-galaxy-data-flow +description: "Translate a reviewed Workflow Brief and its Galaxy interface into a Galaxy data-flow design brief." +--- + +# workflow-brief-to-galaxy-data-flow + +Follow the procedure below and use the artifact/reference sections as the runtime contract. + +## When To Use + +- Translate a reviewed Workflow Brief and its Galaxy interface into a Galaxy data-flow design brief. + +## Inputs + +- Read artifact `workflow-brief`. Schema: workflow-brief-schema. Produced by `freeform-summary-to-workflow-brief`, `nextflow-summary-to-workflow-brief`. Expert-reviewed implementation contract governing scientific scope, requirements, and acceptance criteria. +- Read artifact `freeform-summary`. Optional; absence is allowed and must be reported honestly. Produced by `interview-to-freeform-summary`, `summarize-paper`. Optional narrative source evidence retained from brief production; consult it for details within the selected scope. +- Read artifact `summary-nextflow`. Optional; absence is allowed and must be reported honestly. Schema: summary-nextflow. Produced by `summarize-nextflow`. Optional structured Nextflow source evidence; consult it without treating source implementation as Galaxy design. +- Read artifact `workflow-brief-galaxy-interface`. Produced by `workflow-brief-to-galaxy-interface`. Preceding Galaxy interface brief from workflow-brief-to-galaxy-interface that pins inputs, outputs, and labels. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried workflow-run knowledge open-requirements-ledger initialized by the interface Mold. + +## Outputs + +- Write artifact `workflow-brief-galaxy-data-flow` as `workflow-brief-galaxy-data-flow.md`. Format: `markdown`. Reviewable Markdown brief: abstract operations, collection map/reduce choices, shape-changing placeholder steps, unresolved Galaxy tool needs, confidence, open questions. +- Write artifact `open-requirements-ledger` as `open-requirements.ledger.yml`. Format: `yaml`. Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact. + +## Required Tools + +- None declared. Procedure should not assume external CLIs are present. + +## Load Upfront + +- `references/notes/galaxy-data-flow-draft-contract.md`: Research note copied verbatim into the bundle. Keep the data-flow brief separate from gxformat2 templating and concrete step implementation. +- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Inherit open entries rather than re-deriving them, close the ones this brief's wiring and collection decisions settle, and append data-flow obligations the narrative leaves open — a described transformation with no named tool, an implied intermediate the source never states, an ordering the prose doesn't fix. +- `references/notes/workflow-brief-design.md`: Research note copied verbatim into the bundle. Keep reviewed intent authoritative while using source summaries only as supporting evidence. +- `references/schemas/workflow-brief-schema.schema.json`: Schema file copied verbatim into the bundle. Carry the required implementation-contract shape into this design step. + +## Load On Demand + +- `references/patterns/galaxy-collection-patterns.md`: Pattern note copied verbatim into the bundle. Ground collection-shape choices in curated, corpus-observed operation and recipe patterns. Use when: selecting collection cleanup, reshape, identifier, or collection-tabular bridge patterns. +- `references/patterns/galaxy-conditionals-patterns.md`: Pattern note copied verbatim into the bundle. Ground conditional-branch and optional-step choices in curated, corpus-observed Galaxy when/pick_value patterns. Use when: data-flow translation needs optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches. +- `references/patterns/galaxy-interval-patterns.md`: Pattern note copied verbatim into the bundle. Ground genomic-interval operation choices in curated, corpus-observed Galaxy interval recipes. Use when: the workflow operates on genomic intervals (BED/GFF/VCF coordinate features) and data-flow translation needs overlap, merge, coverage, windowing, masking, or set-algebra steps. +- `references/patterns/galaxy-tabular-patterns.md`: Pattern note copied verbatim into the bundle. Ground tabular bridge and table-operation choices in curated, corpus-observed operation patterns. Use when: data-flow translation needs filtering, joining, aggregation, pivoting, or tabular-collection bridges. +- `references/notes/galaxy-sample-sheet-collections.md`: Research note copied verbatim into the bundle. Preserve per-row metadata on the data-flow side: keep sample_sheet column_definitions wired through identifier-keyed steps instead of dropping into parallel parameter inputs, and re-attach metadata after map-over steps that lose it. Use when: the interface brief carries a sample_sheet[:paired|:paired_or_unpaired|:record] input, or supporting evidence describes per-sample/per-record metadata that must survive map-over steps. + +## Validation + +- None declared. + +## Procedure + +Read the reviewed Workflow Brief plus the preceding Galaxy interface brief and emit a reviewable Markdown data-flow brief. Consult retained source evidence when available. Capture abstract operations, collection map/reduce choices, shape-changing placeholder transformations, unresolved Galaxy tool needs, confidence, and open questions. + +The Workflow Brief governs scope and requirements. A `freeform-summary` or `summary-nextflow` can substantiate details within that scope but cannot broaden it or turn source implementation into assumed Galaxy wiring. Translate what the brief, interface, and evidence support; classify the rest as unresolved tool needs or ledger obligations. If correct implementation requires changing reviewed intent, propose a brief change and stop rather than silently applying it. + +The output is not gxformat2 and should not resolve exact Tool Shed tools. workflow-brief-to-galaxy-template turns this handoff and the interface brief into a skeleton. + +## Feedback Mode + +- Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. +- When enabled, read `_feedback.md` before doing the work and use its registered `foundry-feedback.ledger.yml` protocol. +- Preserve harness-owned run and phase state. Append only concrete observations about a canonical Foundry source asset or a related project that this run showed to be at fault; do not put ordinary workflow requirements in this ledger. +- Before reporting completion, make one explicit pass over the work you just did. Do not ask yourself whether anything was unclear — recall what happened: where you guessed at something the instructions should have settled, needed information this bundle does not carry, hit an instruction that contradicted another or contradicted the artifacts in front of you, used a packaged reference that did not cover your case, or did something the procedure never describes. +- Append an entry for each such event that clears the protocol's bar. If none do, append nothing and report `no feedback` explicitly. Silence and a clean pass are not the same thing, and nothing downstream can tell them apart unless you say which one it was. +- Pass the same ledger path to any subagent used for this work, and merge updates serially so one writer cannot overwrite another. + +## Runtime Notes + +- Do not read Foundry source files at runtime; use only files packaged in this skill bundle and user-supplied artifacts. +- Preserve declared artifact filenames unless the user or harness supplies explicit paths. +- Carry unresolved assumptions into the output artifact instead of silently inventing missing source evidence. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_feedback.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_feedback.md new file mode 100644 index 0000000..61a4bd4 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_feedback.md @@ -0,0 +1,143 @@ +# Foundry feedback ledger + +The `foundry-feedback-ledger` is an opt-in runtime artifact for observations about the Foundry +assets used during a real cast-skill or pipeline run. It records actionable gaps, defects, +friction, and wishes about Molds and their authored references. It does not record unmet +requirements in the workflow being built; those belong in the open-requirements ledger. + +Every entry is destined for `galaxyproject/foundry`. When a run shows that a related project is +at fault — Galaxy, a tool wrapper, the IWC corpus, a harness CLI — that observation still belongs +here, recorded with the analysis that points at the upstream. The Foundry is where the run +context lives, so it is where the issue is tracked; a maintainer forwards from there. Never tell +a user to take an observation to another tracker instead. + +The registered filename is `foundry-feedback.ledger.yml`. Feedback-aware callers pass the same +file through every skill invocation in a run. + +## Ledger shape + +```yaml +run: + pipeline: nextflow-to-galaxy + run_slug: sarek-galaxy + status: running + phases: + - { n: 1, kind: mold, skill: summarize-nextflow, status: done, feedback_checked: true } + - { n: 2, kind: mold, skill: nextflow-summary-to-galaxy-interface, status: running } +entries: + - id: channel-shape-note-silent-on-optional-outputs + raised_by: nextflow-summary-to-galaxy-interface + observed_in: + mold: nextflow-summary-to-galaxy-interface + path: content/molds/nextflow-summary-to-galaxy-interface/index.md + revision: 6 + content_hash: 82d8... + foundry_head: d9e0b09 + subject: + kind: research + label: nextflow-to-galaxy-channel-shape-mapping + locator: content/research/nextflow-to-galaxy-channel-shape-mapping/index.md + content_hash: a18c... + kind: gap + severity: major + what: "The note does not cover optional outputs." + expected: "State how an absent optional channel maps to the Galaxy interface." + evidence: "Interface design stopped at the optional output decision." + status: open + issue: null +``` + +Entry `kind` is `gap`, `defect`, `friction`, or `wish`. Severity is `blocker`, `major`, or +`minor`. Status is `open`, `filed`, `duplicate`, or `wontfix`. + +## Run lifecycle + +For a pipeline run, the harness creates the file before phase 1 with `run.status: running`, an +empty `entries` list, and the complete top-level phase roster from its assembly record. All +phases begin `pending`. The harness changes a phase to `running` immediately before invocation +and to `done` on success. A phase that terminates the run and the run itself become `failed`; a +graceful user stop makes the run `cancelled`. A hard interruption naturally leaves the run +`running`, which is distinguishable from success without a recovery write. + +A phase also carries `feedback_checked`. The harness sets it true only when that phase reported a +feedback outcome — entry ids it appended, or an explicit `no feedback`. It is absent or false +otherwise, including for a phase that simply never said. This is what separates a run that looked +for friction and found none from a run that never looked. + +Loop phases remain one row and increment `iterations` after each successful iteration. Branch +phases remain one row and record the chosen path in `selected`. After every intended phase is +done, the harness sets `run.status: complete`. + +For a standalone skill invocation, the first feedback-aware skill creates a ledger with +`pipeline: null`, the caller's run slug when one exists, and one phase row for itself. It owns +that row's lifecycle as well as any entry it appends. + +No ledger means feedback mode was off. An empty `entries` list is evidence of a clean run only +when `run.status` is `complete` **and** every phase is `feedback_checked`. A complete run with +unchecked phases is complete but unreviewed; an empty incomplete ledger is neither. + +## When to append + +Appending is a deliberate pass, not a background reflex. Before a skill reports completion it +recalls what actually happened during its work — where it guessed at something the instructions +should have settled, needed information its bundle does not carry, hit an instruction that +contradicted another or contradicted the artifacts in front of it, used a reference that did not +cover the case, or did something its procedure never describes. Those are recallable events. "Was +anything unclear?" is not a useful question to ask at that moment and reliably answers itself no. + +The pass ends in one of two outcomes, and the outcome is always stated: entries appended, or +`no feedback`. A skill that says nothing has not reported a clean pass. + +Two entry shapes are admissible. Both require that `what` names a concrete problem, `expected` +states a proposed correction, and the entry is not already present in this ledger for the same +subject and correction. + +**A Foundry source asset.** The observation concerns Foundry-authored content, implementation, or +a registry that could be changed directly, and `subject.locator` canonically identifies it. This +is the ordinary case and the one the clustering and hash checks are built for. + +**A related project.** The run showed that a project the Foundry drives or cites is at fault — +Galaxy, a Tool Shed wrapper, the IWC corpus, `gxwf`, planemo. Use `subject.kind: related-project`, +name the project and the specific component in `label`, and put its public identity in +`locator` (a repository URL, a tool id, an IWC workflow id). There is no `content_hash`. State +in `what` and `evidence` what the run actually observed, and in `expected` what the upstream +would have to do. The entry is still filed against the Foundry. + +Do not use feedback entries for ordinary workflow requirements, tool-selection uncertainty, or +an unsupported preference with nothing concrete to change. + +## Canonical identity and provenance + +For a Foundry subject, use a repository-relative authored path as `subject.locator`; for a +package-export schema, use `package://#`. Generated skills and harnesses are +never subjects; identify the Mold, Pipeline, caster, assembler, or registry that must change. + +Initial subject kinds are `mold`, `pipeline`, `pattern`, `source-pattern`, `research`, `schema`, +`prompt`, `cli-tool`, `cli-command`, `meta`, `implementation`, and `registry`, plus +`related-project` for the upstream case above. `label` is for display only; the locator is the +identity and clustering key. + +A `related-project` subject is the one kind whose locator is not a Foundry path: it is public +upstream identity, carries no content hash, and is never checked against Foundry main. + +For the running Mold, copy `mold.name`, `mold.path`, `mold.revision`, and `mold.content_hash` +from `_provenance.json` into `observed_in`. Record its `mold.commit` value as `foundry_head`, but +never use that commit as exact content identity: it can be null or predate the bytes in the cast. +For a packaged reference, copy its `refs[].src` and `refs[].src_hash` into the subject locator and +content hash. When the subject is the Mold itself, use its authored path and +`observed_in.content_hash`. + +## Safe updates + +Read the complete ledger immediately before changing it. Preserve run state and all entries you +do not own. Append a complete entry in one rewrite; do not leave a partial YAML document. The +harness is the sole writer of pipeline lifecycle fields, while each skill writes only entries. + +Keep evidence concise and safe to upstream. Never copy credentials, tokens, private repository +URLs, proprietary source text, patient or participant data, or user-identifying filesystem +paths into the ledger. Replace a run-directory prefix with `/` and summarize sensitive +evidence instead of quoting it. + +The reporting skill may set an open entry to `filed`, `duplicate`, or `wontfix` and record the +resolved issue URL after the corresponding disposition is confirmed. It never deletes an entry; +the ledger remains the run audit trail. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_provenance.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_provenance.json new file mode 100644 index 0000000..f9465b4 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_provenance.json @@ -0,0 +1,226 @@ +{ + "provenance_schema_version": 4, + "cast_target": "claude", + "mold": { + "name": "workflow-brief-to-galaxy-data-flow", + "path": "content/molds/workflow-brief-to-galaxy-data-flow/index.md", + "revision": 4, + "content_hash": "ce9672837d2b670a64e838b2dbe634b3b07aec40d172d01fb3a7504b6d7294cd", + "commit": "94ce4ea20107680b467d03e8889b63689774e325" + }, + "cast_at": "2026-11-10T10:05:13.149Z", + "refs": [ + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-collection-patterns]]", + "src": "content/patterns/galaxy-collection-patterns.md", + "dst": "references/patterns/galaxy-collection-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Ground collection-shape choices in curated, corpus-observed operation and recipe patterns.", + "trigger": "When selecting collection cleanup, reshape, identifier, or collection-tabular bridge patterns.", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-conditionals-patterns]]", + "src": "content/patterns/galaxy-conditionals-patterns.md", + "dst": "references/patterns/galaxy-conditionals-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Ground conditional-branch and optional-step choices in curated, corpus-observed Galaxy when/pick_value patterns.", + "trigger": "When data-flow translation needs optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-interval-patterns]]", + "src": "content/patterns/galaxy-interval-patterns.md", + "dst": "references/patterns/galaxy-interval-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Ground genomic-interval operation choices in curated, corpus-observed Galaxy interval recipes.", + "trigger": "When the workflow operates on genomic intervals (BED/GFF/VCF coordinate features) and data-flow translation needs overlap, merge, coverage, windowing, masking, or set-algebra steps.", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-tabular-patterns]]", + "src": "content/patterns/galaxy-tabular-patterns.md", + "dst": "references/patterns/galaxy-tabular-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Ground tabular bridge and table-operation choices in curated, corpus-observed operation patterns.", + "trigger": "When data-flow translation needs filtering, joining, aggregation, pivoting, or tabular-collection bridges.", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-data-flow-draft-contract]]", + "src": "content/research/galaxy-data-flow-draft-contract/index.md", + "dst": "references/notes/galaxy-data-flow-draft-contract.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Keep the data-flow brief separate from gxformat2 templating and concrete step implementation.", + "verification": "Promote after two worked brief-to-Galaxy translations preserve this Mold boundary without moving fields.", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-sample-sheet-collections]]", + "src": "content/research/galaxy-sample-sheet-collections/index.md", + "dst": "references/notes/galaxy-sample-sheet-collections.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Preserve per-row metadata on the data-flow side: keep sample_sheet column_definitions wired through identifier-keyed steps instead of dropping into parallel parameter inputs, and re-attach metadata after map-over steps that lose it.", + "trigger": "When the interface brief carries a sample_sheet[:paired|:paired_or_unpaired|:record] input, or supporting evidence describes per-sample/per-record metadata that must survive map-over steps.", + "src_hash": "98736fdcd0c4f4698af4e0cd4c539b33e3d0db872de1e10487556b21033030ec", + "dst_hash": "98736fdcd0c4f4698af4e0cd4c539b33e3d0db872de1e10487556b21033030ec", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[open-requirements-ledger]]", + "src": "content/research/open-requirements-ledger/index.md", + "dst": "references/notes/open-requirements-ledger.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's wiring and collection decisions settle, and append data-flow obligations the narrative leaves open — a described transformation with no named tool, an implied intermediate the source never states, an ordering the prose doesn't fix.", + "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[workflow-brief-design]]", + "src": "content/research/workflow-brief-design/index.md", + "dst": "references/notes/workflow-brief-design.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Keep reviewed intent authoritative while using source summaries only as supporting evidence.", + "verification": "Confirm a worked run preserves scope and the brief's bytes while resolving Galaxy data-flow choices.", + "src_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "dst_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[workflow-brief-schema]]", + "src": "package://@galaxy-foundry/gxwf-foundry#workflowBriefSchema", + "dst": "references/schemas/workflow-brief-schema.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Carry the required implementation-contract shape into this design step.", + "src_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "dst_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "source": "deterministic" + } + ], + "artifacts": { + "produces": [ + { + "id": "workflow-brief-galaxy-data-flow", + "kind": "markdown", + "default_filename": "workflow-brief-galaxy-data-flow.md", + "description": "Reviewable Markdown brief: abstract operations, collection map/reduce choices, shape-changing placeholder steps, unresolved Galaxy tool needs, confidence, open questions." + }, + { + "id": "open-requirements-ledger", + "kind": "yaml", + "default_filename": "open-requirements.ledger.yml", + "description": "Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact." + } + ], + "consumes": [ + { + "id": "workflow-brief", + "description": "Expert-reviewed implementation contract governing scientific scope, requirements, and acceptance criteria.", + "inherited_schema": "[[workflow-brief-schema]]", + "producers": [ + "freeform-summary-to-workflow-brief", + "nextflow-summary-to-workflow-brief" + ] + }, + { + "id": "freeform-summary", + "description": "Optional narrative source evidence retained from brief production; consult it for details within the selected scope.", + "optional": true, + "producers": [ + "interview-to-freeform-summary", + "summarize-paper" + ] + }, + { + "id": "summary-nextflow", + "description": "Optional structured Nextflow source evidence; consult it without treating source implementation as Galaxy design.", + "optional": true, + "inherited_schema": "[[summary-nextflow]]", + "producers": [ + "summarize-nextflow" + ] + }, + { + "id": "workflow-brief-galaxy-interface", + "description": "Preceding Galaxy interface brief from [[workflow-brief-to-galaxy-interface]] that pins inputs, outputs, and labels.", + "producers": [ + "workflow-brief-to-galaxy-interface" + ] + }, + { + "id": "open-requirements-ledger", + "description": "Carried workflow-run knowledge [[open-requirements-ledger]] initialized by the interface Mold.", + "producers": [ + "advance-galaxy-draft-step", + "apply-galaxy-workflow-changeset", + "compare-against-iwc-exemplar", + "cwl-summary-to-galaxy-data-flow", + "cwl-summary-to-galaxy-interface", + "cwl-summary-to-galaxy-template", + "freeform-summary-to-galaxy-data-flow", + "freeform-summary-to-galaxy-interface", + "freeform-summary-to-galaxy-template", + "implement-galaxy-tool-step", + "interview-to-galaxy-workflow-changeset", + "mature-galaxy-workflow-for-iwc", + "nextflow-summary-to-galaxy-data-flow", + "nextflow-summary-to-galaxy-interface", + "nextflow-summary-to-galaxy-reference-data", + "nextflow-summary-to-galaxy-template", + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" + ] + } + ] + } +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_verify.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_verify.json new file mode 100644 index 0000000..f40b330 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/_verify.json @@ -0,0 +1,31 @@ +{ + "verify_schema_version": 1, + "entries": [ + { + "artifact_id": "summary-nextflow", + "direction": "input", + "kind": "json", + "default_filename": "summary-nextflow.json", + "schema": "[[summary-nextflow]]", + "validator_bin": "foundry", + "args": [ + "validate-summary-nextflow", + "{artifact_path}" + ] + }, + { + "artifact_id": "workflow-brief", + "direction": "input", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "validator_bin": "foundry", + "args": [ + "validate-workflow-brief", + "--schema-sha256", + "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "{artifact_path}" + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md new file mode 100644 index 0000000..853abc2 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/galaxy-data-flow-draft-contract.md @@ -0,0 +1,131 @@ +--- +type: research +title: "Galaxy data-flow draft contract" +tags: + - target/galaxy +status: draft +created: 2026-05-02 +revised: 2026-09-23 +revision: 4 +related_notes: + - "[[nextflow-to-galaxy-channel-shape-mapping]]" + - "[[nextflow-operators-to-galaxy-collection-recipes]]" + - "[[galaxy-workflow-draft]]" +related_molds: + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" + - "[[advance-galaxy-draft-step]]" +sources: + - "https://github.com/galaxyproject/foundry/issues/54" +summary: "Defines the proposed boundary between Galaxy data-flow drafts, gxformat2 templates, and concrete step implementation." +--- + +# Galaxy Data-Flow Draft Contract + +This is an architectural contract, not a schema. Evidence is strongest for Mold and Pipeline boundaries. Proposed fields are speculative until exercised by two or three worked translations. + +## Boundary + +The data-flow draft owns a target-shaped abstract DAG for Galaxy. It should not be valid `gxformat2` and should not resolve exact Tool Shed tools. + +Data-flow draft owns: + +- Galaxy-facing workflow inputs and outputs. +- Abstract nodes, edges, branches, collection mapping, collection reduction, and placeholder transformations. +- Input/output shape decisions such as `File`, `list`, `paired`, `list:paired`, or `list:list`. +- Conceptual Galaxy idioms: map-over, reduction, Apply Rules, collection cleanup, identifier synchronization, tabular bridge, fan-in / combine (concatenate, merge, or union N sources into one — a distinct node, not a side-effect of a reshape). +- Abstract unresolved tool needs with input and output shapes. +- Confidence and rationale on inferred nodes, edges, transforms, and tool needs. + +The Galaxy template owns: + +- A `gxformat2` skeleton. +- Ordered steps, labels, workflow inputs, workflow outputs, and the producer→consumer connection graph. +- Per-step wrapper resolution to the tier its evidence supports — **evidence-gated, not source-gated** (tiers defined in [[galaxy-workflow-draft-format]]): a fully **Resolved** step (concrete `tool_id`, changeset, `state`) when wrapper and parameters are pinned jointly; **Identity-pinned** (`tool_id` only, `_plan_state` kept) when the wrapper is named but not its settings; **Deferred** (`tool_id: TODO`, full `_plan_*`) otherwise. +- Placeholder collection-operation or Apply Rules steps only when the data-flow draft says they are necessary. +- `_plan_*` handoff units (intent, evidence, constraints) for every step left short of Resolved. + +Concrete step implementation owns: + +- For every step the template left short of **Resolved**: the changeset (`tool_version`, `tool_shed_repository` owner/repository metadata), bound `state` parameters, and `input_connections` — resolving any `tool_id: TODO` and confirming or correcting any `tool_id` the template pinned. Steps the template already Resolved pass through — confirm, do not re-derive. +- Concrete built-in collection-operation steps and parameters. +- Validation with `gxwf` and repair after schema/lint failures. + +## Proposed Body-Level Contract + +Do not add these as frontmatter fields yet. + +| Concept | Status | Notes | +|---|---|---| +| `source_summary_ref` | Speculative | Reference to the source summary consumed by the Mold. | +| `workflow_inputs` | Speculative | Abstract Galaxy-facing inputs and collection types. | +| `workflow_outputs` | Speculative | Intended outputs and source-summary provenance. | +| `nodes` | Speculative | Abstract operations, not concrete Galaxy tools. | +| `edges` | Speculative | Data dependencies, shape before/after, and source evidence. | +| `galaxy_idioms` | Speculative | Map-over, reduction, Apply Rules, collection filter, tabular bridge, etc. | +| `unresolved_tool_needs` | Speculative | Per abstract step needs for `discover-shed-tool` or `author-galaxy-tool-wrapper`. | +| `placeholder_transformations` | Speculative | Shape/text/table transforms needed for Galaxy semantics but not concretely implemented. | +| `confidence` | Speculative | Prefer `high`, `medium`, `low` plus rationale. | +| `handoff_notes` | Speculative | Instructions to template, exemplar comparison, and per-step Molds. | +| `open_questions` | Speculative | Semantic/tooling issues carried forward. | + +## Handoff Examples + +Unresolved tool need: + +- [[nextflow-summary-to-galaxy-data-flow]] emits an abstract node such as `trim FASTQ reads`, input `list:paired fastq`, output `list:paired fastq`, tool need `read trimming`, confidence `medium`. +- [[nextflow-summary-to-galaxy-template]] creates a placeholder step with TODOs and collection-shaped connections. +- The harness routes through tool discovery or wrapper authoring. +- [[implement-galaxy-tool-step]] fills exact Galaxy tool metadata, parameters, and connections. + +Collection cleanup after fan-out: + +- Data-flow records a conceptual cleanup transform after a mapped step. +- Template materializes a placeholder collection-operation step. +- Implementation chooses exact built-in tool and parameters. + +Identifier-derived reshaping: + +- Data-flow records desired input shape, output shape, and identifier transformation. +- Template emits an Apply Rules placeholder only if needed. +- Implementation fills concrete rule JSON after exemplar comparison confirms the shape. + +IWC exemplar comparison: + +- Data-flow and template should hand [[compare-against-iwc-exemplar]] the abstract topology, placeholder transformations, unresolved tool needs, and confidence notes. +- Exemplar comparison should flag structural divergence, not resolve tools. + +## Confidence Guidance + +- Attach confidence to the smallest useful unit: node, edge, transformation, or tool need. +- Use qualitative `high`, `medium`, `low` until examples justify a richer schema. +- Require rationale for low confidence. +- Do not reuse source-summary `warnings[]` as data-flow confidence. +- Keep evidence quality distinct from translation confidence. A claim can be corpus-observed but still low-confidence for a specific workflow. + +## Risks + +If the data-flow draft is too broad, it duplicates the template Mold, makes premature tool decisions, and leaks harness routing into Mold output. + +If it is too narrow, the template Mold receives source-summary details without Galaxy-shaped semantics, exemplar comparison has only a surface skeleton to diff, and tool discovery loses input/output shape context. + +## Evidence + +- [[nextflow-to-galaxy-channel-shape-mapping]] and [[nextflow-operators-to-galaxy-collection-recipes]] show why a Galaxy-shaped abstraction is needed between source summary and `gxformat2` template. +- `content/pipelines/nextflow-to-galaxy.md` places data-flow before template, exemplar comparison, and per-step implementation. +- `content/molds/summarize-nextflow/index.md` says source summarization should not produce Galaxy data flow. +- `content/molds/implement-galaxy-tool-step/index.md` owns concrete step implementation. + +## TODOs + +- Decide whether to author `galaxy-data-flow-draft.schema.json` now or wait for worked examples. +- Decide whether confidence should be a single enum or per-axis vector. +- Decide how much ordering guidance belongs in data-flow versus template. +- Decide whether built-in collection operations should be abstract placeholders or concrete template steps. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/galaxy-sample-sheet-collections.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/galaxy-sample-sheet-collections.md new file mode 100644 index 0000000..1dca0da --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/galaxy-sample-sheet-collections.md @@ -0,0 +1,143 @@ +--- +type: research +title: "Galaxy sample_sheet collection types" +tags: + - target/galaxy +status: draft +created: 2026-05-05 +revised: 2026-05-06 +revision: 2 +related_notes: + - "[[galaxy-collection-semantics]]" + - "[[galaxy-collection-tools]]" + - "[[nextflow-workflow-io-semantics]]" + - "[[nextflow-params-to-galaxy-inputs]]" + - "[[nextflow-path-glob-to-galaxy-datatype]]" + - "[[nextflow-to-galaxy-channel-shape-mapping]]" + - "[[nextflow-to-galaxy-reference-data-mapping]]" +related_molds: + - "[[nextflow-summary-to-galaxy-interface]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" +sources: + - "Galaxy PR #19305 (Implement Sample Sheets), merged 2025-07-30" + - "lib/galaxy/model/dataset_collections/types/sample_sheet.py" + - "lib/galaxy/model/dataset_collections/types/sample_sheet_util.py" + - "lib/galaxy/model/dataset_collections/type_description.py" + - "lib/galaxy/schema/schema.py (SampleSheetColumnDefinition, SampleSheetRow)" + - "lib/galaxy/tools/wrappers.py (DatasetCollectionWrapper.sample_sheet_row)" + - "lib/galaxy/tools/sample_sheet_to_tabular.xml" + - "lib/galaxy/webapps/galaxy/api/dataset_collections.py (sample_sheet_workbook endpoints)" + - "lib/galaxy/model/migrations/alembic/versions_gxy/3af58c192752_implement_sample_sheets.py" +summary: "Galaxy's sample_sheet collection family: typed column metadata, four variants, mapping rules, validator allowlist." +--- + +# Galaxy sample_sheet collection types + +Reference for the Galaxy backend shape that targets structured per-row metadata — the natural landing zone for Nextflow `samplesheetToList` parameters and for any source-side idiom that pairs typed metadata columns with dataset references. + +## Shape + +A `sample_sheet` is a list-shaped collection where each element carries a typed `columns` row, and the parent collection carries a `column_definitions` schema. The model (`lib/galaxy/model/__init__.py`) adds two nullable JSON columns: `dataset_collection.column_definitions` and `dataset_collection_element.columns`. Existing collections are unaffected. + +``` +DatasetCollection + collection_type: "sample_sheet" + column_definitions: [{name, type, optional, ...}] + elements: + DatasetCollectionElement(element_identifier, columns: [v1, v2, ...], hda or child_collection) +``` + +## Column definition vocabulary + +`SampleSheetColumnDefinition` (lib/galaxy/schema/schema.py:384-405): + +``` +type: "string" | "int" | "float" | "boolean" | "element_identifier" +optional: bool +default_value, validators[], restrictions[], suggestions[] +``` + +- `element_identifier` cross-references another row's identifier in the same collection. Captures patterns like nf-core `control_sample`. Within-collection only — no cross-collection refs. +- `restrictions[]` enumerates allowed values (maps to nf-schema `enum`). +- `validators[]` is restricted to the **safe allowlist** (`AnySafeValidatorModel`, lib/galaxy/tool_util_models/parameter_validators.py): `regex`, `in_range`, `length`. `expression` and other arbitrary-Python validators are rejected. Translation from richer nf-schema validation must drop to prose or fall through. +- String values and column names are restricted to `[\w\-_ ?]*` — no tabs, newlines, quotes, or most specials. Necessary for safe CSV/TSV serialization. + +## Variants + +Enforced by `COLLECTION_TYPE_REGEX` in `type_description.py`: + +| Variant | Inner element shape | Typical source | +|---|---|---| +| `sample_sheet` | one dataset per row | one path column in nf-schema | +| `sample_sheet:paired` | `paired` (forward/reverse) | two path columns, paired-end fastq | +| `sample_sheet:paired_or_unpaired` | mixed single or pair | mixed-library sample sheets | +| `sample_sheet:record` | `record` (named typed slots) | heterogeneous per-sample artifacts | + +Constraints to plan around: + +- `sample_sheet` must be **outermost**. `list:sample_sheet` and `sample_sheet:list` are invalid. +- `prototype_elements()` is not implemented — sample-sheet-shaped pre-creation isn't possible because element count is unknown until data exists. + +## Mapping rules + +`allow_implicit_mapping` is `True` for everything except `record`, so sample sheets behave like lists for mapping: + +- `sample_sheet` over a `(dataset → dataset)` tool produces an implicit `sample_sheet`-shaped output. **`column_definitions` and `columns` are not propagated** — metadata lives only on the input collection. +- `sample_sheet:paired` over a `paired` input produces a `sample_sheet`-shaped output (one job per row). +- A `sample_sheet` can be reduced by a `multiple=true` data input. +- Two sample sheets with identical structure can be linked for dot-product execution. + +## Tool-side access + +Tools read columns at runtime through `DatasetCollectionWrapper.sample_sheet_row(element_identifier)` (lib/galaxy/tools/wrappers.py:643-707), which builds a `{element_identifier: row}` dict from element `columns`. The built-in `__SAMPLE_SHEET_TO_TABULAR__` tool (lib/galaxy/tools/sample_sheet_to_tabular.xml) iterates and tab-joins for downstream tabular consumers. Tool inputs declare acceptance via `data_collection collection_type="sample_sheet,sample_sheet:paired,..."`. + +`DataCollectionToolParameter` carries `column_definitions` through `to_dict()`, which lets the workflow editor render column-definition forms for sample-sheet inputs. + +## API surfaces + +- `POST /api/dataset_collections` accepts `column_definitions` and `rows` (a `{element_identifier: [values]}` dict). +- `POST /api/tools/fetch` carries `column_definitions` at target level and per-element `row` for remote-URI sample-sheet creation. +- Workbook endpoints (`/api/sample_sheet_workbook[/parse]`, `/api/dataset_collections/{id}/sample_sheet_workbook[/parse]`) generate and parse XLSX/CSV/TSV using openpyxl, with header rows, dropdowns for `restrictions`, type validation, and an instructions sheet. + +## Workflow integration + +`InputCollectionModule` (lib/galaxy/workflow/modules.py) validates `column_definitions` at workflow save and propagates them to the runtime form. Galaxy workflow YAML accepts: + +```yaml +inputs: + chipseq_data: + type: collection + collection_type: sample_sheet:paired + column_definitions: + - {type: string, name: condition, optional: false} + - {type: int, name: replicate, optional: false} + - {type: element_identifier, name: control_sample, optional: true} +``` + +This is the structured-input lever a Nextflow → Galaxy translation reaches for when source pipelines use `samplesheetToList(params.input, "assets/schema_input.json")`. + +## Mapping from Nextflow sample sheets + +| Nextflow / nf-schema | Galaxy `sample_sheet*` | +|---|---| +| `samplesheetToList(params.x, schema)` materialization | `sample_sheet` collection input bound to `params.x` | +| nf-schema column `type: string\|integer\|number\|boolean` | column `type: string\|int\|float\|boolean` | +| Column annotated `meta:` (carried in `meta` map) | non-path column in `columns[]` | +| Path-typed column (one per row) | element dataset (variant `sample_sheet`) | +| Two path columns (fastq pair) | inner `paired` (variant `sample_sheet:paired`) | +| Mixed single/paired rows | inner `paired_or_unpaired` | +| Heterogeneous typed per-row artifacts | inner `record` (variant `sample_sheet:record`) | +| nf-schema `enum` | `restrictions[]` | +| nf-schema `pattern` | `regex` validator | +| nf-core `control_sample` style cross-row reference | `element_identifier` column type | + +Edges to flag, not silently squash, when translating: + +- nf-schema validation richer than regex/in_range/length cannot be expressed; downgrade to prose with confidence note. +- Cross-sample-sheet references have no Galaxy expression — keep as plain `string` and warn. +- Ad-hoc `splitCsv(header: true)` parsing without an `assets/schema_*.json` produces no column types — translation needs ad-hoc inference; this note's mapping does not apply. +- Output side has no automatic propagation: a tool mapped over a `sample_sheet` produces a `sample_sheet`-shaped collection without `column_definitions`. Carry-forward must be explicit (re-attaching metadata via `__SAMPLE_SHEET_TO_TABULAR__` or rules DSL `add_column_from_sample_sheet_index`). + +## Why this matters for the Foundry + +Without sample_sheet shape awareness, `nextflow-summary-to-galaxy-interface` defaults nf-core sample-sheet inputs to `list:paired` or a flat file input, dropping per-row metadata. Surfacing the column schema lets the interface Mold pick the right variant and lets the data-flow Mold preserve `meta` fields through identifier-keyed wiring instead of inventing parallel parameter inputs. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/open-requirements-ledger.md new file mode 100644 index 0000000..761183e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/open-requirements-ledger.md @@ -0,0 +1,231 @@ +--- +type: research +title: "Open-requirements ledger" +tags: + - target/galaxy +status: draft +created: 2026-06-16 +revised: 2026-11-10 +revision: 7 +related_notes: + - "[[galaxy-workflow-draft-format]]" +related_molds: + - "[[advance-galaxy-draft-step]]" + - "[[repair-galaxy-draft-topology]]" + - "[[implement-galaxy-tool-step]]" +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." +--- + +# Open-requirements ledger + +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. + +## Framing: obligations the pipeline discharges, not questions a human answers + +This is deliberately *not* an "open questions for the user" list. The pipeline is autonomous — no human-in-the-loop gate is assumed. The ledger's consumers are **Molds and the loop's convergence gate**, with human readout a secondary affordance. An entry is closed by a downstream Mold doing work (wiring a producer, picking a wrapper, settling a value), or — when nothing can close it — surrendered: written into the final draft as a known, labelled gap rather than silently dropped or fabricated around. + +The distinction matters because a "questions for a human" framing leaks an operator's personal interaction style into a tool meant to run inside anyone's harness. The ledger must behave identically cast into a fresh harness with no ambient configuration; that holds only because every obligation is **materialized in the artifact**, never carried as operator habit. + +## Entry shape (v1, loose) + +Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries are intentionally low-ceremony for v1 — enough structure to count and close, no contract pretending to be machine-parameterizable yet. A reasonable per-entry shape: + +```yaml +- id: sccmec-evidence-missing + status: open # open | resolved | surrendered + kind: gap # gap | dropped — see below; omit for gap + blocking: true # true only when a structural edit closes it; the convergence gate counts these + raised_by: implement-galaxy-tool-step # Mold that appended it + step: classify_context # draft step the obligation attaches to (if any) + unmet: "SCCmec-region candidate output category" + missing: "no wired input carries SCCmec cassette evidence" + resolved_by: null # Mold that closed it, once closed + supersedes: null # id whose justification this entry refutes, once resolved + note: "" # how it was closed or why surrendered +``` + +`kind` separates the two things an open entry can mean, which v1 conflated. A **`gap`** is +an obligation the pipeline cannot discharge: no producer is discoverable, Galaxy has no +datatype for the format, the source pinned no value. A **`dropped`** entry is source work +the pipeline *declined to carry* — a plane cut from the design, a parameter surface not +exposed, an output the repair removed. Both are honest; they are not the same admission, +and a reader cannot tell them apart from `unmet` / `missing` alone. Omit `kind` for a +plain gap. + +A `dropped` entry carries two more fields: + +```yaml +- id: annotation-planes-not-built + status: open + kind: dropped + raised_by: nextflow-summary-to-galaxy-data-flow + units: "25 of 31 designed nodes (rnaseq_short, gnomon, annot_proc, convert)" + because: "`gxwf tool-search asn1` returns no wrapper reading seq-submit; searched 2026-08-29" + unmet: "the pipeline's actual product — an annotated genome" + missing: "the carried spine masks an assembly; it does not annotate a genome" + note: "" +``` + +`units` names what was dropped in the source's own terms, so the drops can be totalled +without re-reading the design. + +`because` says why, and **must carry its citation** — the source construct that is not +there, the target limitation named specifically enough that someone could check it, the +search actually run. A drop's reason is a claim about the world, and the first real run is +why the burden exists: 25 of 31 planes were cut because "each needs an authored Galaxy +wrapper for an undocumented binary available only inside the pipeline's container," and a +Tool Shed search one phase later found the whole pipeline already wrapped at the exact +pinned version. The claim was not dishonest. It was never checked, and nothing in the +entry showed that it had not been. + +A reason you can cite, cite. A reason you cannot cite is still recorded — write it plainly +and leave it uncited rather than dressing it as evidence — but an uncited drop is not a +finding the run may reason from afterwards. It is a debt, and it should read as one. + +Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. + +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. + +## How to use the ledger + +This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. + +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. + +**Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. + +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. + +**Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. + +**Carry the rest untouched.** Entries you neither raise nor close pass through unchanged, statuses and provenance intact. Never renumber, re-word, or drop another Mold's entry. + +**Never fabricate around an entry.** An open obligation is the honest state. Inventing a connection, a tool id, or a value so the artifact looks complete converts a tracked gap into a silent defect — the specific failure the ledger replaces. + +**Record a drop as a drop.** When a decision removes source work from what the run will +carry — narrowing a design, cutting a plane, dropping an output during repair — append a +`kind: dropped` entry naming the `units` and a cited `because`. A cut recorded only as +prose in a brief is invisible to every later Mold; a cut recorded as a `gap` reads as +something the pipeline could not do rather than something it chose not to do. Recording +the drop is not the same as discharging it, and a well-written entry is not a substitute +for the work — an entry makes a cut *legible*, never *justified*. + +**A refuted justification reopens its decision.** When work you do contradicts the +`because` another entry rests on — discovery finds the wrapper a drop assumed absent, a +later brief supplies the evidence a narrowing assumed missing — set `supersedes` to that +entry's `id` on your own resolved entry, and return the superseded entry to `status: open` +with its `because` struck and the contradiction quoted in its `note`. Do not leave a +refuted rationale standing: the terminal writes these into the final artifact, and a +surrendered gap whose stated reason is known to be false is worse than an unexplained one. +Reopening does not oblige *you* to do the dropped work — it obliges the run to stop +claiming a settled reason it no longer has. + +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. + +**Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. + +## Role in topology repair + +In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): + +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. + +So the ledger is not a convenience here; it is the countable state the termination guard depends on. + +## Escalation budget (loop-level state) + +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: + +```yaml +topology_repair: + escalations: 2 # repairs invoked this run + cap: 5 # hard ceiling; at cap, still-open entries are surrendered + open_history: [4, 3, 2] # open blocking-entry count after each escalation +entries: + - id: sccmec-evidence-missing + status: open + blocking: true + # … per-entry shape above +``` + +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. +- `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. + +This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. + +## Threaded through the whole design tier + +The ledger is not loop-only. Every design-tier Mold in the source→Galaxy pipelines **consumes and re-emits** it — the interface, reference-data, data-flow, IWC-comparison, and template Molds (`*-summary-to-galaxy-interface`, `*-summary-to-galaxy-data-flow`, `*-summary-to-galaxy-reference-data`, `compare-against-iwc-exemplar`, `*-summary-to-galaxy-template`), alongside the change-set Molds on the update path. This is the direct answer to #281: the chain stops re-deriving the same unknowns because each Mold inherits them. + +The template Mold's computability review pass is the notable appender ahead of the loop — `*-summary-to-galaxy-template` re-reads each settled step and, where an output needs evidence no input carries, wires the producer or records the gap as a blocking entry. Source-summary Molds upstream of the design tier keep their own free-text open-questions; the design tier is where those formalize into the ledger. + +## Open work + +- Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. +- Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. +- Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. +- Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. +- Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. +- Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/workflow-brief-design.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/workflow-brief-design.md new file mode 100644 index 0000000..8bc1e6d --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/notes/workflow-brief-design.md @@ -0,0 +1,96 @@ +--- +type: research +title: "Workflow Brief" +tags: + - meta +status: draft +created: 2026-09-17 +revised: 2026-10-19 +revision: 6 +summary: "Markdown Workflow Brief definition and section contract for scope, constraints, agent environment, and explicit blockers." +--- + +# Workflow Brief + +A Workflow Brief is an expert-editable document describing what a workflow should achieve, what it should leave out, and the constraints under which an agent will build and run it. It is an input to design and implementation. The draft workflow captures the proposed steps, wiring, and tool state. + +## Document shape + +The brief is Markdown with a title and two sections: `## Workflow` and `## Agent Environment`. The workflow needs an objective; most details are optional. The agent environment must contain a description, which can explicitly record what still needs to be checked. + +The `workflowBriefSchema` declaration uses the shared engine described in [[markdown-document-contract]] and lives in `packages/gxwf-foundry/src/workflow-brief.ts`. An illustrative document lives in `packages/gxwf-foundry/test/fixtures/workflow-brief/read-alignment.md`. + +| Heading | Required | Content | +| --- | --- | --- | +| `## Workflow` | Yes | Scientific intent and scope. | +| `### Objective` | Yes, under Workflow | Scientific goal and desired outcome. | +| `### Sources` | No | Papers, interviews, repositories, relevant source pins, and evidence availability. | +| `### Scope` | No | Optional `#### Included` and `#### Excluded` describe selected work and boundaries. | +| `### Inputs and outputs` | No | Scientific roles and source-supported formats or organization. | +| `### Requirements and preferences` | No | Expert or source requirements, their strength, rationale, and evidence. | +| `### Acceptance criteria` | No | High-level behaviors that establish success. | +| `### Open questions` | No | Unresolved choices that do not prevent work, each with a proposed answer and its evidence. | +| `### Decisions` | No | Expert decisions and rationale. | +| `### Related artifacts` | No | Links to source evidence and other relevant inputs. | +| `### Blockers` | No, under Workflow | Unresolved issues that prevent design or implementation. Any content blocks work. | +| `## Agent Environment` | Yes | Agent tooling and operating conditions. | +| `### Tooling` | No | Expected Foundry tooling and versions, and what is known about availability. | +| `### Constraints` | No | Agent workspace, network, installation, resource, and access restrictions. | +| `### Containerization` | No | Container preference and Docker, Singularity, or Apptainer availability. | +| `### Blockers` | No, under Agent Environment | Environment issues that prevent work. Any content blocks work. | + +Headings are case-insensitive; section order is flexible and additional sections are allowed. Recognized sections are unique within their parent. If present, ordinary sections need content beyond headings, comments, or separators. Blockers may be empty. Scope's Included and Excluded subsections are optional. + +## Workflow content + +The writing agent must record evidence and expert intent without making Galaxy design choices. It must not assume Galaxy datatypes, dataset collection structures, tool availability, interface labels, or concrete tool versions. Source formats and explicitly named source tools can be cited as evidence; they do not establish a Galaxy mapping or an installed wrapper. An expert may supply specific Galaxy requirements here, and the agent must identify them as expert-provided rather than inferred. + +The agent must not write a step graph, concrete tool state, or final test declarations into the brief. Interface design, data-flow design, tool discovery, and test development belong to the downstream design and implementation Molds. Inputs and outputs stay at scientific roles and source-supported descriptions. Acceptance criteria stay high-level; detailed fixtures and assertions belong to test development. + +Requirements and preferences describe what the expert or source asks for. Agent Environment Constraints describe restrictions on the agent's work. Uncertainty that prevents proceeding belongs in Blockers. Record every other unresolved choice in Open questions with a proposed answer and the evidence for it, such as source behavior, the expert's wording, or the available test data. Propose the answer the evidence best supports. Where expert intent and source behavior conflict, propose the expert's intent and cite the conflict. When the evidence is thin, still propose an answer, mark it low confidence, and say what evidence is missing. The expert accepts a proposal by leaving it unedited and overrides it by editing the answer. + +Write each open question as a bullet with `Proposed:` and `Evidence:` sub-bullets. Add `Confidence: low` and `Missing:` when the evidence is thin: + +```md +- Should runs of one sample be merged before trimming? + - Proposed: yes, merge them as the source does. + - Evidence: the source merges sample-sheet rows sharing a sample ID. The selection does not mention merging. + - Confidence: low + - Missing: whether the expert's samples ever span several runs. +``` + +## Agent environment + +Tooling records whether expected tools such as `gxwf`, `foundry`, and `planemo` are available at the required versions. Distinguish intended requirements from observed availability. Containerization records the preference and whether Docker, Singularity, or Apptainer is actually usable, rather than assuming that an executable name proves a working engine. + +A separate preflight result should record observed versions, container availability, source/reference access, writable paths, timestamps, and diagnostics. A requested installation strategy is not evidence of an installed executable. + +## Validation and blockers + +```sh +foundry validate-workflow-brief workflow-brief.md +foundry check-workflow-brief workflow-brief.md --json +``` + +Validation checks the title, required sections, unique recognized headings, content, and parent-scoped subsections. Example headings in fences, quotes, lists, or comments do not satisfy requirements. Success exits `0`, structural failures exit `3`, and input read failures exit `1`. + +The static readiness check adds a check for `### Blockers` under both Workflow and Agent Environment. A structurally valid document with content in either section exits `4`; a structurally valid document without blockers exits `0`. Omit or leave Blockers empty when there are no blockers. Do not write “none” there: any content, including that word, is blocking. Comments and separators alone are empty. The JSON result includes `valid`, `ready`, `errors`, and blocker bodies with their parent and source line. + +A clear static check only establishes structure and the absence of declared blockers. It does not establish semantic completeness, expert approval, or successful environment preflight. The harness must stop on declared blockers and apply its own review and preflight checks before implementation. + +The exported `WorkflowBrief` and `WorkflowBriefSection` types describe the parsed document. `WorkflowBriefReadinessResult` describes the static check. These checks read the document without changing it. + +## Lifecycle + +1. A source-to-brief producer assembles evidence, proposed scope, and unanswered questions without Galaxy design assumptions. +2. The expert edits the brief and settles blockers needed before the next stage. +3. The harness checks structure, declared blockers, review, and environment readiness. +4. Brief-to-Galaxy consumes the brief and referenced evidence, then runs design, draft, implementation, and test development. + +Brief-to-Galaxy and every design or implementation step must leave the brief unchanged. Record durable workflow knowledge in `open-requirements.ledger.yml`: evidence as discoveries, authorized choices as decisions, unmet needs as obligations, and changes to expert intent as proposed brief-change recommendations. Ordinary phase progress remains harness state. Updating the brief belongs to a separate expert editing step; an implementation agent must not resolve a blocker by rewriting its input. + +A hand-authored brief is a first-class input. Implementation should work in a fresh session using the document and its referenced artifacts. + +## Open design questions + +- Should expert questions live here or link to a separate decision record? diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md new file mode 100644 index 0000000..b00b1d5 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-collection-patterns.md @@ -0,0 +1,94 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: collection patterns" +aliases: + - "Galaxy collection pattern MOC" + - "collection transformation patterns" + - "IWC collection pattern map" +tags: + - target/galaxy + - topic/galaxy-transform + - topic/collection-transform +status: draft +created: 2026-05-02 +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." +related_notes: + - "[[iwc-transformations-survey]]" + - "[[iwc-conditionals-survey]]" +related_patterns: + - "[[manifest-to-mapped-collection-lifecycle]]" + - "[[cleanup-sync-and-publish-nonempty-results]]" + - "[[reshape-relabel-remap-by-collection-axis]]" + - "[[fan-in-bundle-consume-and-flatten]]" + - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" + - "[[sync-collections-by-identifier]]" + - "[[harmonize-by-sortlist-from-identifiers]]" + - "[[regex-relabel-via-tabular]]" + - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" + - "[[collection-swap-nesting-with-apply-rules]]" + - "[[collection-split-identifier-via-rules]]" + - "[[collection-build-list-paired-with-apply-rules]]" + - "[[tabular-to-collection-by-row]]" + - "[[tabular-concatenate-collection-to-table]]" + - "[[tabular-pivot-collection-to-wide]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: collection patterns + +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. + +## Enter or leave a collection + +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. + +## Keep mapped siblings aligned + +First identify what changed. Membership, order, and labels are separate decisions. + +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. + +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. + +## Change the collection axes + +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. + +## Follow a map-over lifecycle + +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. + +## See also + +- [[iwc-transformations-survey]] — collection-transform survey and evidence trail. +- [[galaxy-tabular-patterns]] — companion MOC for tabular operations. +- [[galaxy-interval-patterns]] — companion MOC for coordinate-feature operations. +- [[galaxy-sequence-patterns]] — companion MOC for sequence-record operations. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md new file mode 100644 index 0000000..2dfe890 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-conditionals-patterns.md @@ -0,0 +1,62 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: conditionals patterns" +aliases: + - "Galaxy conditional pattern MOC" + - "Galaxy when patterns" + - "conditional workflow patterns" +tags: + - target/galaxy + - topic/galaxy-transform +status: draft +created: 2026-05-02 +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." +related_notes: + - "[[iwc-conditionals-survey]]" +related_patterns: + - "[[conditional-run-optional-step]]" + - "[[conditional-route-between-alternative-outputs]]" + - "[[conditional-gate-on-nonempty-result]]" + - "[[conditional-transform-or-pass-through]]" + - "[[collection-cleanup-after-mapover-failure]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: conditionals patterns + +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. + +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. + +## Skip work without replacing its output + +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. + +## Continue with one value + +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. + +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. + +## See Also + +- [[iwc-conditionals-survey]] — conditionals survey and evidence trail. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md new file mode 100644 index 0000000..de9ec07 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-interval-patterns.md @@ -0,0 +1,88 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: genomic interval patterns" +aliases: + - "Galaxy interval pattern MOC" + - "genomic interval transformation patterns" + - "IWC interval pattern map" +tags: + - target/galaxy + - topic/galaxy-transform + - topic/interval-transform +status: draft +created: 2026-06-10 +revised: 2026-09-23 +revision: 4 +summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." +related_notes: + - "[[iwc-interval-operations-survey]]" +related_patterns: + - "[[interval-overlap-filter]]" + - "[[interval-coverage]]" + - "[[interval-merge-overlapping]]" + - "[[interval-window-flank]]" + - "[[interval-consensus-by-multi-intersect]]" + - "[[interval-mask-by-set-algebra]]" + - "[[interval-windowed-coverage]]" + - "[[tabular-synthesize-bed-from-3col]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: genomic interval patterns + +The runtime-facing map for Galaxy **coordinate-feature** choices — operations that understand `chrom/start/end/strand`, as opposed to opaque-column [[galaxy-tabular-patterns]] or container-shaped [[galaxy-collection-patterns]]. Use it before loading raw survey notes; [[iwc-interval-operations-survey]] is the evidence backing, these pages are the actionable references. + +This is the smallest of the three data-shape MOCs by design. Interval algebra is a real but moderate cluster in IWC — concentrated in epigenetics peak-consensus and SARS-CoV-2 masking — and its highest-value units are **recipes**, not single operations. Reach for the recipes first when your need is a multi-step construction. + +## Overlap + +- [[interval-overlap-filter]] — keep/drop/annotate features by overlap with a second set (`bedtools intersect`, or VCF-native `vcfvcfintersect`). + +## Set operations + +- [[interval-merge-overlapping]] — collapse a set's own overlapping features into single spans (`bedtools merge` / `gops_merge`). +- [[interval-mask-by-set-algebra]] — *(recipe)* compute regions from regions: concat → merge → subtract (the `gops_*` set-algebra mask). + +## Windows & coverage + +- [[interval-window-flank]] — extend features into neighborhood windows (`bedtools slop`). +- [[interval-coverage]] — genome-wide depth (bedgraph) or reads-in-given-regions counts (`bedtools genomecov` / `coverage`). + +## Recipes + +- [[interval-consensus-by-multi-intersect]] — reproducible features across replicates: multi-intersect → count-threshold → intersect back. +- [[interval-mask-by-set-algebra]] — compute a mask: union (concat+merge) then difference (subtract). +- [[interval-windowed-coverage]] — quantify signal in fixed windows: slop → merge → coverage. + +## Bridges + +- **interval ↔ collection** — `bedtools` ops map over per-sample collections via `reduce_or_iterate: iterate`; multi-intersect consumes a whole collection at once. See [[galaxy-collection-patterns]]. +- **interval ↔ tabular** — coordinate *construction* ([[tabular-synthesize-bed-from-3col]], and the indel-aware variant-to-BED arithmetic in [[interval-mask-by-set-algebra]]) and coordinate *filtering* (`Filter1 c4 < N` on a bedgraph, [[tabular-filter-by-column-value]]). The line: tabular treats columns as opaque; interval understands coordinates. See [[galaxy-tabular-patterns]]. + +## Gaps (no corpus exemplar, no page) + +Per corpus-first, these have zero IWC uptake and get no pattern page; documented here so the absence is explicit, not an oversight. GTN training-corpus counts below are grounded in [[iwc-interval-operations-survey]] §GTN cross-reference (non-IWC signal): + +- **`closest` / proximity** — the "nearest feature + distance" operation. The **interval-algebra form** (`bedtools closest`, fetch-closest, `windowbed`) is absent from both IWC and GTN. The *task* is absent from IWC entirely; in GTN it appears only as a **domain step** (`deeptools computeMatrix` reference-point mode, TSS-relative signal), which #268 scopes out. This is the operation that motivated the MOC (#268). The natural tool — `bedtools_closestbed` (`-d` for distance) — exists in the same `iuc/bedtools` suite this MOC's other tools come from; it simply has no corpus exemplar, so per corpus-first there is no recurring pattern to author, and an agent that needs nearest-feature-plus-distance reaches for it directly. (`computeMatrix` covers only the narrower ChIP/ATAC TSS-proximity case, not a general substitute.) +- **`complement`** — taught in one GTN tutorial (assembly), zero IWC. +- **coordinate-aware `sort`** — taught across three GTN tutorials; IWC sorts intervals with tabular `sort1` instead. +- **`makewindows`, `map`, `window` (`-w` neighborhood join), `annotate`, `jaccard`** — zero in both corpora. + +These are tracked as IWC-input-blocked candidates (GitHub `requires-iwc-inputs`); a page follows only when an IWC workflow uses the operation. + +## See also + +- [[iwc-interval-operations-survey]] — interval-operation survey and evidence trail. +- [[galaxy-sequence-patterns]] — companion MOC for sequence-record operations; shares the extract/mask-by-region bridge (`getfasta`/`maskfasta` produce sequence from these intervals). +- [[galaxy-tabular-patterns]] — companion MOC for opaque-column tabular operations. +- [[galaxy-collection-patterns]] — companion MOC for collection-container operations. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md new file mode 100644 index 0000000..591b3f5 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/patterns/galaxy-tabular-patterns.md @@ -0,0 +1,82 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: tabular patterns" +aliases: + - "Galaxy tabular pattern MOC" + - "tabular transformation patterns" + - "IWC tabular pattern map" +tags: + - target/galaxy + - topic/galaxy-transform + - topic/tabular-transform +status: draft +created: 2026-05-02 +revised: 2026-10-15 +revision: 4 +summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." +related_notes: + - "[[iwc-tabular-operations-survey]]" +related_patterns: + - "[[tabular-filter-by-column-value]]" + - "[[tabular-filter-by-regex]]" + - "[[tabular-cut-and-reorder-columns]]" + - "[[tabular-compute-new-column]]" + - "[[tabular-join-on-key]]" + - "[[tabular-group-and-aggregate-with-datamash]]" + - "[[tabular-sql-query]]" + - "[[tabular-prepend-header]]" + - "[[tabular-synthesize-bed-from-3col]]" + - "[[tabular-split-taxonomy-string]]" + - "[[tabular-relabel-by-row-counter]]" + - "[[tabular-to-collection-by-row]]" + - "[[tabular-concatenate-collection-to-table]]" + - "[[tabular-pivot-collection-to-wide]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: tabular patterns + +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. + +## Rows and columns in one table + +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. + +## Combine or summarize tables + +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. + +## Text-processing recipes + +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. + +## Between tables and collections + +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. + +## See also + +- [[iwc-tabular-operations-survey]] — tabular-operation survey and evidence trail. +- [[galaxy-sequence-patterns]] — companion MOC for sequence-record operations; FASTA↔tabular is the dominant sequence seam. +- [[galaxy-interval-patterns]] — companion MOC for coordinate-feature operations. +- [[galaxy-collection-patterns]] — companion MOC for collection operations. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/schemas/workflow-brief-schema.schema.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/schemas/workflow-brief-schema.schema.json new file mode 100644 index 0000000..92c431e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-data-flow/references/schemas/workflow-brief-schema.schema.json @@ -0,0 +1,103 @@ +{ + "name": "Workflow Brief", + "title": "required", + "sections": [ + { + "heading": "Workflow", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Objective", + "min": 1, + "max": 1, + "content": true + }, + { + "heading": "Sources", + "max": 1, + "content": true + }, + { + "heading": "Scope", + "max": 1, + "content": true, + "sections": [ + { + "heading": "Included", + "max": 1, + "content": true + }, + { + "heading": "Excluded", + "max": 1, + "content": true + } + ] + }, + { + "heading": "Inputs and outputs", + "max": 1, + "content": true + }, + { + "heading": "Requirements and preferences", + "max": 1, + "content": true + }, + { + "heading": "Acceptance criteria", + "max": 1, + "content": true + }, + { + "heading": "Open questions", + "max": 1, + "content": true + }, + { + "heading": "Decisions", + "max": 1, + "content": true + }, + { + "heading": "Related artifacts", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + }, + { + "heading": "Agent Environment", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Tooling", + "max": 1, + "content": true + }, + { + "heading": "Constraints", + "max": 1, + "content": true + }, + { + "heading": "Containerization", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/SKILL.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/SKILL.md new file mode 100644 index 0000000..1cc9121 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/SKILL.md @@ -0,0 +1,72 @@ +--- +name: workflow-brief-to-galaxy-interface +description: "Map a reviewed Workflow Brief into a Galaxy workflow interface design brief, consulting retained source evidence when available." +--- + +# workflow-brief-to-galaxy-interface + +Follow the procedure below and use the artifact/reference sections as the runtime contract. + +## When To Use + +- Map a reviewed Workflow Brief into a Galaxy workflow interface design brief, consulting retained source evidence when available. + +## Inputs + +- Read artifact `workflow-brief`. Schema: workflow-brief-schema. Produced by `freeform-summary-to-workflow-brief`, `nextflow-summary-to-workflow-brief`. Expert-reviewed implementation contract. Its objective, scope, requirements, acceptance criteria, and exclusions govern every interface decision. +- Read artifact `freeform-summary`. Optional; absence is allowed and must be reported honestly. Produced by `interview-to-freeform-summary`, `summarize-paper`. Optional narrative source evidence; it may clarify the selected scope but cannot broaden or override the brief. +- Read artifact `summary-nextflow`. Optional; absence is allowed and must be reported honestly. Schema: summary-nextflow. Produced by `summarize-nextflow`. Optional structured Nextflow source evidence; process, channel, parameter, and test facts are evidence rather than assumed Galaxy design. +- Read artifact `open-requirements-ledger`. Optional; absence is allowed and must be reported honestly. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried workflow-run knowledge open-requirements-ledger. Absent on the first design Mold of a run; initialize an empty ledger. + +## Outputs + +- Write artifact `workflow-brief-galaxy-interface` as `workflow-brief-galaxy-interface.md`. Format: `markdown`. Reviewable Markdown brief: Galaxy workflow inputs, outputs, labels, collection shapes, checkpoint outputs, Workflow Brief provenance, supporting-evidence citations, confidence, and open questions. +- Write artifact `open-requirements-ledger` as `open-requirements.ledger.yml`. Format: `yaml`. Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact. + +## Required Tools + +- None declared. Procedure should not assume external CLIs are present. + +## Load Upfront + +- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Inherit open entries rather than re-deriving them, close the ones this brief's input, output, and label decisions settle, and append interface obligations the narrative never settles — an output the source names but never specifies, a parameter stated only qualitatively, a collection shape the prose doesn't determine. +- `references/notes/workflow-brief-design.md`: Research note copied verbatim into the bundle. Preserve the authority boundary between reviewed intent, optional source evidence, and downstream Galaxy design decisions. +- `references/schemas/workflow-brief-schema.schema.json`: Schema file copied verbatim into the bundle. Validate the required Workflow Brief before treating it as the implementation contract. + +## Load On Demand + +- `references/notes/galaxy-collection-semantics.md`: Research note copied verbatim into the bundle. Choose Galaxy collection input shapes (File / list / paired / list:paired / record) from the source's per-sample, paired, grouped, or nested data descriptions. Use when: the brief or supporting evidence describes paired reads, per-sample groups, nested or grouped inputs, or any input that should become a Galaxy dataset collection. +- `references/notes/galaxy-collection-semantics.yml`: Companion file copied verbatim into the bundle. Sibling of `references/notes/galaxy-collection-semantics.md`; read it where that note directs. +- `references/notes/galaxy-sample-sheet-collections.md`: Research note copied verbatim into the bundle. Pick the right sample_sheet variant and translate described per-sample column metadata into Galaxy column_definitions when the source describes sample-sheet-shaped inputs. Use when: the brief or supporting evidence describes a sample sheet, a per-sample/per-record table, or any table mapping samples to files that should become a Galaxy collection or sample-sheet input. +- `references/notes/galaxy-workflow-testability-design.md`: Research note copied verbatim into the bundle. Choose stable workflow input/output labels and promoted checkpoint outputs that future tests can address. Use when: deciding labels, public outputs, checkpoint outputs, or fixture-compatible collection inputs. + +## Validation + +- None declared. + +## Procedure + +Read the reviewed, unchanged Workflow Brief and emit a reviewable Markdown interface brief for a Galaxy workflow. Capture workflow inputs, workflow outputs, labels, Galaxy collection shapes, checkpoint outputs worth exposing for tests, provenance, confidence, and open questions. + +The Workflow Brief is authoritative for objective, included and excluded scope, expert requirements, and acceptance criteria. Consult a retained `freeform-summary` or `summary-nextflow` only as supporting evidence for the selected work. Evidence may supply methods, formats, parameters, datasets, or uncertainty that the concise brief omits, but it cannot silently add analyses or override an expert decision. A hand-authored brief with no source summary is valid. + +Translate what the reviewed contract and available evidence support into Galaxy interface decisions. Record evidence as discoveries, authorized mappings as decisions, missing actionable detail as obligations, and any needed change to expert intent as a proposed brief-change recommendation in the ledger. Never edit the brief or apply a proposed change here. + +Before the interface decisions, record each of the brief's open questions as a ledger decision, following the Workflow Brief open questions protocol in open-requirements-ledger. Record every question, including ones about data flow or tests rather than the interface, so none is lost before a later skill reads the ledger. A brief with no open questions adds nothing. + +The output is not a gxformat2 skeleton and not a workflow schema. It is a design handoff consumed by workflow-brief-to-galaxy-data-flow, workflow-brief-to-galaxy-template, and later test-plan work. + +## Feedback Mode + +- Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. +- When enabled, read `_feedback.md` before doing the work and use its registered `foundry-feedback.ledger.yml` protocol. +- Preserve harness-owned run and phase state. Append only concrete observations about a canonical Foundry source asset or a related project that this run showed to be at fault; do not put ordinary workflow requirements in this ledger. +- Before reporting completion, make one explicit pass over the work you just did. Do not ask yourself whether anything was unclear — recall what happened: where you guessed at something the instructions should have settled, needed information this bundle does not carry, hit an instruction that contradicted another or contradicted the artifacts in front of you, used a packaged reference that did not cover your case, or did something the procedure never describes. +- Append an entry for each such event that clears the protocol's bar. If none do, append nothing and report `no feedback` explicitly. Silence and a clean pass are not the same thing, and nothing downstream can tell them apart unless you say which one it was. +- Pass the same ledger path to any subagent used for this work, and merge updates serially so one writer cannot overwrite another. + +## Runtime Notes + +- Do not read Foundry source files at runtime; use only files packaged in this skill bundle and user-supplied artifacts. +- Preserve declared artifact filenames unless the user or harness supplies explicit paths. +- Carry unresolved assumptions into the output artifact instead of silently inventing missing source evidence. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_feedback.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_feedback.md new file mode 100644 index 0000000..61a4bd4 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_feedback.md @@ -0,0 +1,143 @@ +# Foundry feedback ledger + +The `foundry-feedback-ledger` is an opt-in runtime artifact for observations about the Foundry +assets used during a real cast-skill or pipeline run. It records actionable gaps, defects, +friction, and wishes about Molds and their authored references. It does not record unmet +requirements in the workflow being built; those belong in the open-requirements ledger. + +Every entry is destined for `galaxyproject/foundry`. When a run shows that a related project is +at fault — Galaxy, a tool wrapper, the IWC corpus, a harness CLI — that observation still belongs +here, recorded with the analysis that points at the upstream. The Foundry is where the run +context lives, so it is where the issue is tracked; a maintainer forwards from there. Never tell +a user to take an observation to another tracker instead. + +The registered filename is `foundry-feedback.ledger.yml`. Feedback-aware callers pass the same +file through every skill invocation in a run. + +## Ledger shape + +```yaml +run: + pipeline: nextflow-to-galaxy + run_slug: sarek-galaxy + status: running + phases: + - { n: 1, kind: mold, skill: summarize-nextflow, status: done, feedback_checked: true } + - { n: 2, kind: mold, skill: nextflow-summary-to-galaxy-interface, status: running } +entries: + - id: channel-shape-note-silent-on-optional-outputs + raised_by: nextflow-summary-to-galaxy-interface + observed_in: + mold: nextflow-summary-to-galaxy-interface + path: content/molds/nextflow-summary-to-galaxy-interface/index.md + revision: 6 + content_hash: 82d8... + foundry_head: d9e0b09 + subject: + kind: research + label: nextflow-to-galaxy-channel-shape-mapping + locator: content/research/nextflow-to-galaxy-channel-shape-mapping/index.md + content_hash: a18c... + kind: gap + severity: major + what: "The note does not cover optional outputs." + expected: "State how an absent optional channel maps to the Galaxy interface." + evidence: "Interface design stopped at the optional output decision." + status: open + issue: null +``` + +Entry `kind` is `gap`, `defect`, `friction`, or `wish`. Severity is `blocker`, `major`, or +`minor`. Status is `open`, `filed`, `duplicate`, or `wontfix`. + +## Run lifecycle + +For a pipeline run, the harness creates the file before phase 1 with `run.status: running`, an +empty `entries` list, and the complete top-level phase roster from its assembly record. All +phases begin `pending`. The harness changes a phase to `running` immediately before invocation +and to `done` on success. A phase that terminates the run and the run itself become `failed`; a +graceful user stop makes the run `cancelled`. A hard interruption naturally leaves the run +`running`, which is distinguishable from success without a recovery write. + +A phase also carries `feedback_checked`. The harness sets it true only when that phase reported a +feedback outcome — entry ids it appended, or an explicit `no feedback`. It is absent or false +otherwise, including for a phase that simply never said. This is what separates a run that looked +for friction and found none from a run that never looked. + +Loop phases remain one row and increment `iterations` after each successful iteration. Branch +phases remain one row and record the chosen path in `selected`. After every intended phase is +done, the harness sets `run.status: complete`. + +For a standalone skill invocation, the first feedback-aware skill creates a ledger with +`pipeline: null`, the caller's run slug when one exists, and one phase row for itself. It owns +that row's lifecycle as well as any entry it appends. + +No ledger means feedback mode was off. An empty `entries` list is evidence of a clean run only +when `run.status` is `complete` **and** every phase is `feedback_checked`. A complete run with +unchecked phases is complete but unreviewed; an empty incomplete ledger is neither. + +## When to append + +Appending is a deliberate pass, not a background reflex. Before a skill reports completion it +recalls what actually happened during its work — where it guessed at something the instructions +should have settled, needed information its bundle does not carry, hit an instruction that +contradicted another or contradicted the artifacts in front of it, used a reference that did not +cover the case, or did something its procedure never describes. Those are recallable events. "Was +anything unclear?" is not a useful question to ask at that moment and reliably answers itself no. + +The pass ends in one of two outcomes, and the outcome is always stated: entries appended, or +`no feedback`. A skill that says nothing has not reported a clean pass. + +Two entry shapes are admissible. Both require that `what` names a concrete problem, `expected` +states a proposed correction, and the entry is not already present in this ledger for the same +subject and correction. + +**A Foundry source asset.** The observation concerns Foundry-authored content, implementation, or +a registry that could be changed directly, and `subject.locator` canonically identifies it. This +is the ordinary case and the one the clustering and hash checks are built for. + +**A related project.** The run showed that a project the Foundry drives or cites is at fault — +Galaxy, a Tool Shed wrapper, the IWC corpus, `gxwf`, planemo. Use `subject.kind: related-project`, +name the project and the specific component in `label`, and put its public identity in +`locator` (a repository URL, a tool id, an IWC workflow id). There is no `content_hash`. State +in `what` and `evidence` what the run actually observed, and in `expected` what the upstream +would have to do. The entry is still filed against the Foundry. + +Do not use feedback entries for ordinary workflow requirements, tool-selection uncertainty, or +an unsupported preference with nothing concrete to change. + +## Canonical identity and provenance + +For a Foundry subject, use a repository-relative authored path as `subject.locator`; for a +package-export schema, use `package://#`. Generated skills and harnesses are +never subjects; identify the Mold, Pipeline, caster, assembler, or registry that must change. + +Initial subject kinds are `mold`, `pipeline`, `pattern`, `source-pattern`, `research`, `schema`, +`prompt`, `cli-tool`, `cli-command`, `meta`, `implementation`, and `registry`, plus +`related-project` for the upstream case above. `label` is for display only; the locator is the +identity and clustering key. + +A `related-project` subject is the one kind whose locator is not a Foundry path: it is public +upstream identity, carries no content hash, and is never checked against Foundry main. + +For the running Mold, copy `mold.name`, `mold.path`, `mold.revision`, and `mold.content_hash` +from `_provenance.json` into `observed_in`. Record its `mold.commit` value as `foundry_head`, but +never use that commit as exact content identity: it can be null or predate the bytes in the cast. +For a packaged reference, copy its `refs[].src` and `refs[].src_hash` into the subject locator and +content hash. When the subject is the Mold itself, use its authored path and +`observed_in.content_hash`. + +## Safe updates + +Read the complete ledger immediately before changing it. Preserve run state and all entries you +do not own. Append a complete entry in one rewrite; do not leave a partial YAML document. The +harness is the sole writer of pipeline lifecycle fields, while each skill writes only entries. + +Keep evidence concise and safe to upstream. Never copy credentials, tokens, private repository +URLs, proprietary source text, patient or participant data, or user-identifying filesystem +paths into the ledger. Replace a run-directory prefix with `/` and summarize sensitive +evidence instead of quoting it. + +The reporting skill may set an open entry to `filed`, `duplicate`, or `wontfix` and record the +resolved issue URL after the corresponding disposition is confirmed. It never deletes an entry; +the ledger remains the run audit trail. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_provenance.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_provenance.json new file mode 100644 index 0000000..27c749b --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_provenance.json @@ -0,0 +1,197 @@ +{ + "provenance_schema_version": 4, + "cast_target": "claude", + "mold": { + "name": "workflow-brief-to-galaxy-interface", + "path": "content/molds/workflow-brief-to-galaxy-interface/index.md", + "revision": 5, + "content_hash": "c84d428e03ab42e7b33335c72de6728d5bf26726e4a9986bf49343e6461a2fcd", + "commit": "94ce4ea20107680b467d03e8889b63689774e325" + }, + "cast_at": "2026-11-10T10:05:13.165Z", + "refs": [ + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-collection-semantics]]", + "src": "content/research/galaxy-collection-semantics/index.md", + "dst": "references/notes/galaxy-collection-semantics.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Choose Galaxy collection input shapes (File / list / paired / list:paired / record) from the source's per-sample, paired, grouped, or nested data descriptions.", + "trigger": "When the brief or supporting evidence describes paired reads, per-sample groups, nested or grouped inputs, or any input that should become a Galaxy dataset collection.", + "license": "MIT", + "license_file": "LICENSES/galaxy.LICENSE", + "src_hash": "4ddc7587e187cf38a3dca02d6966e2f6ea58f9faf9bd97034ad513eb4c4c638b", + "dst_hash": "4ddc7587e187cf38a3dca02d6966e2f6ea58f9faf9bd97034ad513eb4c4c638b", + "source": "deterministic", + "license_file_hash": "3fdd32b75cfad3266871456423f4d91529953629fe4bc803c3df2f445fd8a858" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-collection-semantics]]", + "src": "content/research/galaxy-collection-semantics/galaxy-collection-semantics.yml", + "dst": "references/notes/galaxy-collection-semantics.yml", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Choose Galaxy collection input shapes (File / list / paired / list:paired / record) from the source's per-sample, paired, grouped, or nested data descriptions.", + "trigger": "When the brief or supporting evidence describes paired reads, per-sample groups, nested or grouped inputs, or any input that should become a Galaxy dataset collection.", + "companion_of": "references/notes/galaxy-collection-semantics.md", + "license": "MIT", + "license_file": "LICENSES/galaxy.LICENSE", + "src_hash": "f4e9d0a55459b6080b826e2ae41a920c9404f95fda0e518be8f86bc3d2e73e6e", + "dst_hash": "f4e9d0a55459b6080b826e2ae41a920c9404f95fda0e518be8f86bc3d2e73e6e", + "source": "deterministic", + "license_file_hash": "3fdd32b75cfad3266871456423f4d91529953629fe4bc803c3df2f445fd8a858" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-sample-sheet-collections]]", + "src": "content/research/galaxy-sample-sheet-collections/index.md", + "dst": "references/notes/galaxy-sample-sheet-collections.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Pick the right sample_sheet variant and translate described per-sample column metadata into Galaxy column_definitions when the source describes sample-sheet-shaped inputs.", + "trigger": "When the brief or supporting evidence describes a sample sheet, a per-sample/per-record table, or any table mapping samples to files that should become a Galaxy collection or sample-sheet input.", + "src_hash": "98736fdcd0c4f4698af4e0cd4c539b33e3d0db872de1e10487556b21033030ec", + "dst_hash": "98736fdcd0c4f4698af4e0cd4c539b33e3d0db872de1e10487556b21033030ec", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-workflow-testability-design]]", + "src": "content/research/galaxy-workflow-testability-design/index.md", + "dst": "references/notes/galaxy-workflow-testability-design.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Choose stable workflow input/output labels and promoted checkpoint outputs that future tests can address.", + "trigger": "When deciding labels, public outputs, checkpoint outputs, or fixture-compatible collection inputs.", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[open-requirements-ledger]]", + "src": "content/research/open-requirements-ledger/index.md", + "dst": "references/notes/open-requirements-ledger.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Inherit open entries rather than re-deriving them, close the ones this brief's input, output, and label decisions settle, and append interface obligations the narrative never settles — an output the source names but never specifies, a parameter stated only qualitatively, a collection shape the prose doesn't determine.", + "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[workflow-brief-design]]", + "src": "content/research/workflow-brief-design/index.md", + "dst": "references/notes/workflow-brief-design.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Preserve the authority boundary between reviewed intent, optional source evidence, and downstream Galaxy design decisions.", + "verification": "Confirm a worked run does not broaden brief scope from richer source evidence and leaves the brief unchanged.", + "src_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "dst_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[workflow-brief-schema]]", + "src": "package://@galaxy-foundry/gxwf-foundry#workflowBriefSchema", + "dst": "references/schemas/workflow-brief-schema.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Validate the required Workflow Brief before treating it as the implementation contract.", + "src_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "dst_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "source": "deterministic" + } + ], + "artifacts": { + "produces": [ + { + "id": "workflow-brief-galaxy-interface", + "kind": "markdown", + "default_filename": "workflow-brief-galaxy-interface.md", + "description": "Reviewable Markdown brief: Galaxy workflow inputs, outputs, labels, collection shapes, checkpoint outputs, Workflow Brief provenance, supporting-evidence citations, confidence, and open questions." + }, + { + "id": "open-requirements-ledger", + "kind": "yaml", + "default_filename": "open-requirements.ledger.yml", + "description": "Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact." + } + ], + "consumes": [ + { + "id": "workflow-brief", + "description": "Expert-reviewed implementation contract. Its objective, scope, requirements, acceptance criteria, and exclusions govern every interface decision.", + "inherited_schema": "[[workflow-brief-schema]]", + "producers": [ + "freeform-summary-to-workflow-brief", + "nextflow-summary-to-workflow-brief" + ] + }, + { + "id": "freeform-summary", + "description": "Optional narrative source evidence; it may clarify the selected scope but cannot broaden or override the brief.", + "optional": true, + "producers": [ + "interview-to-freeform-summary", + "summarize-paper" + ] + }, + { + "id": "summary-nextflow", + "description": "Optional structured Nextflow source evidence; process, channel, parameter, and test facts are evidence rather than assumed Galaxy design.", + "optional": true, + "inherited_schema": "[[summary-nextflow]]", + "producers": [ + "summarize-nextflow" + ] + }, + { + "id": "open-requirements-ledger", + "description": "Carried workflow-run knowledge [[open-requirements-ledger]]. Absent on the first design Mold of a run; initialize an empty ledger.", + "optional": true, + "producers": [ + "advance-galaxy-draft-step", + "apply-galaxy-workflow-changeset", + "compare-against-iwc-exemplar", + "cwl-summary-to-galaxy-data-flow", + "cwl-summary-to-galaxy-interface", + "cwl-summary-to-galaxy-template", + "freeform-summary-to-galaxy-data-flow", + "freeform-summary-to-galaxy-interface", + "freeform-summary-to-galaxy-template", + "implement-galaxy-tool-step", + "interview-to-galaxy-workflow-changeset", + "mature-galaxy-workflow-for-iwc", + "nextflow-summary-to-galaxy-data-flow", + "nextflow-summary-to-galaxy-interface", + "nextflow-summary-to-galaxy-reference-data", + "nextflow-summary-to-galaxy-template", + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" + ] + } + ] + } +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_verify.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_verify.json new file mode 100644 index 0000000..f40b330 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/_verify.json @@ -0,0 +1,31 @@ +{ + "verify_schema_version": 1, + "entries": [ + { + "artifact_id": "summary-nextflow", + "direction": "input", + "kind": "json", + "default_filename": "summary-nextflow.json", + "schema": "[[summary-nextflow]]", + "validator_bin": "foundry", + "args": [ + "validate-summary-nextflow", + "{artifact_path}" + ] + }, + { + "artifact_id": "workflow-brief", + "direction": "input", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "validator_bin": "foundry", + "args": [ + "validate-workflow-brief", + "--schema-sha256", + "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "{artifact_path}" + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-collection-semantics.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-collection-semantics.md new file mode 100644 index 0000000..0d92bcf --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-collection-semantics.md @@ -0,0 +1,43 @@ +--- +type: research +title: "Galaxy collection semantics" +tags: + - target/galaxy +status: draft +created: 2026-04-30 +revised: 2026-05-05 +revision: 3 +license: MIT +license_file: LICENSES/galaxy.LICENSE +related_notes: + - "[[galaxy-xsd]]" + - "[[galaxy-collection-tools]]" + - "[[galaxy-apply-rules-dsl]]" + - "[[nextflow-to-galaxy-channel-shape-mapping]]" + - "[[nextflow-operators-to-galaxy-collection-recipes]]" + - "[[galaxy-tool-job-failure-reference]]" + - "[[galaxy-workflow-invocation-failure-reference]]" + - "[[iwc-transformations-survey]]" + - "[[galaxy-discover-datasets]]" +sources: + - "https://github.com/galaxyproject/galaxy/blob/7765fae934fbfdee77e3be5f5b235e43735273ae/lib/galaxy/model/dataset_collections/types/collection_semantics.yml" +companions: + - "galaxy-collection-semantics.yml" +summary: "Vendored formal spec of Galaxy dataset-collection mapping/reduction semantics, with labeled examples and pinned test references." +--- + +> **Vendored from upstream**, pinned at SHA `7765fae`. One file travels with this note: +> +> - `galaxy-collection-semantics.yml` — the structured source. **Agents and casting should consume this.** It carries the `tests:` blocks that pin concrete Galaxy test names; the rendered upstream view drops them. +> +> The Foundry also vendors Galaxy's auto-generated MyST/LaTeX rendering of the same YAML, which +> is what the human view below renders from. It is a site asset: casting does not carry it, so +> do not look for it beside this note in a skill bundle. Sync is manual. +> +> **When to consult:** authoring or reasoning about Molds and patterns that touch `data_collection` inputs, map-over / reduction shape changes, sub-collection mapping, `paired_or_unpaired`, or `sample_sheet`. + +```vendored-myst +file: galaxy-collection-semantics.upstream.myst +source: https://github.com/galaxyproject/galaxy/blob/7765fae934fbfdee77e3be5f5b235e43735273ae/doc/source/dev/collection_semantics.md +sha: 7765fae +``` diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-collection-semantics.yml b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-collection-semantics.yml new file mode 100644 index 0000000..650e39e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-collection-semantics.yml @@ -0,0 +1,1250 @@ +- doc: | + # Collection Semantics + + This document describes the semantics around working with Galaxy dataset collections. + In particular it describes how they operate within Galaxy tools and workflows. + + :::{admonition} You Probably Don't Need to Read This + :class: caution + + Any significantly sophisticated workflow language will have ways to collect data + into arrays or vectors or dictionaries and apply operations across this data (mapping) + or reduce the dimensionality of this data (reductions). Typically, this is explicitly + annotated with map functions or for loops. Galaxy however is designed to be a point + and click interface for connecting steps and running tools. It is important that steps + just connect and just do the most natural thing - and this is what Galaxy does. + This document just provides a mathematical formalism to that "what should just + intuitively work" that can be used to document test cases and help with implementation. + This is reference documentation not user documentation, Galaxy should just work. + ::: + + ## Mapping + + If a tool consumes a simple dataset parameter and produces a simple dataset parameter, + then any collection type may be "mapped over" the data input to that tool. The result of + that is the tool being applied to each element of the collection and "implicit collections" + being created from the outputs that are produced from those operations. Those implicit + collections have the same element identifiers in the same order as the input collection that is + mapped over. Each element of the implicit collections correspond to their own job and + Galaxy very naturally and intuitively parallelizes jobs without extra work from the user + and without any knowledge of the tool. + +- example: + label: BASIC_MAPPING_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_collection" + workflow_runtime: + framework_test: "collection_semantics_cat_0" + workflow_editor: "accepts paired data -> data connection" + +- example: + label: BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" + workflow_runtime: + framework_test: "collection_semantics_cat_1" + workflow_editor: "accepts paired_or_unpaired data -> data connection" + +- example: + label: BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED + assumptions: + - datasets: [d_u] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {unpaired: d_u}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + unpaired: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_u}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_2" + workflow_editor: "accepts paired_or_unpaired data -> data connection" + +- example: + label: BASIC_MAPPING_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [list, {i1: "d_1", ..., in: "d_n"}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + workflow_editor: "accepts collection data -> data connection" + +- doc: | + The above description of mapping over inputs works naturally and as expected for + nested collections. + +- example: + label: NESTED_LIST_MAPPING + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tools.py::TestToolsApi::test_map_over_nested_collections" + workflow_editor: "accepts list:list data -> data connection" + +- example: + label: BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:paired_or_unpaired" + elements: + el1: + type: nested_elements + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_data_with_list_paired_or_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_3" + workflow_editor: "accepts list:paired_or_unpaired data -> data connection" + +- doc: | + + For tools with multiple data inputs, the tool can be executed with individual + datasets for the non-mapped over input and each tool execution will just be executed + with that dataset. The dataset not mapped over serves as the input for each execution. + +- example: + label: BASIC_MAPPING_INCLUDING_SINGLE_DATASET + assumptions: + - datasets: ["d_1,...,d_n", "d_o"] + - tool: + in: {i: dataset, i2: dataset} + out: {o: dataset} + - collections: + C: ["list", {i1: d_1, ..., in: d_n}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + i2: {type: dataset, ref: d_o} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}, i2: {type: dataset, ref: d_o}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}, i2: {type: dataset, ref: d_o}}} + output: o + tests: + tool_runtime: + api_test: "test_tools.py::TestToolsApi::test_map_over_collected_and_individual_datasets" + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_0" + +- doc: | + If a tool consumes two input datasets and produces one output dataset, you can map two + collections with identical structure (same element identifiers in the same order) over + the respective inputs and the result is an implicit collection with the same structure + as the inputs and where each output in the implicit collection corresponds to the tool + being executed with the two inputs corresponding to that position in the input + collections. + + The default behavior here is the collections are linked and the act of mapping over + inputs to the tool are sort of a flat map or a dot product. No extra dimensionality + in the resulting collections. + + From a user perspective this means if you start with a collection and apply a bunch + of map over operations on tools - the results will all continue to match and work together + very naturally - again without extra work by the user and without extra knowledge + by the tool author. + +- example: + label: BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE + assumptions: + - datasets: ["d1_1,...,d1_n", "d2_1,...,d2_n"] + - tool: + in: {i: dataset, i2: dataset} + out: {o: dataset} + - collections: + C1: ["list", {i1: d1_1, ..., in: d1_n}] + C2: ["list", {i1: d2_1, ..., in: d2_n}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C1} + i2: {type: map_over, collection: C2} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_1}, i2: {type: dataset, ref: d2_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_n}, i2: {type: dataset, ref: d2_n}}} + output: o + tests: + tool_runtime: + api_test: test_tools.py::TestToolsApi::test_map_over_two_collections + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_1" + +- doc: "## Reduction" +- doc: | + Not all tool executions result in implicit collections and mapping + over inputs. Tool inputs of ``type`` ``data_collection`` can consume + collections directly and do not necessarily result in mapping over. + + Tools that consume collections and output datasets effectively + reduce the dimension of the Galaxy data structure. When used at runtime + this is often referred to as a "reduction" in the code. + +- example: + label: COLLECTION_INPUT_PAIRED + assumptions: + - datasets: ["d_f", "d_r"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + tool_runtime: + tool: collection_paired_test + workflow_editor: "accepts paired -> paired connection" + +- example: + label: COLLECTION_INPUT_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {el1: d_1, ..., eln: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_0" + workflow_editor: "accepts list -> list connection" + +- example: + label: COLLECTION_INPUT_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + tool_runtime: + tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_0" + workflow_editor: "accepts paired_or_unpaired data -> paired_or_unpaired connection" + +- example: + label: COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + tool_runtime: + tool: collection_list_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_0" + workflow_editor: "accepts list:paired_or_unpaired data -> list:paired_or_unpaired connection" + +- doc: | + For nested collections where each rank is a ``list`` or a ``paired`` collection, + then collection inputs must match every part of the collection type input definition. + +- example: + label: COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects connecting paired -> list" + +- example: + label: COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects connecting list -> paired" + +- example: + label: COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired:paired -> list:paired connection" + +- example: + label: COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired:paired -> list:paired_or_unpaired connection" + +- doc: | + In addition to explicit collection inputs, tool inputs of ``type`` ``data`` + where ``multiple="true"`` can consume lists directly. This is likewise a + "reduction" and does not result in implicit collection creation. + +- example: + label: LIST_REDUCTION + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: dataset_list, refs: ["d_1", "...", "d_n"]} + tests: + tool_runtime: + api_test: "test_tools.py::TestToolsApi::test_reduce_collections" + workflow_runtime: + framework_test: "collection_semantics_multi_data_optional_0" + workflow_editor: "treats multi data input as list input" + +- doc: | + Paired collections cannot be reduced this way. ``paired`` is not meant + to represent a list/array/vector data structure - it is more like a tuple. + +- example: + label: PAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired input on multi-data input" + +- example: + label: PAIRED_OR_UNPAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired_or_unpaired input on multi-data input" + +- doc: "## Sub-collection Mapping" +- doc: | + ![](https://planemo.readthedocs.io/en/master/_images/subcollection_mapping_identifiers.svg) + +- example: + label: MAPPING_LIST_PAIRED_OVER_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + "C\\_PAIRED": [paired, {forward: d_f,reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: list + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_simple_subcollection_mapping" + workflow_editor: "accepts list:paired -> paired connection" + +- doc: | + The natural extension of multiple data input parameters consuming list collections as described + above when discussing reductions is that nested lists of lists (``list:list``) can be mapped + over a multiple data input parameter. Each nested list will be reduced by this operation but the + results will be mapped over. The result will be a list with the same structure as the outer list + of the input collection. + +- example: + label: NESTED_LIST_REDUCTION + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: list} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_1"]}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_n"]}}} + output: o + tests: + workflow_editor: "maps list:list over multi data input" + +- doc: | + Just as a paired collection won't be reduced by a multiple data input, any sort of nested + collection ending in a paired collection cannot be mapped over such an input. So a multiple + data input parameter cannot be mapped over by a list of pairs (``list:paired``) for instance. + +- example: + label: LIST_PAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + is_valid: false + tests: + workflow_editor: "rejects list:paired input on multi-data input" + +- example: + label: LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired_or_unpaired} + is_valid: false + tests: + workflow_editor: "rejects list:paired_or_unpaired input on multi-data input" + +- doc: "## paired_or_unpaired Collections" +- doc: | + The collection type ``paired_or_unpaired`` is meant to serve as a stand-in for + an entity that can be either a single dataset or what is effectively a ``paired`` + dataset collection. These collections either have one element with identifier + ``unpaired`` or two elements with identifiers ``forward`` and ``reverse``. + + Tools can declare a data_collection input with collection type ``paired_or_unpaired`` + and that input will consume either an explicit ``paired_or_unpaired`` collection + normally or can consume a ``paired`` input. + +- example: + label: PAIRED_OR_UNPAIRED_CONSUMES_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + - "C_AS_MIXED = CollectionInstance" + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: collection, ref: C_AS_MIXED} + tests: + tool_runtime: + tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_1" + workflow_editor: "accepts paired -> paired_or_unpaired connection" + +- doc: | + + The inverse of this doesn't work intentionally. In some ways a ``paired`` collection + acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired + collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` + dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist + in ``paired_or_unpaired`` collection. + +- example: + label: PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired_or_unpaired -> paired connection" + +- doc: | + + The same logic holds for mapping, lists of paired datasets (``list:paired``) can be mapped over these + ``paired_or_unpaired`` inputs and mixed lists of pairs (``list:paired_or_unpaired``) cannot + be mapped over a ``paired`` input. Following the same logic, ``list:paired_or_unpaired`` cannot + be mapped over a ``list`` input or multiple data input. + +- example: + label: MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el: {forward: d_f, reverse: d_r}}] + C_AS_MIXED: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_2" + workflow_editor: "accepts list:paired -> paired_or_unpaired connection" + +- example: + label: PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_paired_or_unpaired_not_consumed_by_paired_when_mapping" + workflow_editor: "rejects list:paired_or_unpaired -> paired connection" + +- example: + label: PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false + tests: + workflow_editor: "rejects list:paired_or_unpaired -> list connection" + +- doc: | + This logic extends naturally into higher dimensional collections. A ``list:list:paired`` + can be mapped over either a ``paired_or_unpaired`` input to produce a nested list (``list:list``) + or a ``list:paired_or_unpaired`` input to produce a flat list (``list``). + +- example: + label: MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list:paired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + C_AS_MIXED: ["list:list:paired_or_unpaired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_3" + workflow_editor: "accepts list:list:paired -> paired_or_unpaired connection" + +- doc: | + + In order for ``paired_or_unpaired`` collections to also act as a single dataset, + a flat list can be mapped over a such an input with a special sub collection mapping + type of 'single_datasets'. + +- example: + label: MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_list" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_7" + workflow_editor: "accepts list -> paired_or_unpaired connection" + +- doc: | + This treatment of lists without pairing extends to nested structures naturally. + For instance, a list of list of datasets (``list:list``) can be mapped over a + ``paired_or_unpaired`` input to produce a nested list of lists (``list:list``) + with a structure matching the input. Likewise, the nested list can be mapped over + a ``list:paired_or_unpaired`` input to produce a flat list with the same structure + as the outer list of the input. + +- example: + label: MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_UNPAIRED_j = CollectionInstance for each dataset dj in C" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + workflow_editor: "accepts list:list -> paired_or_unpaired connection" + +- example: + label: MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_LPU_i = inner list of C at position i, treated as list:paired_or_unpaired via single_datasets" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: "list:paired_or_unpaired"} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_1}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_list_paired_or_unpaired_with_list_of_lists" + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_2" + workflow_editor: "accepts list:list -> list:paired_or_unpaired connection" + +- doc: | + Due only to implementation time, the special casing of allowing paired_or_unpaired + act as both datasets and paired collections only works when it is the deepest + collection type. So while list:paired can be consumed by a list:paired_or_unpaired + input, a paired:list cannot be consumed by a paired_or_unpaired:list input though + it should be able to for consistency. We have focused our time on data structures + more likely to be used in actual Galaxy analyses given current and guessed future + usage. + +- doc: "## sample_sheet Collections" +- doc: | + The collection type ``sample_sheet`` attaches typed, columnar metadata to each + element of a dataset collection. For mapping and type matching purposes, + ``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool + inputs, matched against ``list`` collection inputs, and composed with inner types + (``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry + is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list`` + output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata + that plain lists do not. + +- example: + label: SAMPLE_SHEET_MAPPING + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: "d_1", ..., in: "d_n"}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_4" + workflow_editor: "accepts sample_sheet data -> data connection (maps like list)" + +- example: + label: SAMPLE_SHEET_MATCHES_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_1" + workflow_editor: "accepts sample_sheet -> list connection (canMatch)" + +- doc: | + Sub-collection mapping works the same as for ``list`` composites. A + ``sample_sheet:paired`` can be mapped over a ``paired`` collection input, + extracting each inner pair and producing a ``sample_sheet`` implicit output. + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + "C\\_PAIRED": [paired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_5" + workflow_editor: "accepts sample_sheet:paired -> paired connection (maps over like list:paired)" + +- example: + label: SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_0" + workflow_editor: "accepts sample_sheet:paired -> list:paired connection (canMatch)" + +- doc: | + The ``paired_or_unpaired`` integration rules carry over from ``list`` to + ``sample_sheet``. A flat ``sample_sheet`` can be mapped over a + ``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping, + and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as + ``list:paired`` can. + +- example: + label: SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_sample_sheet" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_8" + workflow_editor: "accepts sample_sheet -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + C_AS_MIXED: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_6" + workflow_editor: "accepts sample_sheet:paired -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_1" + workflow_editor: "accepts sample_sheet:paired_or_unpaired -> list:paired_or_unpaired connection (canMatch)" + +- doc: | + The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list`` + input because sample sheets carry all the structural information lists have (plus + metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because + lists lack the ``column_definitions`` and per-element ``columns`` metadata that + sample sheet consumers expect. + +- example: + label: LIST_NOT_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list -> sample_sheet connection (asymmetry)" + +- example: + label: LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list:paired -> sample_sheet:paired connection (asymmetry)" + +- example: + label: SAMPLE_SHEET_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_sample_sheet_0" + workflow_editor: "accepts sample_sheet -> sample_sheet connection" diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-sample-sheet-collections.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-sample-sheet-collections.md new file mode 100644 index 0000000..1dca0da --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-sample-sheet-collections.md @@ -0,0 +1,143 @@ +--- +type: research +title: "Galaxy sample_sheet collection types" +tags: + - target/galaxy +status: draft +created: 2026-05-05 +revised: 2026-05-06 +revision: 2 +related_notes: + - "[[galaxy-collection-semantics]]" + - "[[galaxy-collection-tools]]" + - "[[nextflow-workflow-io-semantics]]" + - "[[nextflow-params-to-galaxy-inputs]]" + - "[[nextflow-path-glob-to-galaxy-datatype]]" + - "[[nextflow-to-galaxy-channel-shape-mapping]]" + - "[[nextflow-to-galaxy-reference-data-mapping]]" +related_molds: + - "[[nextflow-summary-to-galaxy-interface]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" +sources: + - "Galaxy PR #19305 (Implement Sample Sheets), merged 2025-07-30" + - "lib/galaxy/model/dataset_collections/types/sample_sheet.py" + - "lib/galaxy/model/dataset_collections/types/sample_sheet_util.py" + - "lib/galaxy/model/dataset_collections/type_description.py" + - "lib/galaxy/schema/schema.py (SampleSheetColumnDefinition, SampleSheetRow)" + - "lib/galaxy/tools/wrappers.py (DatasetCollectionWrapper.sample_sheet_row)" + - "lib/galaxy/tools/sample_sheet_to_tabular.xml" + - "lib/galaxy/webapps/galaxy/api/dataset_collections.py (sample_sheet_workbook endpoints)" + - "lib/galaxy/model/migrations/alembic/versions_gxy/3af58c192752_implement_sample_sheets.py" +summary: "Galaxy's sample_sheet collection family: typed column metadata, four variants, mapping rules, validator allowlist." +--- + +# Galaxy sample_sheet collection types + +Reference for the Galaxy backend shape that targets structured per-row metadata — the natural landing zone for Nextflow `samplesheetToList` parameters and for any source-side idiom that pairs typed metadata columns with dataset references. + +## Shape + +A `sample_sheet` is a list-shaped collection where each element carries a typed `columns` row, and the parent collection carries a `column_definitions` schema. The model (`lib/galaxy/model/__init__.py`) adds two nullable JSON columns: `dataset_collection.column_definitions` and `dataset_collection_element.columns`. Existing collections are unaffected. + +``` +DatasetCollection + collection_type: "sample_sheet" + column_definitions: [{name, type, optional, ...}] + elements: + DatasetCollectionElement(element_identifier, columns: [v1, v2, ...], hda or child_collection) +``` + +## Column definition vocabulary + +`SampleSheetColumnDefinition` (lib/galaxy/schema/schema.py:384-405): + +``` +type: "string" | "int" | "float" | "boolean" | "element_identifier" +optional: bool +default_value, validators[], restrictions[], suggestions[] +``` + +- `element_identifier` cross-references another row's identifier in the same collection. Captures patterns like nf-core `control_sample`. Within-collection only — no cross-collection refs. +- `restrictions[]` enumerates allowed values (maps to nf-schema `enum`). +- `validators[]` is restricted to the **safe allowlist** (`AnySafeValidatorModel`, lib/galaxy/tool_util_models/parameter_validators.py): `regex`, `in_range`, `length`. `expression` and other arbitrary-Python validators are rejected. Translation from richer nf-schema validation must drop to prose or fall through. +- String values and column names are restricted to `[\w\-_ ?]*` — no tabs, newlines, quotes, or most specials. Necessary for safe CSV/TSV serialization. + +## Variants + +Enforced by `COLLECTION_TYPE_REGEX` in `type_description.py`: + +| Variant | Inner element shape | Typical source | +|---|---|---| +| `sample_sheet` | one dataset per row | one path column in nf-schema | +| `sample_sheet:paired` | `paired` (forward/reverse) | two path columns, paired-end fastq | +| `sample_sheet:paired_or_unpaired` | mixed single or pair | mixed-library sample sheets | +| `sample_sheet:record` | `record` (named typed slots) | heterogeneous per-sample artifacts | + +Constraints to plan around: + +- `sample_sheet` must be **outermost**. `list:sample_sheet` and `sample_sheet:list` are invalid. +- `prototype_elements()` is not implemented — sample-sheet-shaped pre-creation isn't possible because element count is unknown until data exists. + +## Mapping rules + +`allow_implicit_mapping` is `True` for everything except `record`, so sample sheets behave like lists for mapping: + +- `sample_sheet` over a `(dataset → dataset)` tool produces an implicit `sample_sheet`-shaped output. **`column_definitions` and `columns` are not propagated** — metadata lives only on the input collection. +- `sample_sheet:paired` over a `paired` input produces a `sample_sheet`-shaped output (one job per row). +- A `sample_sheet` can be reduced by a `multiple=true` data input. +- Two sample sheets with identical structure can be linked for dot-product execution. + +## Tool-side access + +Tools read columns at runtime through `DatasetCollectionWrapper.sample_sheet_row(element_identifier)` (lib/galaxy/tools/wrappers.py:643-707), which builds a `{element_identifier: row}` dict from element `columns`. The built-in `__SAMPLE_SHEET_TO_TABULAR__` tool (lib/galaxy/tools/sample_sheet_to_tabular.xml) iterates and tab-joins for downstream tabular consumers. Tool inputs declare acceptance via `data_collection collection_type="sample_sheet,sample_sheet:paired,..."`. + +`DataCollectionToolParameter` carries `column_definitions` through `to_dict()`, which lets the workflow editor render column-definition forms for sample-sheet inputs. + +## API surfaces + +- `POST /api/dataset_collections` accepts `column_definitions` and `rows` (a `{element_identifier: [values]}` dict). +- `POST /api/tools/fetch` carries `column_definitions` at target level and per-element `row` for remote-URI sample-sheet creation. +- Workbook endpoints (`/api/sample_sheet_workbook[/parse]`, `/api/dataset_collections/{id}/sample_sheet_workbook[/parse]`) generate and parse XLSX/CSV/TSV using openpyxl, with header rows, dropdowns for `restrictions`, type validation, and an instructions sheet. + +## Workflow integration + +`InputCollectionModule` (lib/galaxy/workflow/modules.py) validates `column_definitions` at workflow save and propagates them to the runtime form. Galaxy workflow YAML accepts: + +```yaml +inputs: + chipseq_data: + type: collection + collection_type: sample_sheet:paired + column_definitions: + - {type: string, name: condition, optional: false} + - {type: int, name: replicate, optional: false} + - {type: element_identifier, name: control_sample, optional: true} +``` + +This is the structured-input lever a Nextflow → Galaxy translation reaches for when source pipelines use `samplesheetToList(params.input, "assets/schema_input.json")`. + +## Mapping from Nextflow sample sheets + +| Nextflow / nf-schema | Galaxy `sample_sheet*` | +|---|---| +| `samplesheetToList(params.x, schema)` materialization | `sample_sheet` collection input bound to `params.x` | +| nf-schema column `type: string\|integer\|number\|boolean` | column `type: string\|int\|float\|boolean` | +| Column annotated `meta:` (carried in `meta` map) | non-path column in `columns[]` | +| Path-typed column (one per row) | element dataset (variant `sample_sheet`) | +| Two path columns (fastq pair) | inner `paired` (variant `sample_sheet:paired`) | +| Mixed single/paired rows | inner `paired_or_unpaired` | +| Heterogeneous typed per-row artifacts | inner `record` (variant `sample_sheet:record`) | +| nf-schema `enum` | `restrictions[]` | +| nf-schema `pattern` | `regex` validator | +| nf-core `control_sample` style cross-row reference | `element_identifier` column type | + +Edges to flag, not silently squash, when translating: + +- nf-schema validation richer than regex/in_range/length cannot be expressed; downgrade to prose with confidence note. +- Cross-sample-sheet references have no Galaxy expression — keep as plain `string` and warn. +- Ad-hoc `splitCsv(header: true)` parsing without an `assets/schema_*.json` produces no column types — translation needs ad-hoc inference; this note's mapping does not apply. +- Output side has no automatic propagation: a tool mapped over a `sample_sheet` produces a `sample_sheet`-shaped collection without `column_definitions`. Carry-forward must be explicit (re-attaching metadata via `__SAMPLE_SHEET_TO_TABULAR__` or rules DSL `add_column_from_sample_sheet_index`). + +## Why this matters for the Foundry + +Without sample_sheet shape awareness, `nextflow-summary-to-galaxy-interface` defaults nf-core sample-sheet inputs to `list:paired` or a flat file input, dropping per-row metadata. Surfacing the column schema lets the interface Mold pick the right variant and lets the data-flow Mold preserve `meta` fields through identifier-keyed wiring instead of inventing parallel parameter inputs. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md new file mode 100644 index 0000000..3ba5b73 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/galaxy-workflow-testability-design.md @@ -0,0 +1,62 @@ +--- +type: research +tags: + - target/galaxy +status: draft +created: 2026-05-03 +revised: 2026-09-23 +revision: 3 +related_notes: + - "[[iwc-workflow-testability-survey]]" + - "[[iwc-test-data-conventions]]" + - "[[planemo-asserts-idioms]]" + - "[[iwc-shortcuts-anti-patterns]]" + - "[[planemo-workflow-test-architecture]]" + - "[[implement-galaxy-workflow-test]]" + - "[[gxformat2-schema]]" + - "[[gxformat2-workflow-inputs]]" + - "[[galaxy-datatypes-conf]]" +summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." +--- + +# Design a Galaxy workflow that can be tested + +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. + +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. + +## Name the public interface before writing tests + +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. + +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. + +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. + +## Expose a checkpoint that can catch the wrong result + +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. + +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. + +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. + +## Make collection members addressable + +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. + +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. + +## Choose inputs that real fixtures can supply + +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. + +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. + +## Express the choice in gxformat2 + +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. + +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. + +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/open-requirements-ledger.md new file mode 100644 index 0000000..761183e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/open-requirements-ledger.md @@ -0,0 +1,231 @@ +--- +type: research +title: "Open-requirements ledger" +tags: + - target/galaxy +status: draft +created: 2026-06-16 +revised: 2026-11-10 +revision: 7 +related_notes: + - "[[galaxy-workflow-draft-format]]" +related_molds: + - "[[advance-galaxy-draft-step]]" + - "[[repair-galaxy-draft-topology]]" + - "[[implement-galaxy-tool-step]]" +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." +--- + +# Open-requirements ledger + +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. + +## Framing: obligations the pipeline discharges, not questions a human answers + +This is deliberately *not* an "open questions for the user" list. The pipeline is autonomous — no human-in-the-loop gate is assumed. The ledger's consumers are **Molds and the loop's convergence gate**, with human readout a secondary affordance. An entry is closed by a downstream Mold doing work (wiring a producer, picking a wrapper, settling a value), or — when nothing can close it — surrendered: written into the final draft as a known, labelled gap rather than silently dropped or fabricated around. + +The distinction matters because a "questions for a human" framing leaks an operator's personal interaction style into a tool meant to run inside anyone's harness. The ledger must behave identically cast into a fresh harness with no ambient configuration; that holds only because every obligation is **materialized in the artifact**, never carried as operator habit. + +## Entry shape (v1, loose) + +Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries are intentionally low-ceremony for v1 — enough structure to count and close, no contract pretending to be machine-parameterizable yet. A reasonable per-entry shape: + +```yaml +- id: sccmec-evidence-missing + status: open # open | resolved | surrendered + kind: gap # gap | dropped — see below; omit for gap + blocking: true # true only when a structural edit closes it; the convergence gate counts these + raised_by: implement-galaxy-tool-step # Mold that appended it + step: classify_context # draft step the obligation attaches to (if any) + unmet: "SCCmec-region candidate output category" + missing: "no wired input carries SCCmec cassette evidence" + resolved_by: null # Mold that closed it, once closed + supersedes: null # id whose justification this entry refutes, once resolved + note: "" # how it was closed or why surrendered +``` + +`kind` separates the two things an open entry can mean, which v1 conflated. A **`gap`** is +an obligation the pipeline cannot discharge: no producer is discoverable, Galaxy has no +datatype for the format, the source pinned no value. A **`dropped`** entry is source work +the pipeline *declined to carry* — a plane cut from the design, a parameter surface not +exposed, an output the repair removed. Both are honest; they are not the same admission, +and a reader cannot tell them apart from `unmet` / `missing` alone. Omit `kind` for a +plain gap. + +A `dropped` entry carries two more fields: + +```yaml +- id: annotation-planes-not-built + status: open + kind: dropped + raised_by: nextflow-summary-to-galaxy-data-flow + units: "25 of 31 designed nodes (rnaseq_short, gnomon, annot_proc, convert)" + because: "`gxwf tool-search asn1` returns no wrapper reading seq-submit; searched 2026-08-29" + unmet: "the pipeline's actual product — an annotated genome" + missing: "the carried spine masks an assembly; it does not annotate a genome" + note: "" +``` + +`units` names what was dropped in the source's own terms, so the drops can be totalled +without re-reading the design. + +`because` says why, and **must carry its citation** — the source construct that is not +there, the target limitation named specifically enough that someone could check it, the +search actually run. A drop's reason is a claim about the world, and the first real run is +why the burden exists: 25 of 31 planes were cut because "each needs an authored Galaxy +wrapper for an undocumented binary available only inside the pipeline's container," and a +Tool Shed search one phase later found the whole pipeline already wrapped at the exact +pinned version. The claim was not dishonest. It was never checked, and nothing in the +entry showed that it had not been. + +A reason you can cite, cite. A reason you cannot cite is still recorded — write it plainly +and leave it uncited rather than dressing it as evidence — but an uncited drop is not a +finding the run may reason from afterwards. It is a debt, and it should read as one. + +Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. + +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. + +## How to use the ledger + +This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. + +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. + +**Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. + +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. + +**Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. + +**Carry the rest untouched.** Entries you neither raise nor close pass through unchanged, statuses and provenance intact. Never renumber, re-word, or drop another Mold's entry. + +**Never fabricate around an entry.** An open obligation is the honest state. Inventing a connection, a tool id, or a value so the artifact looks complete converts a tracked gap into a silent defect — the specific failure the ledger replaces. + +**Record a drop as a drop.** When a decision removes source work from what the run will +carry — narrowing a design, cutting a plane, dropping an output during repair — append a +`kind: dropped` entry naming the `units` and a cited `because`. A cut recorded only as +prose in a brief is invisible to every later Mold; a cut recorded as a `gap` reads as +something the pipeline could not do rather than something it chose not to do. Recording +the drop is not the same as discharging it, and a well-written entry is not a substitute +for the work — an entry makes a cut *legible*, never *justified*. + +**A refuted justification reopens its decision.** When work you do contradicts the +`because` another entry rests on — discovery finds the wrapper a drop assumed absent, a +later brief supplies the evidence a narrowing assumed missing — set `supersedes` to that +entry's `id` on your own resolved entry, and return the superseded entry to `status: open` +with its `because` struck and the contradiction quoted in its `note`. Do not leave a +refuted rationale standing: the terminal writes these into the final artifact, and a +surrendered gap whose stated reason is known to be false is worse than an unexplained one. +Reopening does not oblige *you* to do the dropped work — it obliges the run to stop +claiming a settled reason it no longer has. + +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. + +**Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. + +## Role in topology repair + +In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): + +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. + +So the ledger is not a convenience here; it is the countable state the termination guard depends on. + +## Escalation budget (loop-level state) + +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: + +```yaml +topology_repair: + escalations: 2 # repairs invoked this run + cap: 5 # hard ceiling; at cap, still-open entries are surrendered + open_history: [4, 3, 2] # open blocking-entry count after each escalation +entries: + - id: sccmec-evidence-missing + status: open + blocking: true + # … per-entry shape above +``` + +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. +- `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. + +This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. + +## Threaded through the whole design tier + +The ledger is not loop-only. Every design-tier Mold in the source→Galaxy pipelines **consumes and re-emits** it — the interface, reference-data, data-flow, IWC-comparison, and template Molds (`*-summary-to-galaxy-interface`, `*-summary-to-galaxy-data-flow`, `*-summary-to-galaxy-reference-data`, `compare-against-iwc-exemplar`, `*-summary-to-galaxy-template`), alongside the change-set Molds on the update path. This is the direct answer to #281: the chain stops re-deriving the same unknowns because each Mold inherits them. + +The template Mold's computability review pass is the notable appender ahead of the loop — `*-summary-to-galaxy-template` re-reads each settled step and, where an output needs evidence no input carries, wires the producer or records the gap as a blocking entry. Source-summary Molds upstream of the design tier keep their own free-text open-questions; the design tier is where those formalize into the ledger. + +## Open work + +- Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. +- Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. +- Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. +- Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. +- Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. +- Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/workflow-brief-design.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/workflow-brief-design.md new file mode 100644 index 0000000..8bc1e6d --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/notes/workflow-brief-design.md @@ -0,0 +1,96 @@ +--- +type: research +title: "Workflow Brief" +tags: + - meta +status: draft +created: 2026-09-17 +revised: 2026-10-19 +revision: 6 +summary: "Markdown Workflow Brief definition and section contract for scope, constraints, agent environment, and explicit blockers." +--- + +# Workflow Brief + +A Workflow Brief is an expert-editable document describing what a workflow should achieve, what it should leave out, and the constraints under which an agent will build and run it. It is an input to design and implementation. The draft workflow captures the proposed steps, wiring, and tool state. + +## Document shape + +The brief is Markdown with a title and two sections: `## Workflow` and `## Agent Environment`. The workflow needs an objective; most details are optional. The agent environment must contain a description, which can explicitly record what still needs to be checked. + +The `workflowBriefSchema` declaration uses the shared engine described in [[markdown-document-contract]] and lives in `packages/gxwf-foundry/src/workflow-brief.ts`. An illustrative document lives in `packages/gxwf-foundry/test/fixtures/workflow-brief/read-alignment.md`. + +| Heading | Required | Content | +| --- | --- | --- | +| `## Workflow` | Yes | Scientific intent and scope. | +| `### Objective` | Yes, under Workflow | Scientific goal and desired outcome. | +| `### Sources` | No | Papers, interviews, repositories, relevant source pins, and evidence availability. | +| `### Scope` | No | Optional `#### Included` and `#### Excluded` describe selected work and boundaries. | +| `### Inputs and outputs` | No | Scientific roles and source-supported formats or organization. | +| `### Requirements and preferences` | No | Expert or source requirements, their strength, rationale, and evidence. | +| `### Acceptance criteria` | No | High-level behaviors that establish success. | +| `### Open questions` | No | Unresolved choices that do not prevent work, each with a proposed answer and its evidence. | +| `### Decisions` | No | Expert decisions and rationale. | +| `### Related artifacts` | No | Links to source evidence and other relevant inputs. | +| `### Blockers` | No, under Workflow | Unresolved issues that prevent design or implementation. Any content blocks work. | +| `## Agent Environment` | Yes | Agent tooling and operating conditions. | +| `### Tooling` | No | Expected Foundry tooling and versions, and what is known about availability. | +| `### Constraints` | No | Agent workspace, network, installation, resource, and access restrictions. | +| `### Containerization` | No | Container preference and Docker, Singularity, or Apptainer availability. | +| `### Blockers` | No, under Agent Environment | Environment issues that prevent work. Any content blocks work. | + +Headings are case-insensitive; section order is flexible and additional sections are allowed. Recognized sections are unique within their parent. If present, ordinary sections need content beyond headings, comments, or separators. Blockers may be empty. Scope's Included and Excluded subsections are optional. + +## Workflow content + +The writing agent must record evidence and expert intent without making Galaxy design choices. It must not assume Galaxy datatypes, dataset collection structures, tool availability, interface labels, or concrete tool versions. Source formats and explicitly named source tools can be cited as evidence; they do not establish a Galaxy mapping or an installed wrapper. An expert may supply specific Galaxy requirements here, and the agent must identify them as expert-provided rather than inferred. + +The agent must not write a step graph, concrete tool state, or final test declarations into the brief. Interface design, data-flow design, tool discovery, and test development belong to the downstream design and implementation Molds. Inputs and outputs stay at scientific roles and source-supported descriptions. Acceptance criteria stay high-level; detailed fixtures and assertions belong to test development. + +Requirements and preferences describe what the expert or source asks for. Agent Environment Constraints describe restrictions on the agent's work. Uncertainty that prevents proceeding belongs in Blockers. Record every other unresolved choice in Open questions with a proposed answer and the evidence for it, such as source behavior, the expert's wording, or the available test data. Propose the answer the evidence best supports. Where expert intent and source behavior conflict, propose the expert's intent and cite the conflict. When the evidence is thin, still propose an answer, mark it low confidence, and say what evidence is missing. The expert accepts a proposal by leaving it unedited and overrides it by editing the answer. + +Write each open question as a bullet with `Proposed:` and `Evidence:` sub-bullets. Add `Confidence: low` and `Missing:` when the evidence is thin: + +```md +- Should runs of one sample be merged before trimming? + - Proposed: yes, merge them as the source does. + - Evidence: the source merges sample-sheet rows sharing a sample ID. The selection does not mention merging. + - Confidence: low + - Missing: whether the expert's samples ever span several runs. +``` + +## Agent environment + +Tooling records whether expected tools such as `gxwf`, `foundry`, and `planemo` are available at the required versions. Distinguish intended requirements from observed availability. Containerization records the preference and whether Docker, Singularity, or Apptainer is actually usable, rather than assuming that an executable name proves a working engine. + +A separate preflight result should record observed versions, container availability, source/reference access, writable paths, timestamps, and diagnostics. A requested installation strategy is not evidence of an installed executable. + +## Validation and blockers + +```sh +foundry validate-workflow-brief workflow-brief.md +foundry check-workflow-brief workflow-brief.md --json +``` + +Validation checks the title, required sections, unique recognized headings, content, and parent-scoped subsections. Example headings in fences, quotes, lists, or comments do not satisfy requirements. Success exits `0`, structural failures exit `3`, and input read failures exit `1`. + +The static readiness check adds a check for `### Blockers` under both Workflow and Agent Environment. A structurally valid document with content in either section exits `4`; a structurally valid document without blockers exits `0`. Omit or leave Blockers empty when there are no blockers. Do not write “none” there: any content, including that word, is blocking. Comments and separators alone are empty. The JSON result includes `valid`, `ready`, `errors`, and blocker bodies with their parent and source line. + +A clear static check only establishes structure and the absence of declared blockers. It does not establish semantic completeness, expert approval, or successful environment preflight. The harness must stop on declared blockers and apply its own review and preflight checks before implementation. + +The exported `WorkflowBrief` and `WorkflowBriefSection` types describe the parsed document. `WorkflowBriefReadinessResult` describes the static check. These checks read the document without changing it. + +## Lifecycle + +1. A source-to-brief producer assembles evidence, proposed scope, and unanswered questions without Galaxy design assumptions. +2. The expert edits the brief and settles blockers needed before the next stage. +3. The harness checks structure, declared blockers, review, and environment readiness. +4. Brief-to-Galaxy consumes the brief and referenced evidence, then runs design, draft, implementation, and test development. + +Brief-to-Galaxy and every design or implementation step must leave the brief unchanged. Record durable workflow knowledge in `open-requirements.ledger.yml`: evidence as discoveries, authorized choices as decisions, unmet needs as obligations, and changes to expert intent as proposed brief-change recommendations. Ordinary phase progress remains harness state. Updating the brief belongs to a separate expert editing step; an implementation agent must not resolve a blocker by rewriting its input. + +A hand-authored brief is a first-class input. Implementation should work in a fresh session using the document and its referenced artifacts. + +## Open design questions + +- Should expert questions live here or link to a separate decision record? diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/schemas/workflow-brief-schema.schema.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/schemas/workflow-brief-schema.schema.json new file mode 100644 index 0000000..92c431e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-interface/references/schemas/workflow-brief-schema.schema.json @@ -0,0 +1,103 @@ +{ + "name": "Workflow Brief", + "title": "required", + "sections": [ + { + "heading": "Workflow", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Objective", + "min": 1, + "max": 1, + "content": true + }, + { + "heading": "Sources", + "max": 1, + "content": true + }, + { + "heading": "Scope", + "max": 1, + "content": true, + "sections": [ + { + "heading": "Included", + "max": 1, + "content": true + }, + { + "heading": "Excluded", + "max": 1, + "content": true + } + ] + }, + { + "heading": "Inputs and outputs", + "max": 1, + "content": true + }, + { + "heading": "Requirements and preferences", + "max": 1, + "content": true + }, + { + "heading": "Acceptance criteria", + "max": 1, + "content": true + }, + { + "heading": "Open questions", + "max": 1, + "content": true + }, + { + "heading": "Decisions", + "max": 1, + "content": true + }, + { + "heading": "Related artifacts", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + }, + { + "heading": "Agent Environment", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Tooling", + "max": 1, + "content": true + }, + { + "heading": "Constraints", + "max": 1, + "content": true + }, + { + "heading": "Containerization", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/SKILL.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/SKILL.md new file mode 100644 index 0000000..f9e269e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/SKILL.md @@ -0,0 +1,103 @@ +--- +name: workflow-brief-to-galaxy-template +description: "Build a gxformat2 skeleton from a reviewed Workflow Brief and its Galaxy design briefs, consulting retained source evidence." +--- + +# workflow-brief-to-galaxy-template + +Follow the procedure below and use the artifact/reference sections as the runtime contract. + +## When To Use + +- Build a gxformat2 skeleton from a reviewed Workflow Brief and its Galaxy design briefs, consulting retained source evidence. + +## Inputs + +- Read artifact `workflow-brief`. Schema: workflow-brief-schema. Produced by `freeform-summary-to-workflow-brief`, `nextflow-summary-to-workflow-brief`. Expert-reviewed implementation contract governing scope and requirements. +- Read artifact `freeform-summary`. Optional; absence is allowed and must be reported honestly. Produced by `interview-to-freeform-summary`, `summarize-paper`. Optional narrative source evidence retained from brief production; consult it for source-backed detail within the selected scope. +- Read artifact `summary-nextflow`. Optional; absence is allowed and must be reported honestly. Schema: summary-nextflow. Produced by `summarize-nextflow`. Optional structured Nextflow source evidence; source processes and channels are evidence, not a Galaxy topology mandate. +- Read artifact `workflow-brief-galaxy-interface`. Produced by `workflow-brief-to-galaxy-interface`. Galaxy interface brief from workflow-brief-to-galaxy-interface that pins workflow inputs, outputs, labels. +- Read artifact `workflow-brief-galaxy-data-flow`. Produced by `workflow-brief-to-galaxy-data-flow`. Galaxy data-flow brief from workflow-brief-to-galaxy-data-flow that pins abstract operations and collection choices. +- Read artifact `iwc-comparison-notes`. Produced by `compare-against-iwc-exemplar`. Structural diff guidance from compare-against-iwc-exemplar (run on the design brief); steers the skeleton toward IWC-aligned structure before per-step authoring. Carries an inline gxformat2 excerpt of the nearest exemplar. +- Read artifact `iwc-exemplar-gxformat2`. Produced by `compare-against-iwc-exemplar`. Cleaned gxformat2 view of the nearest IWC exemplar's relevant subgraph from compare-against-iwc-exemplar; pattern-match the draft's input/collection shapes, map-over wiring, output promotion, and post-job actions against this concrete idiom. Absent when no nearest exemplar was found. +- Read artifact `open-requirements-ledger`. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried obligations ledger open-requirements-ledger: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one. + +## Outputs + +- Write artifact `galaxy-workflow-draft` as `galaxy-workflow-draft.gxwf.yml`. Format: `yaml`. Schema: galaxy-workflow-draft. gxformat2 draft (see galaxy-workflow-draft-format): topology fully resolved (workflow inputs, outputs, step set, edges); tool_id / state / tool_shed_repository and wrapper-determined port names may be TODO with free-text _plan_state / _plan_context / _plan_in / _plan_out per step for later implementation Molds. +- Write artifact `open-requirements-ledger` as `open-requirements.ledger.yml`. Format: `yaml`. Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact. + +## Required Tools + +- **`gxwf`** (gxwf). `npm install -g '@galaxy-tool-util/cli@^1.15.0'`. + Ephemeral run: `npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf`. + Check: `gxwf --help | grep -q draft-validate`. + Docs: https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli + +## Load Upfront + +- `references/notes/galaxy-data-flow-draft-contract.md`: Research note copied verbatim into the bundle. Respect the handoff from the Workflow-Brief-to-Galaxy interface and data-flow briefs to the gxformat2 skeleton. Use when: translating abstract nodes, unresolved tool needs, and placeholder transformations into template TODOs. +- `references/notes/galaxy-workflow-draft-format.md`: Research note copied verbatim into the bundle. Emit the gxformat2 draft superset: TODO tool_id, optional state / tool_shed_repository, and per-step _plan_state / _plan_context planning fields. +- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Close the open entries the settled topology discharges, and append a blocking entry for any settled step whose declared output no wired input can supply — the computability gap gxwf validation cannot see, raised here rather than left for the per-step loop to hit. +- `references/notes/workflow-brief-design.md`: Research note copied verbatim into the bundle. Keep the reviewed brief authoritative while using retained summaries as evidence rather than replacement intent. +- `references/schemas/galaxy-workflow-draft.schema.json`: Schema file copied verbatim into the bundle. Output contract: the emitted gxformat2 draft conforms to galaxy-workflow-draft. Cast bundles the JSON Schema so the skill carries its output shape alongside the draft-validate CLI checks. +- `references/schemas/workflow-brief-schema.schema.json`: Schema file copied verbatim into the bundle. Carry the required implementation-contract shape into skeleton construction. + +## Load On Demand + +- `references/cli/draft-next-step.json`: CLI command reference packaged as a sidecar. Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop. Use when: deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step. +- `references/cli/draft-validate.json`: CLI command reference packaged as a sidecar. Validate the emitted draft against draft-contract rules (sentinel form, topology, _plan_* placement) before handing off. Use when: after writing or modifying the draft workflow file. +- `references/patterns/galaxy-collection-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded collection pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges. +- `references/patterns/galaxy-conditionals-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded conditional pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches. +- `references/patterns/galaxy-interval-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded genomic-interval pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for interval overlap, merge, coverage, windowing, masking, or set-algebra on coordinate features. +- `references/patterns/galaxy-tabular-patterns.md`: Pattern note copied verbatim into the bundle. Use corpus-grounded tabular pattern guidance for unresolved skeleton steps. Use when: adding TODO steps for tabular filtering, projection, joins, aggregation, text-processing recipes, or tabular-collection bridges. +- `references/notes/galaxy-collection-semantics.md`: Research note copied verbatim into the bundle. Preserve Galaxy collection typing and map-over/reduction semantics in the gxformat2 skeleton. Use when: creating workflow inputs, outputs, and placeholder connections involving collections. +- `references/notes/galaxy-collection-semantics.yml`: Companion file copied verbatim into the bundle. Sibling of `references/notes/galaxy-collection-semantics.md`; read it where that note directs. +- `references/notes/galaxy-workflow-comments.md`: Research note copied verbatim into the bundle. Group the settled step set into titled stage frames (the gxformat2 `comments:` array) so the skeleton carries the analysis-stage narrative IWC authors annotate by hand. Schema-legal and optional. Use when: after topology is settled and the skeleton can be partitioned into named analysis stages (inputs, per-stage step clusters, parameter-derivation knots, visualization/outputs). +- `references/notes/galaxy-workflow-testability-design.md`: Research note copied verbatim into the bundle. Choose stable workflow input/output labels, testable checkpoint outputs, and fixture-compatible workflow interfaces while drafting the skeleton. Use when: the template decides workflow inputs, workflow outputs, promoted checkpoints, or collection output identifiers that future tests will need to address. + +## Validation + +- Validate `galaxy-workflow-draft.gxwf.yml` for artifact `galaxy-workflow-draft` against the galaxy-workflow-draft schema when a validator is available. + +## Procedure + +Read the unchanged Workflow Brief and its Galaxy interface and data-flow briefs. Consult the original source artifact and retained `freeform-summary` or `summary-nextflow` when present. Emit a gxformat2 skeleton with workflow inputs, workflow outputs, placeholder steps, rough connections, and TODO slots for later implementation skills. + +The Workflow Brief is the implementation contract. Source artifacts are an evidence dossier: use them to substantiate methods, formats, parameters, and named tools within the selected scope, never to reintroduce excluded work or override expert requirements. A hand-authored brief without a source summary remains a valid input; record consequential missing evidence in the ledger instead of inventing it. + +Topology is this skill's job to settle. The output must be concrete gxformat2: workflow inputs with their final collection shapes and formats, workflow outputs, the step set, the producer→consumer edge graph, branches, and `when:` guards are all decided here. The upstream Workflow-Brief-to-Galaxy interface and data-flow briefs guide those decisions, but if they hedge or leave a topology choice open, this skill makes the call from reviewed intent, source evidence, IWC exemplars, and pattern pages — never emit a topology `TODO`. Wrapper resolution, by contrast, is **evidence-gated, not source-gated**: resolve each tool step to the tier its evidence supports — **Resolved** (fully concrete, no `_plan_*`), **Identity-pinned** (concrete `tool_id`, parameters and changeset left to the per-step skill), or **Deferred** (`tool_id: TODO`) — as defined in galaxy-workflow-draft-format. Capture whatever you defer in the `_plan_*` family (`_plan_state`, `_plan_context`, `_plan_in`, `_plan_out`) so the per-step skill has the source evidence and constraints it needs. + +Evidence tendency: narrative sources name tools less often than structured pipeline summaries, so their steps land in **Deferred** more often — but source evidence that names a specific tool/version can harden to the matching tier, and a corpus-confirmed utility wrapper is not deferred merely because the source is informal. When deferring a domain tool, cite the originating paper section, interview answer, pipeline location, figure, or supplementary table in `_plan_context`, and record vague intent in `_plan_state` so the per-step skill knows the evidence ceiling. + +Before handing off, sanity-check that each step is computable from what feeds it. Once the step set is settled, re-read it and ask, for each step, whether the operation its intent implies can actually be produced from the inputs you wired. The connection graph only knows that ports connect — not what each port is supposed to *contain* — so an output that needs evidence no input carries will validate fine and still be impossible to implement. Where you spot that gap, don't leave it implicit: wire (or add) the step that supplies the missing input, or record the unmet need plainly in `_plan_state` so the per-step skill or a reviewer can act on it rather than discover it late. + +Things worth a second look: + +- an output column or field that no wired input carries; +- an aggregate or summary whose grouping key isn't present upstream; +- a filter or threshold whose criterion isn't produced by any input; +- a join whose key doesn't exist on both sides; +- a step whose `_plan_*` promises more than its `in:` ports can supply; +- if classification step, is that classification/enumeration possible only from inputs. + +Optionally, once topology is settled, group the step set into titled stage frames via the gxformat2 `comments:` array (one frame per analysis stage, `contains_steps:` populated, color decorative) — see galaxy-workflow-comments for the convention. + +Before handing off, check each settled step is computable from what feeds it. The connection graph knows that ports connect, not what they carry — so a declared output that needs evidence no wired input supplies will validate yet can't be implemented. Where you find that gap, wire (or add) the producer; if you can't, record the vague intent in `_plan_state` and append a **blocking** entry to the open-requirements-ledger naming the step, the uncomputable output, and the missing evidence, so the per-step loop or repair-galaxy-draft-topology acts on it rather than discovering it late. + +Output shape is gxformat2 with wrapper-tier relaxations and `_plan_state` / `_plan_context` / `_plan_in` / `_plan_out` per tool step — see galaxy-workflow-draft-format. Refinement open work for those planning fields lives in `refinement.md`. + +## Feedback Mode + +- Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. +- When enabled, read `_feedback.md` before doing the work and use its registered `foundry-feedback.ledger.yml` protocol. +- Preserve harness-owned run and phase state. Append only concrete observations about a canonical Foundry source asset or a related project that this run showed to be at fault; do not put ordinary workflow requirements in this ledger. +- Before reporting completion, make one explicit pass over the work you just did. Do not ask yourself whether anything was unclear — recall what happened: where you guessed at something the instructions should have settled, needed information this bundle does not carry, hit an instruction that contradicted another or contradicted the artifacts in front of you, used a packaged reference that did not cover your case, or did something the procedure never describes. +- Append an entry for each such event that clears the protocol's bar. If none do, append nothing and report `no feedback` explicitly. Silence and a clean pass are not the same thing, and nothing downstream can tell them apart unless you say which one it was. +- Pass the same ledger path to any subagent used for this work, and merge updates serially so one writer cannot overwrite another. + +## Runtime Notes + +- Do not read Foundry source files at runtime; use only files packaged in this skill bundle and user-supplied artifacts. +- Preserve declared artifact filenames unless the user or harness supplies explicit paths. +- Carry unresolved assumptions into the output artifact instead of silently inventing missing source evidence. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_feedback.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_feedback.md new file mode 100644 index 0000000..61a4bd4 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_feedback.md @@ -0,0 +1,143 @@ +# Foundry feedback ledger + +The `foundry-feedback-ledger` is an opt-in runtime artifact for observations about the Foundry +assets used during a real cast-skill or pipeline run. It records actionable gaps, defects, +friction, and wishes about Molds and their authored references. It does not record unmet +requirements in the workflow being built; those belong in the open-requirements ledger. + +Every entry is destined for `galaxyproject/foundry`. When a run shows that a related project is +at fault — Galaxy, a tool wrapper, the IWC corpus, a harness CLI — that observation still belongs +here, recorded with the analysis that points at the upstream. The Foundry is where the run +context lives, so it is where the issue is tracked; a maintainer forwards from there. Never tell +a user to take an observation to another tracker instead. + +The registered filename is `foundry-feedback.ledger.yml`. Feedback-aware callers pass the same +file through every skill invocation in a run. + +## Ledger shape + +```yaml +run: + pipeline: nextflow-to-galaxy + run_slug: sarek-galaxy + status: running + phases: + - { n: 1, kind: mold, skill: summarize-nextflow, status: done, feedback_checked: true } + - { n: 2, kind: mold, skill: nextflow-summary-to-galaxy-interface, status: running } +entries: + - id: channel-shape-note-silent-on-optional-outputs + raised_by: nextflow-summary-to-galaxy-interface + observed_in: + mold: nextflow-summary-to-galaxy-interface + path: content/molds/nextflow-summary-to-galaxy-interface/index.md + revision: 6 + content_hash: 82d8... + foundry_head: d9e0b09 + subject: + kind: research + label: nextflow-to-galaxy-channel-shape-mapping + locator: content/research/nextflow-to-galaxy-channel-shape-mapping/index.md + content_hash: a18c... + kind: gap + severity: major + what: "The note does not cover optional outputs." + expected: "State how an absent optional channel maps to the Galaxy interface." + evidence: "Interface design stopped at the optional output decision." + status: open + issue: null +``` + +Entry `kind` is `gap`, `defect`, `friction`, or `wish`. Severity is `blocker`, `major`, or +`minor`. Status is `open`, `filed`, `duplicate`, or `wontfix`. + +## Run lifecycle + +For a pipeline run, the harness creates the file before phase 1 with `run.status: running`, an +empty `entries` list, and the complete top-level phase roster from its assembly record. All +phases begin `pending`. The harness changes a phase to `running` immediately before invocation +and to `done` on success. A phase that terminates the run and the run itself become `failed`; a +graceful user stop makes the run `cancelled`. A hard interruption naturally leaves the run +`running`, which is distinguishable from success without a recovery write. + +A phase also carries `feedback_checked`. The harness sets it true only when that phase reported a +feedback outcome — entry ids it appended, or an explicit `no feedback`. It is absent or false +otherwise, including for a phase that simply never said. This is what separates a run that looked +for friction and found none from a run that never looked. + +Loop phases remain one row and increment `iterations` after each successful iteration. Branch +phases remain one row and record the chosen path in `selected`. After every intended phase is +done, the harness sets `run.status: complete`. + +For a standalone skill invocation, the first feedback-aware skill creates a ledger with +`pipeline: null`, the caller's run slug when one exists, and one phase row for itself. It owns +that row's lifecycle as well as any entry it appends. + +No ledger means feedback mode was off. An empty `entries` list is evidence of a clean run only +when `run.status` is `complete` **and** every phase is `feedback_checked`. A complete run with +unchecked phases is complete but unreviewed; an empty incomplete ledger is neither. + +## When to append + +Appending is a deliberate pass, not a background reflex. Before a skill reports completion it +recalls what actually happened during its work — where it guessed at something the instructions +should have settled, needed information its bundle does not carry, hit an instruction that +contradicted another or contradicted the artifacts in front of it, used a reference that did not +cover the case, or did something its procedure never describes. Those are recallable events. "Was +anything unclear?" is not a useful question to ask at that moment and reliably answers itself no. + +The pass ends in one of two outcomes, and the outcome is always stated: entries appended, or +`no feedback`. A skill that says nothing has not reported a clean pass. + +Two entry shapes are admissible. Both require that `what` names a concrete problem, `expected` +states a proposed correction, and the entry is not already present in this ledger for the same +subject and correction. + +**A Foundry source asset.** The observation concerns Foundry-authored content, implementation, or +a registry that could be changed directly, and `subject.locator` canonically identifies it. This +is the ordinary case and the one the clustering and hash checks are built for. + +**A related project.** The run showed that a project the Foundry drives or cites is at fault — +Galaxy, a Tool Shed wrapper, the IWC corpus, `gxwf`, planemo. Use `subject.kind: related-project`, +name the project and the specific component in `label`, and put its public identity in +`locator` (a repository URL, a tool id, an IWC workflow id). There is no `content_hash`. State +in `what` and `evidence` what the run actually observed, and in `expected` what the upstream +would have to do. The entry is still filed against the Foundry. + +Do not use feedback entries for ordinary workflow requirements, tool-selection uncertainty, or +an unsupported preference with nothing concrete to change. + +## Canonical identity and provenance + +For a Foundry subject, use a repository-relative authored path as `subject.locator`; for a +package-export schema, use `package://#`. Generated skills and harnesses are +never subjects; identify the Mold, Pipeline, caster, assembler, or registry that must change. + +Initial subject kinds are `mold`, `pipeline`, `pattern`, `source-pattern`, `research`, `schema`, +`prompt`, `cli-tool`, `cli-command`, `meta`, `implementation`, and `registry`, plus +`related-project` for the upstream case above. `label` is for display only; the locator is the +identity and clustering key. + +A `related-project` subject is the one kind whose locator is not a Foundry path: it is public +upstream identity, carries no content hash, and is never checked against Foundry main. + +For the running Mold, copy `mold.name`, `mold.path`, `mold.revision`, and `mold.content_hash` +from `_provenance.json` into `observed_in`. Record its `mold.commit` value as `foundry_head`, but +never use that commit as exact content identity: it can be null or predate the bytes in the cast. +For a packaged reference, copy its `refs[].src` and `refs[].src_hash` into the subject locator and +content hash. When the subject is the Mold itself, use its authored path and +`observed_in.content_hash`. + +## Safe updates + +Read the complete ledger immediately before changing it. Preserve run state and all entries you +do not own. Append a complete entry in one rewrite; do not leave a partial YAML document. The +harness is the sole writer of pipeline lifecycle fields, while each skill writes only entries. + +Keep evidence concise and safe to upstream. Never copy credentials, tokens, private repository +URLs, proprietary source text, patient or participant data, or user-identifying filesystem +paths into the ledger. Replace a run-directory prefix with `/` and summarize sensitive +evidence instead of quoting it. + +The reporting skill may set an open entry to `filed`, `duplicate`, or `wontfix` and record the +resolved issue URL after the corresponding disposition is confirmed. It never deletes an entry; +the ledger remains the run audit trail. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_provenance.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_provenance.json new file mode 100644 index 0000000..a07e44a --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_provenance.json @@ -0,0 +1,365 @@ +{ + "provenance_schema_version": 4, + "cast_target": "claude", + "mold": { + "name": "workflow-brief-to-galaxy-template", + "path": "content/molds/workflow-brief-to-galaxy-template/index.md", + "revision": 8, + "content_hash": "ff684b1c73690303127c8465ccdcdf9ffbfe1c95d72a6ed224ca33f153c5ea1d", + "commit": "db8801f2f0a31746e35cf399dd96cdbb8aa36dcf" + }, + "cast_at": "2026-11-10T10:22:52.390Z", + "refs": [ + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[draft-next-step]]", + "src": "content/cli/gxwf/draft-next-step.md", + "dst": "references/cli/draft-next-step.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Confirm the per-step loop can walk the emitted draft: it descends into draft subworkflows and reports the picked step's full path, so nesting steps in a subworkflow does not hide them from the loop.", + "trigger": "When deciding whether to group steps into a subworkflow, or after writing the draft to confirm the loop picks a step.", + "verification": "Cast the skill, emit a draft with a drafty step inside a subworkflow, confirm draft-next-step returns that step's full path.", + "src_hash": "d4fe7945945a57846fe8b168e7bacba04f739de1d86bc613ae6f0e5879ac2f55", + "dst_hash": "8638645f5974d8072dc3c63e522b7ab25d562f8d6217ea93b61f0a29f371d6f7", + "source": "deterministic" + }, + { + "kind": "cli-command", + "mode": "sidecar", + "ref": "[[draft-validate]]", + "src": "content/cli/gxwf/draft-validate.md", + "dst": "references/cli/draft-validate.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Validate the emitted draft against draft-contract rules (sentinel form, topology, _plan_* placement) before handing off.", + "trigger": "After writing or modifying the draft workflow file.", + "verification": "Cast the skill, run on a representative paper-derived summary, confirm draft-validate diagnostics route back.", + "src_hash": "37cded427fbccf49ff19a6fe8b03f9e90857afe556e5db0e51a01402fbf33f1e", + "dst_hash": "afa59b0815308a340780247df0548909518564d58352a0c695ceca37a07eee5b", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-collection-patterns]]", + "src": "content/patterns/galaxy-collection-patterns.md", + "dst": "references/patterns/galaxy-collection-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Use corpus-grounded collection pattern guidance for unresolved skeleton steps.", + "trigger": "When adding TODO steps for collection cleanup, reshaping, relabeling, identifier synchronization, or collection-tabular bridges.", + "src_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "dst_hash": "2dbb4b7520ba30d3e9fdf68e6133fb35c597e7e8482478b4dee2abe9f415d6ef", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-conditionals-patterns]]", + "src": "content/patterns/galaxy-conditionals-patterns.md", + "dst": "references/patterns/galaxy-conditionals-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Use corpus-grounded conditional pattern guidance for unresolved skeleton steps.", + "trigger": "When adding TODO steps for optional steps, gating on non-empty results, routing between alternative outputs, or transform-or-pass-through branches.", + "src_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "dst_hash": "a021818026c66d41772b20265897ce14fe5de06b765ffe375039e238bff7da4a", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-interval-patterns]]", + "src": "content/patterns/galaxy-interval-patterns.md", + "dst": "references/patterns/galaxy-interval-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Use corpus-grounded genomic-interval pattern guidance for unresolved skeleton steps.", + "trigger": "When adding TODO steps for interval overlap, merge, coverage, windowing, masking, or set-algebra on coordinate features.", + "src_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "dst_hash": "103ad0288566f0bf721d54b8ecd49e2df0ac334b9030a754d884b2e13abdf73e", + "source": "deterministic" + }, + { + "kind": "pattern", + "mode": "verbatim", + "ref": "[[galaxy-tabular-patterns]]", + "src": "content/patterns/galaxy-tabular-patterns.md", + "dst": "references/patterns/galaxy-tabular-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Use corpus-grounded tabular pattern guidance for unresolved skeleton steps.", + "trigger": "When adding TODO steps for tabular filtering, projection, joins, aggregation, text-processing recipes, or tabular-collection bridges.", + "src_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "dst_hash": "54219d2387f3831ebf5a5a918d7f588f75baa120768ed627ec6c9c4b0f309539", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-collection-semantics]]", + "src": "content/research/galaxy-collection-semantics/index.md", + "dst": "references/notes/galaxy-collection-semantics.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Preserve Galaxy collection typing and map-over/reduction semantics in the gxformat2 skeleton.", + "trigger": "When creating workflow inputs, outputs, and placeholder connections involving collections.", + "license": "MIT", + "license_file": "LICENSES/galaxy.LICENSE", + "src_hash": "4ddc7587e187cf38a3dca02d6966e2f6ea58f9faf9bd97034ad513eb4c4c638b", + "dst_hash": "4ddc7587e187cf38a3dca02d6966e2f6ea58f9faf9bd97034ad513eb4c4c638b", + "source": "deterministic", + "license_file_hash": "3fdd32b75cfad3266871456423f4d91529953629fe4bc803c3df2f445fd8a858" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-collection-semantics]]", + "src": "content/research/galaxy-collection-semantics/galaxy-collection-semantics.yml", + "dst": "references/notes/galaxy-collection-semantics.yml", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Preserve Galaxy collection typing and map-over/reduction semantics in the gxformat2 skeleton.", + "trigger": "When creating workflow inputs, outputs, and placeholder connections involving collections.", + "companion_of": "references/notes/galaxy-collection-semantics.md", + "license": "MIT", + "license_file": "LICENSES/galaxy.LICENSE", + "src_hash": "f4e9d0a55459b6080b826e2ae41a920c9404f95fda0e518be8f86bc3d2e73e6e", + "dst_hash": "f4e9d0a55459b6080b826e2ae41a920c9404f95fda0e518be8f86bc3d2e73e6e", + "source": "deterministic", + "license_file_hash": "3fdd32b75cfad3266871456423f4d91529953629fe4bc803c3df2f445fd8a858" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-data-flow-draft-contract]]", + "src": "content/research/galaxy-data-flow-draft-contract/index.md", + "dst": "references/notes/galaxy-data-flow-draft-contract.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Respect the handoff from the Workflow-Brief-to-Galaxy interface and data-flow briefs to the gxformat2 skeleton.", + "trigger": "When translating abstract nodes, unresolved tool needs, and placeholder transformations into template TODOs.", + "verification": "Promote after two worked brief-to-Galaxy templates preserve the design-brief/template split without schema changes.", + "src_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "dst_hash": "605cc3f1e4da8f0c0b2b62a4c05c6d123ab0aff7b5e373b377f5bdb2eb33be67", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-workflow-comments]]", + "src": "content/research/galaxy-workflow-comments/index.md", + "dst": "references/notes/galaxy-workflow-comments.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Group the settled step set into titled stage frames (the gxformat2 `comments:` array) so the skeleton carries the analysis-stage narrative IWC authors annotate by hand. Schema-legal and optional.", + "trigger": "After topology is settled and the skeleton can be partitioned into named analysis stages (inputs, per-stage step clusters, parameter-derivation knots, visualization/outputs).", + "src_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", + "dst_hash": "ac413a82024728288ec4389331688eaa0dfcf0341f0432be4ea36524ff00ab9b", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-workflow-draft-format]]", + "src": "content/research/galaxy-workflow-draft-format/index.md", + "dst": "references/notes/galaxy-workflow-draft-format.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Emit the gxformat2 draft superset: TODO tool_id, optional state / tool_shed_repository, and per-step _plan_state / _plan_context planning fields.", + "verification": "Promote after a downstream per-step implementation Mold consumes _plan_state and _plan_context without re-deriving intent from source evidence.", + "src_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", + "dst_hash": "8bb5cad935b261021277487ef21ae0d2cfc6739908886e88d9a2fcf4acf05597", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-workflow-testability-design]]", + "src": "content/research/galaxy-workflow-testability-design/index.md", + "dst": "references/notes/galaxy-workflow-testability-design.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Choose stable workflow input/output labels, testable checkpoint outputs, and fixture-compatible workflow interfaces while drafting the skeleton.", + "trigger": "When the template decides workflow inputs, workflow outputs, promoted checkpoints, or collection output identifiers that future tests will need to address.", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[open-requirements-ledger]]", + "src": "content/research/open-requirements-ledger/index.md", + "dst": "references/notes/open-requirements-ledger.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Close the open entries the settled topology discharges, and append a blocking entry for any settled step whose declared output no wired input can supply — the computability gap gxwf validation cannot see, raised here rather than left for the per-step loop to hit.", + "verification": "Promote after a worked run shows entries this Mold appends or resolves are consumed downstream without re-derivation.", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[workflow-brief-design]]", + "src": "content/research/workflow-brief-design/index.md", + "dst": "references/notes/workflow-brief-design.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Keep the reviewed brief authoritative while using retained summaries as evidence rather than replacement intent.", + "verification": "Confirm a worked run preserves brief scope and bytes through topology construction.", + "src_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "dst_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[galaxy-workflow-draft]]", + "src": "package://@galaxy-tool-util/schema#galaxyWorkflowDraftJsonSchema", + "dst": "references/schemas/galaxy-workflow-draft.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Output contract: the emitted gxformat2 draft conforms to [[galaxy-workflow-draft]]. Cast bundles the JSON Schema so the skill carries its output shape alongside the [[draft-validate]] CLI checks.", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "dst_hash": "fd49d477f73427ca85236ba602b6c06bcda65123c98ee7c7eb8fd79b177aebfd", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[workflow-brief-schema]]", + "src": "package://@galaxy-foundry/gxwf-foundry#workflowBriefSchema", + "dst": "references/schemas/workflow-brief-schema.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Carry the required implementation-contract shape into skeleton construction.", + "src_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "dst_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "source": "deterministic" + } + ], + "artifacts": { + "produces": [ + { + "id": "galaxy-workflow-draft", + "kind": "yaml", + "default_filename": "galaxy-workflow-draft.gxwf.yml", + "schema": "[[galaxy-workflow-draft]]", + "description": "gxformat2 draft (see [[galaxy-workflow-draft-format]]): topology fully resolved (workflow inputs, outputs, step set, edges); tool_id / state / tool_shed_repository and wrapper-determined port names may be TODO with free-text _plan_state / _plan_context / _plan_in / _plan_out per step for later implementation Molds." + }, + { + "id": "open-requirements-ledger", + "kind": "yaml", + "default_filename": "open-requirements.ledger.yml", + "description": "Carried obligations ledger re-emitted by this step: entries it appended or closed updated, every other entry passed through with its provenance intact." + } + ], + "consumes": [ + { + "id": "workflow-brief", + "description": "Expert-reviewed implementation contract governing scope and requirements.", + "inherited_schema": "[[workflow-brief-schema]]", + "producers": [ + "freeform-summary-to-workflow-brief", + "nextflow-summary-to-workflow-brief" + ] + }, + { + "id": "freeform-summary", + "description": "Optional narrative source evidence retained from brief production; consult it for source-backed detail within the selected scope.", + "optional": true, + "producers": [ + "interview-to-freeform-summary", + "summarize-paper" + ] + }, + { + "id": "summary-nextflow", + "description": "Optional structured Nextflow source evidence; source processes and channels are evidence, not a Galaxy topology mandate.", + "optional": true, + "inherited_schema": "[[summary-nextflow]]", + "producers": [ + "summarize-nextflow" + ] + }, + { + "id": "workflow-brief-galaxy-interface", + "description": "Galaxy interface brief from [[workflow-brief-to-galaxy-interface]] that pins workflow inputs, outputs, labels.", + "producers": [ + "workflow-brief-to-galaxy-interface" + ] + }, + { + "id": "workflow-brief-galaxy-data-flow", + "description": "Galaxy data-flow brief from [[workflow-brief-to-galaxy-data-flow]] that pins abstract operations and collection choices.", + "producers": [ + "workflow-brief-to-galaxy-data-flow" + ] + }, + { + "id": "iwc-comparison-notes", + "description": "Structural diff guidance from [[compare-against-iwc-exemplar]] (run on the design brief); steers the skeleton toward IWC-aligned structure before per-step authoring. Carries an inline gxformat2 excerpt of the nearest exemplar.", + "producers": [ + "compare-against-iwc-exemplar" + ] + }, + { + "id": "iwc-exemplar-gxformat2", + "description": "Cleaned gxformat2 view of the nearest IWC exemplar's relevant subgraph from [[compare-against-iwc-exemplar]]; pattern-match the draft's input/collection shapes, map-over wiring, output promotion, and post-job actions against this concrete idiom. Absent when no nearest exemplar was found.", + "producers": [ + "compare-against-iwc-exemplar" + ] + }, + { + "id": "open-requirements-ledger", + "description": "Carried obligations ledger [[open-requirements-ledger]]: the run's open, resolved, and surrendered entries with their provenance. Absent on the first Mold of a run; start an empty one.", + "producers": [ + "advance-galaxy-draft-step", + "apply-galaxy-workflow-changeset", + "compare-against-iwc-exemplar", + "cwl-summary-to-galaxy-data-flow", + "cwl-summary-to-galaxy-interface", + "cwl-summary-to-galaxy-template", + "freeform-summary-to-galaxy-data-flow", + "freeform-summary-to-galaxy-interface", + "freeform-summary-to-galaxy-template", + "implement-galaxy-tool-step", + "interview-to-galaxy-workflow-changeset", + "mature-galaxy-workflow-for-iwc", + "nextflow-summary-to-galaxy-data-flow", + "nextflow-summary-to-galaxy-interface", + "nextflow-summary-to-galaxy-reference-data", + "nextflow-summary-to-galaxy-template", + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" + ] + } + ] + } +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_required_tools.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_required_tools.json new file mode 100644 index 0000000..1a8a44d --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_required_tools.json @@ -0,0 +1,17 @@ +[ + { + "tool": "gxwf", + "origin": "npm", + "package": "@galaxy-tool-util/cli", + "package_version": "^1.15.0", + "invoke": "gxwf", + "invoke_fallback": "npx --yes --package @galaxy-tool-util/cli@1.15.0 gxwf", + "availability_check": "gxwf --help | grep -q draft-validate", + "docs_url": "https://github.com/jmchilton/galaxy-tool-util-ts/tree/main/packages/cli", + "source": "implied", + "implied_by": [ + "gxwf draft-next-step", + "gxwf draft-validate" + ] + } +] diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_verify.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_verify.json new file mode 100644 index 0000000..f40b330 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/_verify.json @@ -0,0 +1,31 @@ +{ + "verify_schema_version": 1, + "entries": [ + { + "artifact_id": "summary-nextflow", + "direction": "input", + "kind": "json", + "default_filename": "summary-nextflow.json", + "schema": "[[summary-nextflow]]", + "validator_bin": "foundry", + "args": [ + "validate-summary-nextflow", + "{artifact_path}" + ] + }, + { + "artifact_id": "workflow-brief", + "direction": "input", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "validator_bin": "foundry", + "args": [ + "validate-workflow-brief", + "--schema-sha256", + "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "{artifact_path}" + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/cli/draft-next-step.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/cli/draft-next-step.json new file mode 100644 index 0000000..296abe7 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/cli/draft-next-step.json @@ -0,0 +1,42 @@ +{ + "type": "cli-command", + "tool": "gxwf", + "command": "draft-next-step", + "summary": "Pick the next drafty step a harness should work on, or report no remaining work; deterministic topological + alphabetical tiebreak.", + "source_path": "content/cli/gxwf/draft-next-step.md", + "source_revision": 1, + "package": "@galaxy-tool-util/cli", + "description": "Pick the next step a downstream agent should work on (or report no remaining work)", + "synopsis": "gxwf draft-next-step [options] ", + "args": [ + { + "raw": "file", + "name": "file", + "required": true, + "variadic": false, + "description": "Draft workflow file (.gxwf.yml)" + } + ], + "options": [ + { + "flags": "--format ", + "name": "format", + "description": "Force input format: format2 (default; native is rejected)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--output-format ", + "name": "outputFormat", + "description": "Output format: json (default) or markdown", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false, + "defaultValue": "json" + } + ], + "body": "# `gxwf draft-next-step`\n\nDeterministic next-step picker for the per-step authoring loop. Wraps `nextDraftStep` from `@galaxy-tool-util/schema`. Native (.ga) input is rejected — drafts are format2-only.\n\nThe harness loop reads as: `while (next = gxwf draft-next-step ).draft: invoke per-step skill on next.step`.\n\n## Output\n\nDefault output is pretty JSON — the agent-loop wire format. Same input → byte-identical output. `--output-format markdown` renders a human-glance checklist instead.\n\nJSON fields:\n\n- `draft` (bool) — `true` while a drafty step remains, `false` at the terminal cases below.\n- `step` (string array) — the picked step's **path**: a single id at top level (`[\"merge assembled transcripts\"]`), or the full path through containing subworkflow steps when the pick is nested. The leaf (last element) is the step to work on.\n- `work` (string array) — the chosen step's remaining `TODO[...]` sentinels followed by its `_plan_*` fields, in checklist order.\n\nAt the terminal cases the object is just `{ \"draft\": false }` — no `step` or `work` keys.\n\nTwo terminal cases both surface as `draft: false`:\n\n- The workflow's class is already `GalaxyWorkflow` (concretization complete, class promoted).\n- The class is still `GalaxyWorkflowDraft` but every step is fully resolved (no `TODO_*`, no `_plan_*`). The harness's next move is [[draft-extract]] (which calls `promoteFullyConcreteDrafts` to flip the class) followed by terminal [[validate]].\n\n## Examples\n\n```bash\ngxwf draft-next-step workflow.gxwf.yml\ngxwf draft-next-step workflow.gxwf.yml --output-format markdown\n```\n\nHarness loop sketch:\n\n```bash\nwhile [ \"$(gxwf draft-next-step workflow.gxwf.yml | jq -r .draft)\" = \"true\" ]; do\n step=$(gxwf draft-next-step workflow.gxwf.yml | jq -r '.step | last')\n # invoke per-step skill on step, which mutates workflow.gxwf.yml in place\ndone\n```\n\n## Gotchas\n\n- Pick order is **topological** (a step waits until its `in:` sources are concretized) with **alphabetical tiebreak** on step id at the same level. Same input across runs returns the same pick — safe to call repeatedly without state.\n- Descends into draft subworkflows; the returned `step` array is the full path through containing subworkflow steps (leaf last).\n- A `draft: false` result does **not** mean the workflow has passed [[validate]] yet. It means the per-step authoring loop is done; the harness still owns the promote + terminal-validate pass.\n- This command does not mutate the workflow file. It's a pure observer.\n- Insertion order of step dicts does not affect output — semantically equal workflows produce identical picks." +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/cli/draft-validate.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/cli/draft-validate.json new file mode 100644 index 0000000..58fda0f --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/cli/draft-validate.json @@ -0,0 +1,140 @@ +{ + "type": "cli-command", + "tool": "gxwf", + "command": "draft-validate", + "summary": "Validate a `class: GalaxyWorkflowDraft` workflow against draft-contract rules; with --concrete, also validate the extracted concrete subset.", + "source_path": "content/cli/gxwf/draft-validate.md", + "source_revision": 1, + "package": "@galaxy-tool-util/cli", + "description": "Validate a draft Galaxy workflow (class: GalaxyWorkflowDraft)", + "synopsis": "gxwf draft-validate [options] ", + "args": [ + { + "raw": "file", + "name": "file", + "required": true, + "variadic": false, + "description": "Draft workflow file (.gxwf.yml)" + } + ], + "options": [ + { + "flags": "--format ", + "name": "format", + "description": "Force format: format2 (native is rejected for drafts)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--json", + "name": "json", + "description": "Output structured JSON report", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--report-html [file]", + "name": "reportHtml", + "description": "Write HTML report to file (or stdout if omitted)", + "takesArgument": true, + "argumentPlaceholder": "[file]", + "optionalArgument": true, + "negatable": false + }, + { + "flags": "--report-markdown [file]", + "name": "reportMarkdown", + "description": "Write Markdown report to file (or stdout if omitted)", + "takesArgument": true, + "argumentPlaceholder": "[file]", + "optionalArgument": true, + "negatable": false + }, + { + "flags": "--concrete", + "name": "concrete", + "description": "Additionally extract the concrete subset and run the regular `gxwf validate` checks on it", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--cache-dir ", + "name": "cacheDir", + "description": "Tool cache directory (for --concrete tool-state validation)", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--offline", + "name": "offline", + "description": "Do not fetch uncached tools; read only the local cache (uncached tools are skipped)", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--galaxy-url ", + "name": "galaxyUrl", + "description": "Galaxy instance to try as a tool source after the ToolShed when fetching uncached tools", + "takesArgument": true, + "argumentPlaceholder": "", + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--no-tool-state", + "name": "toolState", + "description": "Skip tool-state validation in the --concrete pass (default: enabled when --concrete is set)", + "takesArgument": false, + "optionalArgument": false, + "negatable": true + }, + { + "flags": "--connections", + "name": "connections", + "description": "Validate connection-type compatibility on the --concrete subset (collection algebra, map-over)", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict", + "name": "strict", + "description": "Shorthand for --strict-structure --strict-encoding --strict-state", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict-structure", + "name": "strictStructure", + "description": "Reject unknown keys at envelope/step level", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict-encoding", + "name": "strictEncoding", + "description": "Reject JSON-string tool_state and format2 field misuse", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + }, + { + "flags": "--strict-state", + "name": "strictState", + "description": "Require every tool step to validate; no skips allowed", + "takesArgument": false, + "optionalArgument": false, + "negatable": false + } + ], + "body": "# `gxwf draft-validate`\n\nValidate a draft Galaxy workflow against the **draft contract**: sentinel form, dangling edge references, top-level `_plan_*` placement, `_plan_*` on fully-resolved tool steps, and recursive draft subworkflows. Native (.ga) input is rejected — drafts are format2-only.\n\nDistinct from [[validate]], which validates a fully concrete `class: GalaxyWorkflow` and would reject the draft relaxations outright. Use `draft-validate` during the per-step authoring loop; use `validate` at the terminal pass once `promoteFullyConcreteDrafts` has flipped the class.\n\n## Output\n\nDefault output is human-readable: counted buckets for structure / topology / semantic errors and warnings, plus a one-line survey (TODO sentinel count, paths carrying `_plan_*`). `--json` emits a `SingleDraftValidationReport`; `--report-html` and `--report-markdown` write the same data as a self-contained HTML page or templated Markdown. With `--concrete`, the report carries an optional `concrete: ConcreteValidationReport` whose buckets (`structure_errors`, `strict_structure_errors`, `strict_encoding_errors`, `strict_state_errors`, `tool_state`, `connection_report`) are **absent when the corresponding check did not run** — readers should treat absence as \"not run,\" not as \"passed.\"\n\n## Flags\n\n`--concrete` runs the extract+promote pipeline (`extractConcreteSubset` → `stripPlanFields` → `promoteFullyConcreteDrafts`) and applies the full concrete `gxformat2` validation surface to the result. The following pass-through flags only take effect under `--concrete`; passing them without it prints a stderr warning and no-ops:\n\n- `--cache-dir ` — tool cache for tool-state lookups.\n- `--no-tool-state` — skip tool-state validation on the concrete pass. Combined with `--strict-state`, the strict flag warns + no-ops (there's no state to be strict about).\n- `--connections` — run connection validation on the concrete subset.\n- `--strict` — escalate every strict bucket (structure, encoding, state) to error.\n- `--strict-structure` / `--strict-encoding` / `--strict-state` — escalate one bucket only.\n\nDraft structural errors gate the concrete pass entirely — the pipeline would mutate a malformed input, so the concrete section is skipped and only the draft buckets populate.\n\n## Examples\n\n```bash\ngxwf draft-validate workflow.gxwf.yml\ngxwf draft-validate workflow.gxwf.yml --json\ngxwf draft-validate workflow.gxwf.yml --concrete --json\ngxwf draft-validate workflow.gxwf.yml --concrete --strict --cache-dir ~/.cache/gxwf\ngxwf draft-validate workflow.gxwf.yml --report-html report.html\n```\n\n## Exit codes\n\n- `0` — clean draft (no structure / topology / semantic errors; warnings allowed). Under `--concrete`, also requires the concrete pass to be clean.\n- `1` — draft validation errors (topology or semantic). With `--concrete`, also returned when the concrete projection fails — including strict-* failures that `gxwf validate` would have exited `2` on standalone. The asymmetry is deliberate: the draft itself parsed, only its projection failed.\n- `2` — parse failure, native input rejected, structural decode failure (class is not `GalaxyWorkflowDraft`), or a stdout-sink collision between `--json` and `--report-{html,markdown}=-`.\n\n## Gotchas\n\n- The draft validator is **contract-shaped**, not workflow-shaped. It will not catch tool-state errors, parameter-name mismatches, or other concerns that belong to [[validate]] on the concretized subset. Use `--concrete` (or run [[draft-extract]] | [[validate]] manually) when you need both surfaces.\n- A draft with zero implemented steps still runs the `--concrete` pipeline, but the projection is empty — the concrete buckets report `skipped` semantics (absent in the JSON) rather than a green pass. Signal grows as the loop fills steps in; do not interpret early-loop runs as evidence the workflow concretizes correctly.\n- `_plan_*` on a fully-resolved tool step is a `semanticError`, not a warning. Strip planning fields explicitly via [[draft-extract]] (or `stripPlanFields`) when a step concretizes; the per-step Mold should never carry them forward.\n- Subworkflow draft roots are validated recursively; diagnostics carry the outer step path.\n- `TODO`, `TODO_x`, `TODO-x` are sentinels; `TODONE` and `TODOLIST` are not.\n- Prefer `--json` whenever a cast skill or harness needs to classify diagnostics." +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-collection-semantics.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-collection-semantics.md new file mode 100644 index 0000000..0d92bcf --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-collection-semantics.md @@ -0,0 +1,43 @@ +--- +type: research +title: "Galaxy collection semantics" +tags: + - target/galaxy +status: draft +created: 2026-04-30 +revised: 2026-05-05 +revision: 3 +license: MIT +license_file: LICENSES/galaxy.LICENSE +related_notes: + - "[[galaxy-xsd]]" + - "[[galaxy-collection-tools]]" + - "[[galaxy-apply-rules-dsl]]" + - "[[nextflow-to-galaxy-channel-shape-mapping]]" + - "[[nextflow-operators-to-galaxy-collection-recipes]]" + - "[[galaxy-tool-job-failure-reference]]" + - "[[galaxy-workflow-invocation-failure-reference]]" + - "[[iwc-transformations-survey]]" + - "[[galaxy-discover-datasets]]" +sources: + - "https://github.com/galaxyproject/galaxy/blob/7765fae934fbfdee77e3be5f5b235e43735273ae/lib/galaxy/model/dataset_collections/types/collection_semantics.yml" +companions: + - "galaxy-collection-semantics.yml" +summary: "Vendored formal spec of Galaxy dataset-collection mapping/reduction semantics, with labeled examples and pinned test references." +--- + +> **Vendored from upstream**, pinned at SHA `7765fae`. One file travels with this note: +> +> - `galaxy-collection-semantics.yml` — the structured source. **Agents and casting should consume this.** It carries the `tests:` blocks that pin concrete Galaxy test names; the rendered upstream view drops them. +> +> The Foundry also vendors Galaxy's auto-generated MyST/LaTeX rendering of the same YAML, which +> is what the human view below renders from. It is a site asset: casting does not carry it, so +> do not look for it beside this note in a skill bundle. Sync is manual. +> +> **When to consult:** authoring or reasoning about Molds and patterns that touch `data_collection` inputs, map-over / reduction shape changes, sub-collection mapping, `paired_or_unpaired`, or `sample_sheet`. + +```vendored-myst +file: galaxy-collection-semantics.upstream.myst +source: https://github.com/galaxyproject/galaxy/blob/7765fae934fbfdee77e3be5f5b235e43735273ae/doc/source/dev/collection_semantics.md +sha: 7765fae +``` diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-collection-semantics.yml b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-collection-semantics.yml new file mode 100644 index 0000000..650e39e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-collection-semantics.yml @@ -0,0 +1,1250 @@ +- doc: | + # Collection Semantics + + This document describes the semantics around working with Galaxy dataset collections. + In particular it describes how they operate within Galaxy tools and workflows. + + :::{admonition} You Probably Don't Need to Read This + :class: caution + + Any significantly sophisticated workflow language will have ways to collect data + into arrays or vectors or dictionaries and apply operations across this data (mapping) + or reduce the dimensionality of this data (reductions). Typically, this is explicitly + annotated with map functions or for loops. Galaxy however is designed to be a point + and click interface for connecting steps and running tools. It is important that steps + just connect and just do the most natural thing - and this is what Galaxy does. + This document just provides a mathematical formalism to that "what should just + intuitively work" that can be used to document test cases and help with implementation. + This is reference documentation not user documentation, Galaxy should just work. + ::: + + ## Mapping + + If a tool consumes a simple dataset parameter and produces a simple dataset parameter, + then any collection type may be "mapped over" the data input to that tool. The result of + that is the tool being applied to each element of the collection and "implicit collections" + being created from the outputs that are produced from those operations. Those implicit + collections have the same element identifiers in the same order as the input collection that is + mapped over. Each element of the implicit collections correspond to their own job and + Galaxy very naturally and intuitively parallelizes jobs without extra work from the user + and without any knowledge of the tool. + +- example: + label: BASIC_MAPPING_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_collection" + workflow_runtime: + framework_test: "collection_semantics_cat_0" + workflow_editor: "accepts paired data -> data connection" + +- example: + label: BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" + workflow_runtime: + framework_test: "collection_semantics_cat_1" + workflow_editor: "accepts paired_or_unpaired data -> data connection" + +- example: + label: BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED + assumptions: + - datasets: [d_u] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {unpaired: d_u}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + unpaired: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_u}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_2" + workflow_editor: "accepts paired_or_unpaired data -> data connection" + +- example: + label: BASIC_MAPPING_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [list, {i1: "d_1", ..., in: "d_n"}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + workflow_editor: "accepts collection data -> data connection" + +- doc: | + The above description of mapping over inputs works naturally and as expected for + nested collections. + +- example: + label: NESTED_LIST_MAPPING + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tools.py::TestToolsApi::test_map_over_nested_collections" + workflow_editor: "accepts list:list data -> data connection" + +- example: + label: BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:paired_or_unpaired" + elements: + el1: + type: nested_elements + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_data_with_list_paired_or_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_3" + workflow_editor: "accepts list:paired_or_unpaired data -> data connection" + +- doc: | + + For tools with multiple data inputs, the tool can be executed with individual + datasets for the non-mapped over input and each tool execution will just be executed + with that dataset. The dataset not mapped over serves as the input for each execution. + +- example: + label: BASIC_MAPPING_INCLUDING_SINGLE_DATASET + assumptions: + - datasets: ["d_1,...,d_n", "d_o"] + - tool: + in: {i: dataset, i2: dataset} + out: {o: dataset} + - collections: + C: ["list", {i1: d_1, ..., in: d_n}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + i2: {type: dataset, ref: d_o} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}, i2: {type: dataset, ref: d_o}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}, i2: {type: dataset, ref: d_o}}} + output: o + tests: + tool_runtime: + api_test: "test_tools.py::TestToolsApi::test_map_over_collected_and_individual_datasets" + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_0" + +- doc: | + If a tool consumes two input datasets and produces one output dataset, you can map two + collections with identical structure (same element identifiers in the same order) over + the respective inputs and the result is an implicit collection with the same structure + as the inputs and where each output in the implicit collection corresponds to the tool + being executed with the two inputs corresponding to that position in the input + collections. + + The default behavior here is the collections are linked and the act of mapping over + inputs to the tool are sort of a flat map or a dot product. No extra dimensionality + in the resulting collections. + + From a user perspective this means if you start with a collection and apply a bunch + of map over operations on tools - the results will all continue to match and work together + very naturally - again without extra work by the user and without extra knowledge + by the tool author. + +- example: + label: BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE + assumptions: + - datasets: ["d1_1,...,d1_n", "d2_1,...,d2_n"] + - tool: + in: {i: dataset, i2: dataset} + out: {o: dataset} + - collections: + C1: ["list", {i1: d1_1, ..., in: d1_n}] + C2: ["list", {i1: d2_1, ..., in: d2_n}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C1} + i2: {type: map_over, collection: C2} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_1}, i2: {type: dataset, ref: d2_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_n}, i2: {type: dataset, ref: d2_n}}} + output: o + tests: + tool_runtime: + api_test: test_tools.py::TestToolsApi::test_map_over_two_collections + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_1" + +- doc: "## Reduction" +- doc: | + Not all tool executions result in implicit collections and mapping + over inputs. Tool inputs of ``type`` ``data_collection`` can consume + collections directly and do not necessarily result in mapping over. + + Tools that consume collections and output datasets effectively + reduce the dimension of the Galaxy data structure. When used at runtime + this is often referred to as a "reduction" in the code. + +- example: + label: COLLECTION_INPUT_PAIRED + assumptions: + - datasets: ["d_f", "d_r"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + tool_runtime: + tool: collection_paired_test + workflow_editor: "accepts paired -> paired connection" + +- example: + label: COLLECTION_INPUT_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {el1: d_1, ..., eln: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_0" + workflow_editor: "accepts list -> list connection" + +- example: + label: COLLECTION_INPUT_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + tool_runtime: + tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_0" + workflow_editor: "accepts paired_or_unpaired data -> paired_or_unpaired connection" + +- example: + label: COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + tool_runtime: + tool: collection_list_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_0" + workflow_editor: "accepts list:paired_or_unpaired data -> list:paired_or_unpaired connection" + +- doc: | + For nested collections where each rank is a ``list`` or a ``paired`` collection, + then collection inputs must match every part of the collection type input definition. + +- example: + label: COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects connecting paired -> list" + +- example: + label: COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects connecting list -> paired" + +- example: + label: COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired:paired -> list:paired connection" + +- example: + label: COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired:paired -> list:paired_or_unpaired connection" + +- doc: | + In addition to explicit collection inputs, tool inputs of ``type`` ``data`` + where ``multiple="true"`` can consume lists directly. This is likewise a + "reduction" and does not result in implicit collection creation. + +- example: + label: LIST_REDUCTION + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: dataset_list, refs: ["d_1", "...", "d_n"]} + tests: + tool_runtime: + api_test: "test_tools.py::TestToolsApi::test_reduce_collections" + workflow_runtime: + framework_test: "collection_semantics_multi_data_optional_0" + workflow_editor: "treats multi data input as list input" + +- doc: | + Paired collections cannot be reduced this way. ``paired`` is not meant + to represent a list/array/vector data structure - it is more like a tuple. + +- example: + label: PAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired input on multi-data input" + +- example: + label: PAIRED_OR_UNPAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired_or_unpaired input on multi-data input" + +- doc: "## Sub-collection Mapping" +- doc: | + ![](https://planemo.readthedocs.io/en/master/_images/subcollection_mapping_identifiers.svg) + +- example: + label: MAPPING_LIST_PAIRED_OVER_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + "C\\_PAIRED": [paired, {forward: d_f,reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: list + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_simple_subcollection_mapping" + workflow_editor: "accepts list:paired -> paired connection" + +- doc: | + The natural extension of multiple data input parameters consuming list collections as described + above when discussing reductions is that nested lists of lists (``list:list``) can be mapped + over a multiple data input parameter. Each nested list will be reduced by this operation but the + results will be mapped over. The result will be a list with the same structure as the outer list + of the input collection. + +- example: + label: NESTED_LIST_REDUCTION + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: list} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_1"]}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_n"]}}} + output: o + tests: + workflow_editor: "maps list:list over multi data input" + +- doc: | + Just as a paired collection won't be reduced by a multiple data input, any sort of nested + collection ending in a paired collection cannot be mapped over such an input. So a multiple + data input parameter cannot be mapped over by a list of pairs (``list:paired``) for instance. + +- example: + label: LIST_PAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + is_valid: false + tests: + workflow_editor: "rejects list:paired input on multi-data input" + +- example: + label: LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "dataset"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired_or_unpaired} + is_valid: false + tests: + workflow_editor: "rejects list:paired_or_unpaired input on multi-data input" + +- doc: "## paired_or_unpaired Collections" +- doc: | + The collection type ``paired_or_unpaired`` is meant to serve as a stand-in for + an entity that can be either a single dataset or what is effectively a ``paired`` + dataset collection. These collections either have one element with identifier + ``unpaired`` or two elements with identifiers ``forward`` and ``reverse``. + + Tools can declare a data_collection input with collection type ``paired_or_unpaired`` + and that input will consume either an explicit ``paired_or_unpaired`` collection + normally or can consume a ``paired`` input. + +- example: + label: PAIRED_OR_UNPAIRED_CONSUMES_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired, {forward: d_f, reverse: d_r}] + - "C_AS_MIXED = CollectionInstance" + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: collection, ref: C_AS_MIXED} + tests: + tool_runtime: + tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_1" + workflow_editor: "accepts paired -> paired_or_unpaired connection" + +- doc: | + + The inverse of this doesn't work intentionally. In some ways a ``paired`` collection + acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired + collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` + dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist + in ``paired_or_unpaired`` collection. + +- example: + label: PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects paired_or_unpaired -> paired connection" + +- doc: | + + The same logic holds for mapping, lists of paired datasets (``list:paired``) can be mapped over these + ``paired_or_unpaired`` inputs and mixed lists of pairs (``list:paired_or_unpaired``) cannot + be mapped over a ``paired`` input. Following the same logic, ``list:paired_or_unpaired`` cannot + be mapped over a ``list`` input or multiple data input. + +- example: + label: MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el: {forward: d_f, reverse: d_r}}] + C_AS_MIXED: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_2" + workflow_editor: "accepts list:paired -> paired_or_unpaired connection" + +- example: + label: PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_paired_or_unpaired_not_consumed_by_paired_when_mapping" + workflow_editor: "rejects list:paired_or_unpaired -> paired connection" + +- example: + label: PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false + tests: + workflow_editor: "rejects list:paired_or_unpaired -> list connection" + +- doc: | + This logic extends naturally into higher dimensional collections. A ``list:list:paired`` + can be mapped over either a ``paired_or_unpaired`` input to produce a nested list (``list:list``) + or a ``list:paired_or_unpaired`` input to produce a flat list (``list``). + +- example: + label: MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list:paired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + C_AS_MIXED: ["list:list:paired_or_unpaired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_3" + workflow_editor: "accepts list:list:paired -> paired_or_unpaired connection" + +- doc: | + + In order for ``paired_or_unpaired`` collections to also act as a single dataset, + a flat list can be mapped over a such an input with a special sub collection mapping + type of 'single_datasets'. + +- example: + label: MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_list" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_7" + workflow_editor: "accepts list -> paired_or_unpaired connection" + +- doc: | + This treatment of lists without pairing extends to nested structures naturally. + For instance, a list of list of datasets (``list:list``) can be mapped over a + ``paired_or_unpaired`` input to produce a nested list of lists (``list:list``) + with a structure matching the input. Likewise, the nested list can be mapped over + a ``list:paired_or_unpaired`` input to produce a flat list with the same structure + as the outer list of the input. + +- example: + label: MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_UNPAIRED_j = CollectionInstance for each dataset dj in C" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + workflow_editor: "accepts list:list -> paired_or_unpaired connection" + +- example: + label: MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_LPU_i = inner list of C at position i, treated as list:paired_or_unpaired via single_datasets" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: "list:paired_or_unpaired"} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_1}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_list_paired_or_unpaired_with_list_of_lists" + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_2" + workflow_editor: "accepts list:list -> list:paired_or_unpaired connection" + +- doc: | + Due only to implementation time, the special casing of allowing paired_or_unpaired + act as both datasets and paired collections only works when it is the deepest + collection type. So while list:paired can be consumed by a list:paired_or_unpaired + input, a paired:list cannot be consumed by a paired_or_unpaired:list input though + it should be able to for consistency. We have focused our time on data structures + more likely to be used in actual Galaxy analyses given current and guessed future + usage. + +- doc: "## sample_sheet Collections" +- doc: | + The collection type ``sample_sheet`` attaches typed, columnar metadata to each + element of a dataset collection. For mapping and type matching purposes, + ``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool + inputs, matched against ``list`` collection inputs, and composed with inner types + (``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry + is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list`` + output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata + that plain lists do not. + +- example: + label: SAMPLE_SHEET_MAPPING + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: "d_1", ..., in: "d_n"}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_4" + workflow_editor: "accepts sample_sheet data -> data connection (maps like list)" + +- example: + label: SAMPLE_SHEET_MATCHES_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_1" + workflow_editor: "accepts sample_sheet -> list connection (canMatch)" + +- doc: | + Sub-collection mapping works the same as for ``list`` composites. A + ``sample_sheet:paired`` can be mapped over a ``paired`` collection input, + extracting each inner pair and producing a ``sample_sheet`` implicit output. + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + "C\\_PAIRED": [paired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_5" + workflow_editor: "accepts sample_sheet:paired -> paired connection (maps over like list:paired)" + +- example: + label: SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_0" + workflow_editor: "accepts sample_sheet:paired -> list:paired connection (canMatch)" + +- doc: | + The ``paired_or_unpaired`` integration rules carry over from ``list`` to + ``sample_sheet``. A flat ``sample_sheet`` can be mapped over a + ``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping, + and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as + ``list:paired`` can. + +- example: + label: SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_sample_sheet" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_8" + workflow_editor: "accepts sample_sheet -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + C_AS_MIXED: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_6" + workflow_editor: "accepts sample_sheet:paired -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_1" + workflow_editor: "accepts sample_sheet:paired_or_unpaired -> list:paired_or_unpaired connection (canMatch)" + +- doc: | + The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list`` + input because sample sheets carry all the structural information lists have (plus + metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because + lists lack the ``column_definitions`` and per-element ``columns`` metadata that + sample sheet consumers expect. + +- example: + label: LIST_NOT_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list -> sample_sheet connection (asymmetry)" + +- example: + label: LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list:paired -> sample_sheet:paired connection (asymmetry)" + +- example: + label: SAMPLE_SHEET_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_sample_sheet_0" + workflow_editor: "accepts sample_sheet -> sample_sheet connection" diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md new file mode 100644 index 0000000..853abc2 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-data-flow-draft-contract.md @@ -0,0 +1,131 @@ +--- +type: research +title: "Galaxy data-flow draft contract" +tags: + - target/galaxy +status: draft +created: 2026-05-02 +revised: 2026-09-23 +revision: 4 +related_notes: + - "[[nextflow-to-galaxy-channel-shape-mapping]]" + - "[[nextflow-operators-to-galaxy-collection-recipes]]" + - "[[galaxy-workflow-draft]]" +related_molds: + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[freeform-summary-to-galaxy-data-flow]]" + - "[[workflow-brief-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" + - "[[advance-galaxy-draft-step]]" +sources: + - "https://github.com/galaxyproject/foundry/issues/54" +summary: "Defines the proposed boundary between Galaxy data-flow drafts, gxformat2 templates, and concrete step implementation." +--- + +# Galaxy Data-Flow Draft Contract + +This is an architectural contract, not a schema. Evidence is strongest for Mold and Pipeline boundaries. Proposed fields are speculative until exercised by two or three worked translations. + +## Boundary + +The data-flow draft owns a target-shaped abstract DAG for Galaxy. It should not be valid `gxformat2` and should not resolve exact Tool Shed tools. + +Data-flow draft owns: + +- Galaxy-facing workflow inputs and outputs. +- Abstract nodes, edges, branches, collection mapping, collection reduction, and placeholder transformations. +- Input/output shape decisions such as `File`, `list`, `paired`, `list:paired`, or `list:list`. +- Conceptual Galaxy idioms: map-over, reduction, Apply Rules, collection cleanup, identifier synchronization, tabular bridge, fan-in / combine (concatenate, merge, or union N sources into one — a distinct node, not a side-effect of a reshape). +- Abstract unresolved tool needs with input and output shapes. +- Confidence and rationale on inferred nodes, edges, transforms, and tool needs. + +The Galaxy template owns: + +- A `gxformat2` skeleton. +- Ordered steps, labels, workflow inputs, workflow outputs, and the producer→consumer connection graph. +- Per-step wrapper resolution to the tier its evidence supports — **evidence-gated, not source-gated** (tiers defined in [[galaxy-workflow-draft-format]]): a fully **Resolved** step (concrete `tool_id`, changeset, `state`) when wrapper and parameters are pinned jointly; **Identity-pinned** (`tool_id` only, `_plan_state` kept) when the wrapper is named but not its settings; **Deferred** (`tool_id: TODO`, full `_plan_*`) otherwise. +- Placeholder collection-operation or Apply Rules steps only when the data-flow draft says they are necessary. +- `_plan_*` handoff units (intent, evidence, constraints) for every step left short of Resolved. + +Concrete step implementation owns: + +- For every step the template left short of **Resolved**: the changeset (`tool_version`, `tool_shed_repository` owner/repository metadata), bound `state` parameters, and `input_connections` — resolving any `tool_id: TODO` and confirming or correcting any `tool_id` the template pinned. Steps the template already Resolved pass through — confirm, do not re-derive. +- Concrete built-in collection-operation steps and parameters. +- Validation with `gxwf` and repair after schema/lint failures. + +## Proposed Body-Level Contract + +Do not add these as frontmatter fields yet. + +| Concept | Status | Notes | +|---|---|---| +| `source_summary_ref` | Speculative | Reference to the source summary consumed by the Mold. | +| `workflow_inputs` | Speculative | Abstract Galaxy-facing inputs and collection types. | +| `workflow_outputs` | Speculative | Intended outputs and source-summary provenance. | +| `nodes` | Speculative | Abstract operations, not concrete Galaxy tools. | +| `edges` | Speculative | Data dependencies, shape before/after, and source evidence. | +| `galaxy_idioms` | Speculative | Map-over, reduction, Apply Rules, collection filter, tabular bridge, etc. | +| `unresolved_tool_needs` | Speculative | Per abstract step needs for `discover-shed-tool` or `author-galaxy-tool-wrapper`. | +| `placeholder_transformations` | Speculative | Shape/text/table transforms needed for Galaxy semantics but not concretely implemented. | +| `confidence` | Speculative | Prefer `high`, `medium`, `low` plus rationale. | +| `handoff_notes` | Speculative | Instructions to template, exemplar comparison, and per-step Molds. | +| `open_questions` | Speculative | Semantic/tooling issues carried forward. | + +## Handoff Examples + +Unresolved tool need: + +- [[nextflow-summary-to-galaxy-data-flow]] emits an abstract node such as `trim FASTQ reads`, input `list:paired fastq`, output `list:paired fastq`, tool need `read trimming`, confidence `medium`. +- [[nextflow-summary-to-galaxy-template]] creates a placeholder step with TODOs and collection-shaped connections. +- The harness routes through tool discovery or wrapper authoring. +- [[implement-galaxy-tool-step]] fills exact Galaxy tool metadata, parameters, and connections. + +Collection cleanup after fan-out: + +- Data-flow records a conceptual cleanup transform after a mapped step. +- Template materializes a placeholder collection-operation step. +- Implementation chooses exact built-in tool and parameters. + +Identifier-derived reshaping: + +- Data-flow records desired input shape, output shape, and identifier transformation. +- Template emits an Apply Rules placeholder only if needed. +- Implementation fills concrete rule JSON after exemplar comparison confirms the shape. + +IWC exemplar comparison: + +- Data-flow and template should hand [[compare-against-iwc-exemplar]] the abstract topology, placeholder transformations, unresolved tool needs, and confidence notes. +- Exemplar comparison should flag structural divergence, not resolve tools. + +## Confidence Guidance + +- Attach confidence to the smallest useful unit: node, edge, transformation, or tool need. +- Use qualitative `high`, `medium`, `low` until examples justify a richer schema. +- Require rationale for low confidence. +- Do not reuse source-summary `warnings[]` as data-flow confidence. +- Keep evidence quality distinct from translation confidence. A claim can be corpus-observed but still low-confidence for a specific workflow. + +## Risks + +If the data-flow draft is too broad, it duplicates the template Mold, makes premature tool decisions, and leaks harness routing into Mold output. + +If it is too narrow, the template Mold receives source-summary details without Galaxy-shaped semantics, exemplar comparison has only a surface skeleton to diff, and tool discovery loses input/output shape context. + +## Evidence + +- [[nextflow-to-galaxy-channel-shape-mapping]] and [[nextflow-operators-to-galaxy-collection-recipes]] show why a Galaxy-shaped abstraction is needed between source summary and `gxformat2` template. +- `content/pipelines/nextflow-to-galaxy.md` places data-flow before template, exemplar comparison, and per-step implementation. +- `content/molds/summarize-nextflow/index.md` says source summarization should not produce Galaxy data flow. +- `content/molds/implement-galaxy-tool-step/index.md` owns concrete step implementation. + +## TODOs + +- Decide whether to author `galaxy-data-flow-draft.schema.json` now or wait for worked examples. +- Decide whether confidence should be a single enum or per-axis vector. +- Decide how much ordering guidance belongs in data-flow versus template. +- Decide whether built-in collection operations should be abstract placeholders or concrete template steps. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-comments.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-comments.md new file mode 100644 index 0000000..9256dbf --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-comments.md @@ -0,0 +1,117 @@ +--- +type: research +tags: + - target/galaxy +status: draft +created: 2026-06-12 +revised: 2026-09-23 +revision: 3 +related_notes: + - "[[iwc-comments-survey]]" + - "[[gxformat2-schema]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" +summary: "How to annotate a gxformat2 workflow with editor comments: one titled frame per analysis stage, populate contains_steps, color decorative." +--- + +# Galaxy workflow comments + +Use this note when authoring or translating a Galaxy gxformat2 workflow and +deciding how to annotate it with editor comments — the top-level `comments:` +array the workflow editor renders as on-canvas frames and notes. Comments carry +no execution semantics; they document and visually organize the workflow. This +is presentation guidance, not a `content/patterns/` operation. Corpus evidence +behind every rule here lives in [[iwc-comments-survey]]. + +Comments are optional — roughly one in five IWC workflows use them. Add them when +the workflow has enough steps that a reader benefits from a stage map; skip them +on short linear workflows. + +## 1. Box each analysis stage in a titled frame + +The core move. Once topology is settled, partition the step set into `type: +frame` comments — one frame per logical stage of the analysis — and give each a +`title:` and a `contains_steps:` list of the step labels it groups: + +```yaml +comments: + - type: frame + position: [687.5, 395.8] + size: [498.9, 345.5] + color: green + title: Genome quality evaluation + contains_steps: + - run BUSCO + - summarize BUSCO +``` + +`position` and `size` are editor-canvas geometry (floats); set them so the frame +encloses its steps, or copy them from the nearest IWC exemplar you are adapting. + +## 2. Title by stage, not by tool + +A frame title names the analysis phase, never the tool that implements it — +`Annotation with Braker3`, `Final MF assignment`, `Transcripts assembly`, not +`braker3` or `stringtie`. Three title flavors, all valid: + +- **Stage frames** — one per narrative step of the analysis (the common case). +- **I/O region frames** — `Inputs`, `Data input`, `Visualization` bracket the + workflow's entry and exit regions. +- **Parameter-derivation frames** — box the fiddly utility clusters that compute + a parameter (`Map Strandedness parameter`, `Downsample input BAM to get the + same number of reads`). This is the highest-value annotation: the frame is + often the only place the intent of a non-obvious helper cluster is written + down. Always frame these. + +Titles are human prose — spaces, mixed case, and punctuation are fine. + +## 3. Populate `contains_steps` + +Prefer explicit `contains_steps:` (a list of step labels) over relying on canvas +geometry alone. Both render identically, but a geometry-only frame loses the +machine-readable step→frame mapping — a consumer can only recover membership by +intersecting rectangles. List every step the frame covers. + +## 4. Color is decorative — do not encode meaning + +Pick any editor color: `green`, `yellow`, `turquoise`, `red`, `blue`, `pink`, +`lime`, `orange`, `black`, or `none` (an unfilled outline). Color carries **no +semantic convention** — do not use it to signal input vs output vs compute, and +do not infer role from a frame's color when reading a workflow. When forking a +workflow, keep each frame's color stable so the lineage stays visually +recognizable. + +## 5. Free-floating notes: `markdown` and `text` + +For a labeled region without a box, use a `type: markdown` comment (a `text:` +field holding a markdown body) or `type: text` (a `text:` field plus +`text_size:`). Keep these short — they are mostly one-to-three-word region +headers (`Input files`, `Alpha diversity`). The one place a longer `markdown` +note earns its keep is **workflow-level adaptation guidance** — e.g. "to retarget +this nanopore workflow for Illumina, replace Minimap2 with Bowtie2." Reach for +prose only when a stage title cannot carry the instruction. + +## 6. Comments are reusable scaffold + +Frames travel with workflow forks and templates: when deriving a workflow from an +IWC exemplar, carry its frame set and adjust titles rather than starting bare. +Embedded subworkflows can carry their own `comments:` array, so annotate at the +scope the reader navigates. + +## 7. Schema and validation + +`comments:` is a schema-legal top-level workflow field (and a subworkflow field); +emitted frames pass `gxwf` draft validation. Field shapes: + +| `type:` | Fields | +|---|---| +| `frame` | `position`, `size`, `color`, `title`, `contains_steps` (recommended) | +| `markdown` | `position`, `size`, `color`, `text` | +| `text` | `position`, `size`, `color`, `text`, `text_size` | + +## Cross-references + +- Corpus evidence and distribution: [[iwc-comments-survey]]. +- gxformat2 field schema: [[gxformat2-schema]]. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md new file mode 100644 index 0000000..9fc6e51 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-draft-format.md @@ -0,0 +1,136 @@ +--- +type: research +title: "Galaxy workflow draft format" +tags: + - target/galaxy +status: draft +created: 2026-05-06 +revised: 2026-11-10 +revision: 6 +related_notes: + - "[[gxformat2-schema]]" + - "[[galaxy-data-flow-draft-contract]]" + - "[[discover-shed-tool]]" + - "[[galaxy-workflow-draft]]" + - "[[open-requirements-ledger]]" +related_molds: + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" + - "[[implement-galaxy-tool-step]]" + - "[[advance-galaxy-draft-step]]" + - "[[repair-galaxy-draft-topology]]" +summary: "gxformat2 draft superset: wrapper-tier TODOs (tool_id, state, port names) plus _plan_state / _plan_context / _plan_in / _plan_out per tool step." +--- + +# Galaxy workflow draft format + +The output artifact `galaxy-workflow-draft` produced by the `*-summary-to-galaxy-template` Molds is **gxformat2 with wrapper-tier relaxations and free-text planning fields**, sized to the gap between data-flow design and tool-resolved implementation. + +Topology — workflow inputs and their collection shapes, workflow outputs, the step set, the producer→consumer edge graph, branches, and `when:` guards — is **settled by the template Mold itself**, drawing on the upstream interface and data-flow briefs (see [[galaxy-data-flow-draft-contract]]). The output is concrete gxformat2 with no topology TODOs. Wrapper-tier resolution — which Tool Shed wrapper, what parameters, the wrapper-determined port names that populate `in:` / `out:` / `outputSource` — is **evidence-gated per step** (see Resolution tiers below): a step resolves fully when the evidence pins wrapper and parameters together, pins identity alone when the wrapper is named but its settings are not, and defers entirely when the evidence is weak. + +A step's declared `in:` arity is a topology decision and must be realizable by one wrapper. Combining N sources into one — concatenate, merge, union — is its own node (`gops_concat_1`, `cat1` for two/N datasets; `collapse_dataset` for a collection), never folded into a downstream 1→1 reshape. A step that would declare ≥2 dataset inputs under a reshape intent (awk, text-reformatting, "reshape") is a missing-combine-node smell: split it into an explicit combine step plus single-input reshapes. The relevant concat/merge/union recipes are reachable through the tabular, interval, and collection pattern indices the template already consults. + +## Resolution tiers + +Each tool step is resolved to the tier its evidence supports — **evidence-gated, not source-gated**. The source kind (free-form, nf-core, CWL) shifts how often a step reaches each tier but never caps it; each `*-summary-to-galaxy-template` Mold records its own source tendency. + +- **Resolved** — concrete `tool_id`, `tool_shed_repository` / changeset (or a stable built-in id), and bound `state`; no `_plan_*`. Use when the evidence pins wrapper *and* parameters jointly: a built-in Galaxy tool with brief-determined params (`Filter1`, `Cut1`, a collection operation), or a pattern page / IWC exemplar worked example that supplies `tool_id`, changeset, and parameter values together. `draft-validate` rejects `_plan_*` on a fully-resolved step. +- **Identity-pinned** — concrete `tool_id` with `tool_version: TODO` (changeset deferred); `state` and `tool_shed_repository` absent; `_plan_state` (and any other open `_plan_*`) kept. The `tool_version: TODO` sentinel is load-bearing: it keeps the step drafty for [[draft-next-step]] and not-fully-resolved for [[draft-validate]], so the retained `_plan_state` is legal rather than a `semanticError`. (A step is "fully-resolved" — and so forbidden from carrying `_plan_*` — exactly when it has *no* remaining TODO sentinel in `tool_id`, `tool_version`, or any `in:` / `out:` port; pinning identity without this sentinel would trip that gate.) Use when the evidence confidently names *which* wrapper but not its settings or exact changeset for this context: a portion of an IWC exemplar that [[compare-against-iwc-exemplar]] flags as a high-confidence match, a pattern page's worked example, or a source summary that names a specific `tool_id` with evidence. +- **Deferred** — `tool_id: TODO`, full `_plan_*`. Use when the evidence is weak, multi-candidate, a domain-specific scientific tool with no covering pattern or exemplar, or a corpus gap. + +Pin identity only on strong evidence — a wrapper an exemplar actually uses for the same operation, a worked pattern example, or a tool the source names explicitly — never on plausibility: a wrong pinned `tool_id` lets [[discover-shed-tool]] resolve the wrong tool's changeset instead of searching afresh. Port names follow the wrapper: real on a Resolved step, `TODO_` sentinels until the step resolves. + +The per-step loop does not edit topology — but it may **escalate back to a template-tier Mold** when implementation proves the settled topology can't support a declared step (see "Topology repair escalation" below). Wiring authority stays in the template tier; the loop only gains a path back to it. + +## Relaxations vs. gxformat2 + +For tool steps, when the wrapper has not been picked: + +- `tool_id` and `tool_version` MAY be the literal string `TODO`. Resolution belongs to [[discover-shed-tool]] and the per-step implementation Mold. +- `tool_shed_repository` (the `{ changeset_revision, name, owner, tool_shed }` block) MAY be absent. +- `state` MAY be absent. A draft is authored, so it uses `state` — never the export-side + `tool_state`; see [[gxformat2-schema]] for the distinction. +- Step `out[].id` and `in[]` keys MAY be `TODO_` sentinels (e.g. `TODO_trimmed_paired`, `TODO_input`). Workflow `outputs[].outputSource` MAY reference such a sentinel via `step/TODO_` so the connection graph still resolves syntactically. The connection itself (which step's output feeds which step's input) is topology and MUST be present; only the wrapper-determined port names are deferred. + +Workflow inputs (types, collection shapes, formats, optionality), workflow outputs, step labels, and the producer→consumer edge set MUST be expressed in normal gxformat2 form with concrete values — never `TODO`. The template Mold is the locus where these decisions are made; if the upstream brief leaves a topology choice open, decide it here from source evidence, IWC exemplars, and pattern pages rather than punting downstream. + +## Additions: `_plan_*` planning fields + +Free-text fields per tool step capture the template Mold's intent for the downstream per-step implementation Mold. All optional, but expected on any Identity-pinned or Deferred step (`tool_id` is `TODO`, or `tool_id` is pinned but `state` is absent); a Resolved step carries none. + +- `_plan_state` — parameter binding intent: which knobs matter, value or range, why. Read by the per-step Mold to bind real `state` once a wrapper is selected. +- `_plan_context` — extras the per-step Mold needs to pick a wrapper: source `command:` block, conda packages, Docker/Singularity images, environment variables, preconditions, postconditions, container entrypoints, scratch-disk needs. +- `_plan_in` — wrapper input port mapping intent. The connection graph already says "this source feeds this step"; `_plan_in` records the semantic role of each port and likely wrapper-side names (`single_paired` vs `paired_input` vs `input`) so the per-step Mold can collapse `TODO_*` keys to the real ones. +- `_plan_out` — wrapper output surface intent: which outputs downstream consumers depend on (with the existing edges as evidence) so the per-step Mold can pick a wrapper that exposes them, or insert a follow-up step if it does not. + +The `_plan_*` family is **wrapper-tier**: each entry exists because the wrapper is not yet fully resolved and disappears the moment the step is. Topology decisions do not belong here. + +`_plan_*` fields are **draft-only**. They MUST be removed before the workflow is treated as a runnable gxformat2 document. Any remaining `TODO` / `TODO_*` sentinels MUST also be resolved at the same time. + +## Example (sketch) + +```yaml +class: GalaxyWorkflowDraft +inputs: + reads: + type: collection + collection_type: list:paired + format: fastqsanger.gz +outputs: + trimmed: + outputSource: fastp/TODO_trimmed_paired +steps: + fastp: + tool_id: TODO + label: trim and QC paired reads + in: + TODO_input: reads + out: + - id: TODO_trimmed_paired + - id: TODO_html_report + _plan_state: | + adapter trimming on, quality cutoff ~Q20, min length ~50. + preserve paired-end pairing for downstream alignment. + _plan_context: | + upstream: nf-core FASTP module. + conda: bioconda::fastp=0.23.4 + container: quay.io/biocontainers/fastp:0.23.4--h5f740d0_0 + precondition: paired list collection with sane element identifiers + _plan_in: | + single semantic port `reads`: feeds workflow `reads` (list:paired). + wrapper input port name likely one of `single_paired` | `paired_input` | + `input` depending on which fastp wrapper is picked. + _plan_out: | + need a paired output that preserves list:paired shape (downstream + alignment step consumes it). also expose the HTML report as a + checkpoint output for QC. +``` + +## Why this shape + +- Keeps the artifact recognizably gxformat2 so static tooling (`gxwf validate --no-tool-state`, structural diff against IWC) still applies to topology and port wiring. +- Carves a stable handoff between the template Mold and the per-step implementation Mold without sneaking tool resolution into either side. +- Makes the boundary between topology (settled upstream) and wrapper-tier (deferred) explicit — the `_plan_*` family lives squarely on the deferred side. +- Free-text `_plan_*` is intentional for v1: it lets the templating agent record intent without a contract pretending to be parameterizable yet. Structuring those fields is open work — see each template Mold's `refinement.md`. + +## Topology repair escalation + +The template Mold settles topology, but a settled draft can still be unable to give a correct run. The common case is a **computability** gap: a step whose declared output needs evidence no wired input carries. The connection graph validates — ports connect — yet the step can't be implemented. The template's computability review pass catches most of these and either wires the producer or records the gap in the [[open-requirements-ledger]]; the rest surface in the per-step loop. + +When one surfaces, the per-step loop does not author topology to fix it. It **escalates**: + +- [[implement-galaxy-tool-step]] detects, while implementing, that the declared output can't be computed from the wired inputs, or that the chosen wrapper can't produce it. It appends a blocking entry to the [[open-requirements-ledger]] and falls through rather than fabricating the output. Earlier Molds and the loop raise blocking entries too. +- [[repair-galaxy-draft-topology]] — a template-tier Mold sharing the template's reference set — reads the blocking entries and makes the smallest structural edits that close them. New steps land at draft (wrapper-tier `TODO`) tier, so the existing discover-or-author → implement machinery realizes them. + +This is **repair**, distinct from **settling**: it is narrow (the entries bound it), reactive (an open blocking entry triggers it), and bounded. The loop's convergence gate counts open blocking entries in the ledger; each repair must leave fewer than it found, under a hard cap on escalations. When the cap is reached with entries still open, they are **surrendered** — written into the final draft as labelled gaps — the clean terminal that replaces spin-or-fabricate. So the boundary "topology decisions do not belong in the per-step loop" still holds: the loop never decides topology, it returns the decision to the template tier. + +## Open work + +- Decide whether `_plan_state` should grow structure (parameter-name hints, value ranges, references back to the source summary). +- Decide whether `_plan_context` should be split into typed fields (`source_command`, `conda`, `containers`, `env`, `pre`, `post`). +- Decide structure for `_plan_in` and `_plan_out` once two or three worked drafts exercise them. Candidate `_plan_out`: array of `{ semantic_name, downstream_consumers[], required_for }`. Candidate `_plan_in`: map keyed by the `TODO_*` placeholder key to `{ semantic_name, source_step_output, shape_constraint }`. +- Pick a JSON Schema strategy: extend the gxformat2 schema with `_plan_*` and the relaxed wrapper / port-name rules, or validate the draft via a sibling schema that wraps gxformat2 with the relaxations. +- Specify the strip step that converts a draft into a runnable gxformat2 workflow. It MUST drop all `_plan_*` fields and reject any remaining `TODO` or `TODO_` sentinel in `tool_id`, `tool_version`, `out[].id`, `in[]` keys, or `outputSource`. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md new file mode 100644 index 0000000..3ba5b73 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/galaxy-workflow-testability-design.md @@ -0,0 +1,62 @@ +--- +type: research +tags: + - target/galaxy +status: draft +created: 2026-05-03 +revised: 2026-09-23 +revision: 3 +related_notes: + - "[[iwc-workflow-testability-survey]]" + - "[[iwc-test-data-conventions]]" + - "[[planemo-asserts-idioms]]" + - "[[iwc-shortcuts-anti-patterns]]" + - "[[planemo-workflow-test-architecture]]" + - "[[implement-galaxy-workflow-test]]" + - "[[gxformat2-schema]]" + - "[[gxformat2-workflow-inputs]]" + - "[[galaxy-datatypes-conf]]" +summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." +--- + +# Design a Galaxy workflow that can be tested + +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. + +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. + +## Name the public interface before writing tests + +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. + +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. + +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. + +## Expose a checkpoint that can catch the wrong result + +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. + +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. + +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. + +## Make collection members addressable + +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. + +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. + +## Choose inputs that real fixtures can supply + +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. + +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. + +## Express the choice in gxformat2 + +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. + +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. + +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/open-requirements-ledger.md new file mode 100644 index 0000000..761183e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/open-requirements-ledger.md @@ -0,0 +1,231 @@ +--- +type: research +title: "Open-requirements ledger" +tags: + - target/galaxy +status: draft +created: 2026-06-16 +revised: 2026-11-10 +revision: 7 +related_notes: + - "[[galaxy-workflow-draft-format]]" +related_molds: + - "[[advance-galaxy-draft-step]]" + - "[[repair-galaxy-draft-topology]]" + - "[[implement-galaxy-tool-step]]" +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." +--- + +# Open-requirements ledger + +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. + +## Framing: obligations the pipeline discharges, not questions a human answers + +This is deliberately *not* an "open questions for the user" list. The pipeline is autonomous — no human-in-the-loop gate is assumed. The ledger's consumers are **Molds and the loop's convergence gate**, with human readout a secondary affordance. An entry is closed by a downstream Mold doing work (wiring a producer, picking a wrapper, settling a value), or — when nothing can close it — surrendered: written into the final draft as a known, labelled gap rather than silently dropped or fabricated around. + +The distinction matters because a "questions for a human" framing leaks an operator's personal interaction style into a tool meant to run inside anyone's harness. The ledger must behave identically cast into a fresh harness with no ambient configuration; that holds only because every obligation is **materialized in the artifact**, never carried as operator habit. + +## Entry shape (v1, loose) + +Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries are intentionally low-ceremony for v1 — enough structure to count and close, no contract pretending to be machine-parameterizable yet. A reasonable per-entry shape: + +```yaml +- id: sccmec-evidence-missing + status: open # open | resolved | surrendered + kind: gap # gap | dropped — see below; omit for gap + blocking: true # true only when a structural edit closes it; the convergence gate counts these + raised_by: implement-galaxy-tool-step # Mold that appended it + step: classify_context # draft step the obligation attaches to (if any) + unmet: "SCCmec-region candidate output category" + missing: "no wired input carries SCCmec cassette evidence" + resolved_by: null # Mold that closed it, once closed + supersedes: null # id whose justification this entry refutes, once resolved + note: "" # how it was closed or why surrendered +``` + +`kind` separates the two things an open entry can mean, which v1 conflated. A **`gap`** is +an obligation the pipeline cannot discharge: no producer is discoverable, Galaxy has no +datatype for the format, the source pinned no value. A **`dropped`** entry is source work +the pipeline *declined to carry* — a plane cut from the design, a parameter surface not +exposed, an output the repair removed. Both are honest; they are not the same admission, +and a reader cannot tell them apart from `unmet` / `missing` alone. Omit `kind` for a +plain gap. + +A `dropped` entry carries two more fields: + +```yaml +- id: annotation-planes-not-built + status: open + kind: dropped + raised_by: nextflow-summary-to-galaxy-data-flow + units: "25 of 31 designed nodes (rnaseq_short, gnomon, annot_proc, convert)" + because: "`gxwf tool-search asn1` returns no wrapper reading seq-submit; searched 2026-08-29" + unmet: "the pipeline's actual product — an annotated genome" + missing: "the carried spine masks an assembly; it does not annotate a genome" + note: "" +``` + +`units` names what was dropped in the source's own terms, so the drops can be totalled +without re-reading the design. + +`because` says why, and **must carry its citation** — the source construct that is not +there, the target limitation named specifically enough that someone could check it, the +search actually run. A drop's reason is a claim about the world, and the first real run is +why the burden exists: 25 of 31 planes were cut because "each needs an authored Galaxy +wrapper for an undocumented binary available only inside the pipeline's container," and a +Tool Shed search one phase later found the whole pipeline already wrapped at the exact +pinned version. The claim was not dishonest. It was never checked, and nothing in the +entry showed that it had not been. + +A reason you can cite, cite. A reason you cannot cite is still recorded — write it plainly +and leave it uncited rather than dressing it as evidence — but an uncited drop is not a +finding the run may reason from afterwards. It is a debt, and it should read as one. + +Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. + +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. + +## How to use the ledger + +This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. + +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. + +**Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. + +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. + +**Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. + +**Carry the rest untouched.** Entries you neither raise nor close pass through unchanged, statuses and provenance intact. Never renumber, re-word, or drop another Mold's entry. + +**Never fabricate around an entry.** An open obligation is the honest state. Inventing a connection, a tool id, or a value so the artifact looks complete converts a tracked gap into a silent defect — the specific failure the ledger replaces. + +**Record a drop as a drop.** When a decision removes source work from what the run will +carry — narrowing a design, cutting a plane, dropping an output during repair — append a +`kind: dropped` entry naming the `units` and a cited `because`. A cut recorded only as +prose in a brief is invisible to every later Mold; a cut recorded as a `gap` reads as +something the pipeline could not do rather than something it chose not to do. Recording +the drop is not the same as discharging it, and a well-written entry is not a substitute +for the work — an entry makes a cut *legible*, never *justified*. + +**A refuted justification reopens its decision.** When work you do contradicts the +`because` another entry rests on — discovery finds the wrapper a drop assumed absent, a +later brief supplies the evidence a narrowing assumed missing — set `supersedes` to that +entry's `id` on your own resolved entry, and return the superseded entry to `status: open` +with its `because` struck and the contradiction quoted in its `note`. Do not leave a +refuted rationale standing: the terminal writes these into the final artifact, and a +surrendered gap whose stated reason is known to be false is worse than an unexplained one. +Reopening does not oblige *you* to do the dropped work — it obliges the run to stop +claiming a settled reason it no longer has. + +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. + +**Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. + +## Role in topology repair + +In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): + +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. + +So the ledger is not a convenience here; it is the countable state the termination guard depends on. + +## Escalation budget (loop-level state) + +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: + +```yaml +topology_repair: + escalations: 2 # repairs invoked this run + cap: 5 # hard ceiling; at cap, still-open entries are surrendered + open_history: [4, 3, 2] # open blocking-entry count after each escalation +entries: + - id: sccmec-evidence-missing + status: open + blocking: true + # … per-entry shape above +``` + +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. +- `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. + +This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. + +## Threaded through the whole design tier + +The ledger is not loop-only. Every design-tier Mold in the source→Galaxy pipelines **consumes and re-emits** it — the interface, reference-data, data-flow, IWC-comparison, and template Molds (`*-summary-to-galaxy-interface`, `*-summary-to-galaxy-data-flow`, `*-summary-to-galaxy-reference-data`, `compare-against-iwc-exemplar`, `*-summary-to-galaxy-template`), alongside the change-set Molds on the update path. This is the direct answer to #281: the chain stops re-deriving the same unknowns because each Mold inherits them. + +The template Mold's computability review pass is the notable appender ahead of the loop — `*-summary-to-galaxy-template` re-reads each settled step and, where an output needs evidence no input carries, wires the producer or records the gap as a blocking entry. Source-summary Molds upstream of the design tier keep their own free-text open-questions; the design tier is where those formalize into the ledger. + +## Open work + +- Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. +- Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. +- Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. +- Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. +- Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. +- Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/workflow-brief-design.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/workflow-brief-design.md new file mode 100644 index 0000000..8bc1e6d --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/notes/workflow-brief-design.md @@ -0,0 +1,96 @@ +--- +type: research +title: "Workflow Brief" +tags: + - meta +status: draft +created: 2026-09-17 +revised: 2026-10-19 +revision: 6 +summary: "Markdown Workflow Brief definition and section contract for scope, constraints, agent environment, and explicit blockers." +--- + +# Workflow Brief + +A Workflow Brief is an expert-editable document describing what a workflow should achieve, what it should leave out, and the constraints under which an agent will build and run it. It is an input to design and implementation. The draft workflow captures the proposed steps, wiring, and tool state. + +## Document shape + +The brief is Markdown with a title and two sections: `## Workflow` and `## Agent Environment`. The workflow needs an objective; most details are optional. The agent environment must contain a description, which can explicitly record what still needs to be checked. + +The `workflowBriefSchema` declaration uses the shared engine described in [[markdown-document-contract]] and lives in `packages/gxwf-foundry/src/workflow-brief.ts`. An illustrative document lives in `packages/gxwf-foundry/test/fixtures/workflow-brief/read-alignment.md`. + +| Heading | Required | Content | +| --- | --- | --- | +| `## Workflow` | Yes | Scientific intent and scope. | +| `### Objective` | Yes, under Workflow | Scientific goal and desired outcome. | +| `### Sources` | No | Papers, interviews, repositories, relevant source pins, and evidence availability. | +| `### Scope` | No | Optional `#### Included` and `#### Excluded` describe selected work and boundaries. | +| `### Inputs and outputs` | No | Scientific roles and source-supported formats or organization. | +| `### Requirements and preferences` | No | Expert or source requirements, their strength, rationale, and evidence. | +| `### Acceptance criteria` | No | High-level behaviors that establish success. | +| `### Open questions` | No | Unresolved choices that do not prevent work, each with a proposed answer and its evidence. | +| `### Decisions` | No | Expert decisions and rationale. | +| `### Related artifacts` | No | Links to source evidence and other relevant inputs. | +| `### Blockers` | No, under Workflow | Unresolved issues that prevent design or implementation. Any content blocks work. | +| `## Agent Environment` | Yes | Agent tooling and operating conditions. | +| `### Tooling` | No | Expected Foundry tooling and versions, and what is known about availability. | +| `### Constraints` | No | Agent workspace, network, installation, resource, and access restrictions. | +| `### Containerization` | No | Container preference and Docker, Singularity, or Apptainer availability. | +| `### Blockers` | No, under Agent Environment | Environment issues that prevent work. Any content blocks work. | + +Headings are case-insensitive; section order is flexible and additional sections are allowed. Recognized sections are unique within their parent. If present, ordinary sections need content beyond headings, comments, or separators. Blockers may be empty. Scope's Included and Excluded subsections are optional. + +## Workflow content + +The writing agent must record evidence and expert intent without making Galaxy design choices. It must not assume Galaxy datatypes, dataset collection structures, tool availability, interface labels, or concrete tool versions. Source formats and explicitly named source tools can be cited as evidence; they do not establish a Galaxy mapping or an installed wrapper. An expert may supply specific Galaxy requirements here, and the agent must identify them as expert-provided rather than inferred. + +The agent must not write a step graph, concrete tool state, or final test declarations into the brief. Interface design, data-flow design, tool discovery, and test development belong to the downstream design and implementation Molds. Inputs and outputs stay at scientific roles and source-supported descriptions. Acceptance criteria stay high-level; detailed fixtures and assertions belong to test development. + +Requirements and preferences describe what the expert or source asks for. Agent Environment Constraints describe restrictions on the agent's work. Uncertainty that prevents proceeding belongs in Blockers. Record every other unresolved choice in Open questions with a proposed answer and the evidence for it, such as source behavior, the expert's wording, or the available test data. Propose the answer the evidence best supports. Where expert intent and source behavior conflict, propose the expert's intent and cite the conflict. When the evidence is thin, still propose an answer, mark it low confidence, and say what evidence is missing. The expert accepts a proposal by leaving it unedited and overrides it by editing the answer. + +Write each open question as a bullet with `Proposed:` and `Evidence:` sub-bullets. Add `Confidence: low` and `Missing:` when the evidence is thin: + +```md +- Should runs of one sample be merged before trimming? + - Proposed: yes, merge them as the source does. + - Evidence: the source merges sample-sheet rows sharing a sample ID. The selection does not mention merging. + - Confidence: low + - Missing: whether the expert's samples ever span several runs. +``` + +## Agent environment + +Tooling records whether expected tools such as `gxwf`, `foundry`, and `planemo` are available at the required versions. Distinguish intended requirements from observed availability. Containerization records the preference and whether Docker, Singularity, or Apptainer is actually usable, rather than assuming that an executable name proves a working engine. + +A separate preflight result should record observed versions, container availability, source/reference access, writable paths, timestamps, and diagnostics. A requested installation strategy is not evidence of an installed executable. + +## Validation and blockers + +```sh +foundry validate-workflow-brief workflow-brief.md +foundry check-workflow-brief workflow-brief.md --json +``` + +Validation checks the title, required sections, unique recognized headings, content, and parent-scoped subsections. Example headings in fences, quotes, lists, or comments do not satisfy requirements. Success exits `0`, structural failures exit `3`, and input read failures exit `1`. + +The static readiness check adds a check for `### Blockers` under both Workflow and Agent Environment. A structurally valid document with content in either section exits `4`; a structurally valid document without blockers exits `0`. Omit or leave Blockers empty when there are no blockers. Do not write “none” there: any content, including that word, is blocking. Comments and separators alone are empty. The JSON result includes `valid`, `ready`, `errors`, and blocker bodies with their parent and source line. + +A clear static check only establishes structure and the absence of declared blockers. It does not establish semantic completeness, expert approval, or successful environment preflight. The harness must stop on declared blockers and apply its own review and preflight checks before implementation. + +The exported `WorkflowBrief` and `WorkflowBriefSection` types describe the parsed document. `WorkflowBriefReadinessResult` describes the static check. These checks read the document without changing it. + +## Lifecycle + +1. A source-to-brief producer assembles evidence, proposed scope, and unanswered questions without Galaxy design assumptions. +2. The expert edits the brief and settles blockers needed before the next stage. +3. The harness checks structure, declared blockers, review, and environment readiness. +4. Brief-to-Galaxy consumes the brief and referenced evidence, then runs design, draft, implementation, and test development. + +Brief-to-Galaxy and every design or implementation step must leave the brief unchanged. Record durable workflow knowledge in `open-requirements.ledger.yml`: evidence as discoveries, authorized choices as decisions, unmet needs as obligations, and changes to expert intent as proposed brief-change recommendations. Ordinary phase progress remains harness state. Updating the brief belongs to a separate expert editing step; an implementation agent must not resolve a blocker by rewriting its input. + +A hand-authored brief is a first-class input. Implementation should work in a fresh session using the document and its referenced artifacts. + +## Open design questions + +- Should expert questions live here or link to a separate decision record? diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-collection-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-collection-patterns.md new file mode 100644 index 0000000..b00b1d5 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-collection-patterns.md @@ -0,0 +1,94 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: collection patterns" +aliases: + - "Galaxy collection pattern MOC" + - "collection transformation patterns" + - "IWC collection pattern map" +tags: + - target/galaxy + - topic/galaxy-transform + - topic/collection-transform +status: draft +created: 2026-05-02 +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy collection operation or map-over recipe from the current and required data shapes." +related_notes: + - "[[iwc-transformations-survey]]" + - "[[iwc-conditionals-survey]]" +related_patterns: + - "[[manifest-to-mapped-collection-lifecycle]]" + - "[[cleanup-sync-and-publish-nonempty-results]]" + - "[[reshape-relabel-remap-by-collection-axis]]" + - "[[fan-in-bundle-consume-and-flatten]]" + - "[[collection-cleanup-after-mapover-failure]]" + - "[[collection-unbox-singleton]]" + - "[[sync-collections-by-identifier]]" + - "[[harmonize-by-sortlist-from-identifiers]]" + - "[[regex-relabel-via-tabular]]" + - "[[relabel-via-rules-and-find-replace]]" + - "[[collection-flatten-after-fanout]]" + - "[[collection-build-named-bundle]]" + - "[[collection-swap-nesting-with-apply-rules]]" + - "[[collection-split-identifier-via-rules]]" + - "[[collection-build-list-paired-with-apply-rules]]" + - "[[tabular-to-collection-by-row]]" + - "[[tabular-concatenate-collection-to-table]]" + - "[[tabular-pivot-collection-to-wide]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: collection patterns + +Choose by the collection you have and the shape the next Galaxy step needs. The linked operation pages give tool settings and pitfalls. The recipes cover several operations around map-over, where Galaxy runs a step on each collection element. The [[iwc-transformations-survey]] records the IWC evidence behind these choices. + +## Enter or leave a collection + +- **Table to mapped elements:** [[tabular-to-collection-by-row]] splits a manifest, accession list, or result table into collection elements by row or key. Choose a stable identifier column for later alignment. +- **Individual datasets to a named `list`:** [[collection-build-named-bundle]] assembles separate outputs for publication or a collection-aware consumer. It does not concatenate their file contents. +- **Collection of tables to one long table:** [[tabular-concatenate-collection-to-table]] appends rows. Decide whether to carry each element identifier into every row and whether the inputs have headers. +- **Collection of keyed value tables to one wide table:** [[tabular-pivot-collection-to-wide]] makes elements into columns. Choose the missing-value fill and header behavior for downstream use. Filter empty elements first only when they can occur. +- **Known singleton collection to a dataset:** [[collection-unbox-singleton]] extracts its first element. Use it only when the collection has exactly one meaningful element by construction. + +## Keep mapped siblings aligned + +First identify what changed. Membership, order, and labels are separate decisions. + +- **Empty or failed elements:** [[collection-cleanup-after-mapover-failure]] drops unusable mapped outputs with the matching built-in filter. A replacement dataset preserves the slot when a later step needs the original shape, but it changes the data supplied to that step. +- **Membership:** [[sync-collections-by-identifier]] takes identifiers from the collection that now defines the usable set and filters a sibling to the same names. Filtering preserves the sibling's existing order. +- **Order:** [[harmonize-by-sortlist-from-identifiers]] reorders a sibling by an identifier file when later pairing depends on order. The file must name every element, so this operation does not subset. +- **Labels only:** [[regex-relabel-via-tabular]] derives and applies cleaner element identifiers when the collection structure is already right. [[relabel-via-rules-and-find-replace]] covers relabeling within a structural reshape. + +After dropping elements from one mapped result, use its surviving identifiers to filter siblings before relying on per-element correspondence. Add ordering or relabeling only if the downstream step needs it. + +## Change the collection axes + +- **Nested to flat `list`:** [[collection-flatten-after-fanout]] removes a grouping level after its sample, method, or replicate identity is no longer needed. +- **`list:list` to `list:list` with reversed axes:** [[collection-swap-nesting-with-apply-rules]] changes which level a downstream step maps over. +- **Flat `list` identifiers to `list:list`:** [[collection-split-identifier-via-rules]] derives two list axes from a compound identifier. +- **Identifiers to `list:paired`:** [[collection-build-list-paired-with-apply-rules]] promotes a sample identifier and a forward/reverse role. Use this when pairedness is encoded in identifiers, rather than assuming two sibling collections are aligned. + +## Follow a map-over lifecycle + +- [[manifest-to-mapped-collection-lifecycle]] — start from manifest rows, map work over collection elements, then stabilize labels and output shape. +- [[cleanup-sync-and-publish-nonempty-results]] — clean sparse mapped results, align siblings to the survivors, and run final reports only when useful results remain. +- [[reshape-relabel-remap-by-collection-axis]] — correct the axis a later tool should map over after domain fan-out. Its evidence is narrow, so review the identifier derivation for the specific workflow. +- [[fan-in-bundle-consume-and-flatten]] — bundle parallel outputs for a collection-aware consumer, then flatten or aggregate its results if the next step needs a pooled shape. + +## See also + +- [[iwc-transformations-survey]] — collection-transform survey and evidence trail. +- [[galaxy-tabular-patterns]] — companion MOC for tabular operations. +- [[galaxy-interval-patterns]] — companion MOC for coordinate-feature operations. +- [[galaxy-sequence-patterns]] — companion MOC for sequence-record operations. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md new file mode 100644 index 0000000..2dfe890 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-conditionals-patterns.md @@ -0,0 +1,62 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: conditionals patterns" +aliases: + - "Galaxy conditional pattern MOC" + - "Galaxy when patterns" + - "conditional workflow patterns" +tags: + - target/galaxy + - topic/galaxy-transform +status: draft +created: 2026-05-02 +revised: 2026-10-15 +revision: 4 +summary: "Choose a Galaxy when gate, routed output, fallback, or collection cleanup by what must change downstream." +related_notes: + - "[[iwc-conditionals-survey]]" +related_patterns: + - "[[conditional-run-optional-step]]" + - "[[conditional-route-between-alternative-outputs]]" + - "[[conditional-gate-on-nonempty-result]]" + - "[[conditional-transform-or-pass-through]]" + - "[[collection-cleanup-after-mapover-failure]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: conditionals patterns + +Choose by what should happen when the condition is false: skip a step, select one value for a continuing path, or remove unusable collection members. The linked pages give the construction details. The [[iwc-conditionals-survey]] records the IWC evidence behind them. + +Galaxy steps in the observed patterns receive a boolean through an input named `when` and declare `when: $(inputs.when)`. If later steps require an output from a skipped branch, give them an explicit merge or fallback. A `when` gate alone does not provide that value. + +## Skip work without replacing its output + +- **User choice or another available boolean:** [[conditional-run-optional-step]] gates an optional step or branch whose output need not feed an unconditional downstream step. Gate every step that belongs to the optional branch. +- **Upstream result may be empty:** [[conditional-gate-on-nonempty-result]] derives a boolean from dataset content or collection emptiness, then gates reporting, visualization, or export. The MGnify collection-to-boolean chain is corpus-backed and verified against Galaxy `release_25.1`, but takes four shim steps. Use its known-good shape until a shorter one is verified. + +## Continue with one value + +- **Peer alternatives:** [[conditional-route-between-alternative-outputs]] gates the possible modes and merges their compatible outputs with `pick_value`. If exactly one mode must produce a value, choose merge behavior that detects zero or multiple live outputs rather than silently taking the first. +- **Optional change to an existing value:** [[conditional-transform-or-pass-through]] gates the transform and uses `pick_value` to prefer its output when present, with the original value as the fallback. This preserves one downstream input without duplicating the rest of the workflow. + +## Clean a collection instead + +[[collection-cleanup-after-mapover-failure]] drops or replaces empty or failed elements after map-over. It changes collection membership or contents, while a `when` gate skips a whole step. If dropping elements affects alignment with a sibling collection, see [[galaxy-collection-patterns]] for the next choice. + +`__FILTER_NULL__` can filter null outputs after conditional steps, but the conditionals survey found no IWC use from which to establish a pattern. That absence alone does not make the capability an anti-pattern. + +## See Also + +- [[iwc-conditionals-survey]] — conditionals survey and evidence trail. +- [[galaxy-collection-patterns]] — collection transforms and cleanup after map-over. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-interval-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-interval-patterns.md new file mode 100644 index 0000000..de9ec07 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-interval-patterns.md @@ -0,0 +1,88 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: genomic interval patterns" +aliases: + - "Galaxy interval pattern MOC" + - "genomic interval transformation patterns" + - "IWC interval pattern map" +tags: + - target/galaxy + - topic/galaxy-transform + - topic/interval-transform +status: draft +created: 2026-06-10 +revised: 2026-09-23 +revision: 4 +summary: "Use this MOC to choose corpus-grounded Galaxy genomic interval operations and recipes on coordinate features." +related_notes: + - "[[iwc-interval-operations-survey]]" +related_patterns: + - "[[interval-overlap-filter]]" + - "[[interval-coverage]]" + - "[[interval-merge-overlapping]]" + - "[[interval-window-flank]]" + - "[[interval-consensus-by-multi-intersect]]" + - "[[interval-mask-by-set-algebra]]" + - "[[interval-windowed-coverage]]" + - "[[tabular-synthesize-bed-from-3col]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: genomic interval patterns + +The runtime-facing map for Galaxy **coordinate-feature** choices — operations that understand `chrom/start/end/strand`, as opposed to opaque-column [[galaxy-tabular-patterns]] or container-shaped [[galaxy-collection-patterns]]. Use it before loading raw survey notes; [[iwc-interval-operations-survey]] is the evidence backing, these pages are the actionable references. + +This is the smallest of the three data-shape MOCs by design. Interval algebra is a real but moderate cluster in IWC — concentrated in epigenetics peak-consensus and SARS-CoV-2 masking — and its highest-value units are **recipes**, not single operations. Reach for the recipes first when your need is a multi-step construction. + +## Overlap + +- [[interval-overlap-filter]] — keep/drop/annotate features by overlap with a second set (`bedtools intersect`, or VCF-native `vcfvcfintersect`). + +## Set operations + +- [[interval-merge-overlapping]] — collapse a set's own overlapping features into single spans (`bedtools merge` / `gops_merge`). +- [[interval-mask-by-set-algebra]] — *(recipe)* compute regions from regions: concat → merge → subtract (the `gops_*` set-algebra mask). + +## Windows & coverage + +- [[interval-window-flank]] — extend features into neighborhood windows (`bedtools slop`). +- [[interval-coverage]] — genome-wide depth (bedgraph) or reads-in-given-regions counts (`bedtools genomecov` / `coverage`). + +## Recipes + +- [[interval-consensus-by-multi-intersect]] — reproducible features across replicates: multi-intersect → count-threshold → intersect back. +- [[interval-mask-by-set-algebra]] — compute a mask: union (concat+merge) then difference (subtract). +- [[interval-windowed-coverage]] — quantify signal in fixed windows: slop → merge → coverage. + +## Bridges + +- **interval ↔ collection** — `bedtools` ops map over per-sample collections via `reduce_or_iterate: iterate`; multi-intersect consumes a whole collection at once. See [[galaxy-collection-patterns]]. +- **interval ↔ tabular** — coordinate *construction* ([[tabular-synthesize-bed-from-3col]], and the indel-aware variant-to-BED arithmetic in [[interval-mask-by-set-algebra]]) and coordinate *filtering* (`Filter1 c4 < N` on a bedgraph, [[tabular-filter-by-column-value]]). The line: tabular treats columns as opaque; interval understands coordinates. See [[galaxy-tabular-patterns]]. + +## Gaps (no corpus exemplar, no page) + +Per corpus-first, these have zero IWC uptake and get no pattern page; documented here so the absence is explicit, not an oversight. GTN training-corpus counts below are grounded in [[iwc-interval-operations-survey]] §GTN cross-reference (non-IWC signal): + +- **`closest` / proximity** — the "nearest feature + distance" operation. The **interval-algebra form** (`bedtools closest`, fetch-closest, `windowbed`) is absent from both IWC and GTN. The *task* is absent from IWC entirely; in GTN it appears only as a **domain step** (`deeptools computeMatrix` reference-point mode, TSS-relative signal), which #268 scopes out. This is the operation that motivated the MOC (#268). The natural tool — `bedtools_closestbed` (`-d` for distance) — exists in the same `iuc/bedtools` suite this MOC's other tools come from; it simply has no corpus exemplar, so per corpus-first there is no recurring pattern to author, and an agent that needs nearest-feature-plus-distance reaches for it directly. (`computeMatrix` covers only the narrower ChIP/ATAC TSS-proximity case, not a general substitute.) +- **`complement`** — taught in one GTN tutorial (assembly), zero IWC. +- **coordinate-aware `sort`** — taught across three GTN tutorials; IWC sorts intervals with tabular `sort1` instead. +- **`makewindows`, `map`, `window` (`-w` neighborhood join), `annotate`, `jaccard`** — zero in both corpora. + +These are tracked as IWC-input-blocked candidates (GitHub `requires-iwc-inputs`); a page follows only when an IWC workflow uses the operation. + +## See also + +- [[iwc-interval-operations-survey]] — interval-operation survey and evidence trail. +- [[galaxy-sequence-patterns]] — companion MOC for sequence-record operations; shares the extract/mask-by-region bridge (`getfasta`/`maskfasta` produce sequence from these intervals). +- [[galaxy-tabular-patterns]] — companion MOC for opaque-column tabular operations. +- [[galaxy-collection-patterns]] — companion MOC for collection-container operations. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md new file mode 100644 index 0000000..591b3f5 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/patterns/galaxy-tabular-patterns.md @@ -0,0 +1,82 @@ +--- +type: pattern +pattern_kind: moc +evidence: corpus-observed +title: "Galaxy: tabular patterns" +aliases: + - "Galaxy tabular pattern MOC" + - "tabular transformation patterns" + - "IWC tabular pattern map" +tags: + - target/galaxy + - topic/galaxy-transform + - topic/tabular-transform +status: draft +created: 2026-05-02 +revised: 2026-10-15 +revision: 4 +summary: "Use this MOC to choose corpus-grounded Galaxy tabular transformation patterns." +related_notes: + - "[[iwc-tabular-operations-survey]]" +related_patterns: + - "[[tabular-filter-by-column-value]]" + - "[[tabular-filter-by-regex]]" + - "[[tabular-cut-and-reorder-columns]]" + - "[[tabular-compute-new-column]]" + - "[[tabular-join-on-key]]" + - "[[tabular-group-and-aggregate-with-datamash]]" + - "[[tabular-sql-query]]" + - "[[tabular-prepend-header]]" + - "[[tabular-synthesize-bed-from-3col]]" + - "[[tabular-split-taxonomy-string]]" + - "[[tabular-relabel-by-row-counter]]" + - "[[tabular-to-collection-by-row]]" + - "[[tabular-concatenate-collection-to-table]]" + - "[[tabular-pivot-collection-to-wide]]" +related_molds: + - "[[implement-galaxy-tool-step]]" + - "[[nextflow-summary-to-galaxy-data-flow]]" + - "[[cwl-summary-to-galaxy-data-flow]]" + - "[[nextflow-summary-to-galaxy-template]]" + - "[[cwl-summary-to-galaxy-template]]" + - "[[freeform-summary-to-galaxy-template]]" + - "[[workflow-brief-to-galaxy-template]]" + - "[[compare-against-iwc-exemplar]]" +--- + +# Galaxy: tabular patterns + +Choose by the shape of the input and the result you need. These patterns treat table columns as values, even when a column contains coordinates or sequence text. For coordinate-aware operations use [[galaxy-interval-patterns]], and for FASTA records use [[galaxy-sequence-patterns]]. Open the linked operation page for tool parameters and examples. [[iwc-tabular-operations-survey]] records the IWC evidence behind the choices. + +## Rows and columns in one table + +- [[tabular-filter-by-column-value]] — use `Filter1` for a Python predicate over column positions (`c1`, `c2`, …), such as a status comparison. Set its header handling for the input. +- [[tabular-filter-by-regex]] — use `tp_grep_tool` to keep or drop whole lines by regex, including comment lines. It does not distinguish columns or preserve a header independently of the match. The page covers `Grep1` when header preservation is needed. +- [[tabular-cut-and-reorder-columns]] — use `Cut1` to select existing columns and put them in output order. It does not calculate values. +- [[tabular-compute-new-column]] — use `column_maker` for a row-wise expression that inserts, replaces, or appends a column. + +## Combine or summarize tables + +- [[tabular-join-on-key]] — align two tables by key with `tp_easyjoin_tool`. For many same-shaped key/value files, the page covers `tp_multijoin_tool`. Choose the missing-value fill to match downstream meaning. +- [[tabular-group-and-aggregate-with-datamash]] — reduce rows by one or more keys with `datamash_ops`, or reduce the whole file without keys. Check whether input order already groups the keys. +- [[tabular-sql-query]] — use `query_tabular` when the operation needs SQL semantics such as a window function, anti-join, named multi-table join, or a combined projection, computation, and filter. Use the simpler operations above for a single predicate, cut, or computed column. + +## Text-processing recipes + +- [[tabular-prepend-header]] — add a fixed first line with awk, accounting for any existing header. +- [[tabular-synthesize-bed-from-3col]] — turn chromosome, start, and end columns into six-column BED, including the coordinate conversion and BED datatype. +- [[tabular-split-taxonomy-string]] — expand a semicolon-delimited lineage into rank columns, accounting for missing ranks. +- [[tabular-relabel-by-row-counter]] — generate labels from row order when that order is the intended identity source. + +## Between tables and collections + +- [[tabular-to-collection-by-row]] — split a manifest or table by row/key into collection elements so the next tool can run over them. Choose a stable, unique identifier column. +- [[tabular-concatenate-collection-to-table]] — stack rows from a collection of tables into one table. Decide whether to retain element identifiers as row provenance and keep only one header. +- [[tabular-pivot-collection-to-wide]] — align a collection of two-column `(id, value)` tables by id into a wide table, with one value column per element. Choose what missing cells mean. For two ordinary tables, use [[tabular-join-on-key]] instead. + +## See also + +- [[iwc-tabular-operations-survey]] — tabular-operation survey and evidence trail. +- [[galaxy-sequence-patterns]] — companion MOC for sequence-record operations; FASTA↔tabular is the dominant sequence seam. +- [[galaxy-interval-patterns]] — companion MOC for coordinate-feature operations. +- [[galaxy-collection-patterns]] — companion MOC for collection operations. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json new file mode 100644 index 0000000..dd3a94a --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/schemas/galaxy-workflow-draft.schema.json @@ -0,0 +1,2970 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "WorkflowStepSchema": { + "type": "object", + "required": [], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "top", + "left" + ], + "properties": { + "top": { + "type": "number" + }, + "left": { + "type": "number" + } + }, + "additionalProperties": false + } + ] + }, + "tool_id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "tool_shed_repository": { + "$ref": "#/$defs/Shared18" + }, + "tool_version": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "errors": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "in": { + "$ref": "#/$defs/Shared19" + }, + "out": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/Shared4" + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$ref": "#/$defs/Shared4" + } + }, + { + "type": "null" + } + ] + }, + "state": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "tool_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "post_job_actions": { + "anyOf": [ + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "title": "unknown" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "enum": [ + "tool", + "subworkflow", + "pause", + "pick_value" + ] + } + ] + }, + "run": { + "$ref": "#/$defs/Shared0" + }, + "runtime_inputs": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "when": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + "Shared0": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "class": { + "enum": [ + "GalaxyWorkflow" + ], + "type": "string" + }, + "comments": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Shared2" + }, + "type": "array" + }, + { + "additionalProperties": { + "$ref": "#/$defs/Shared2" + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "creator": { + "$ref": "#/$defs/Shared6" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "inputs": { + "$ref": "#/$defs/Shared1" + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "license": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputs": { + "$ref": "#/$defs/Shared17" + }, + "release": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "report": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "markdown": { + "type": "string" + } + }, + "required": [ + "markdown" + ], + "type": "object" + } + ] + }, + "steps": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WorkflowStepSchema" + }, + "type": "array" + }, + { + "additionalProperties": { + "$ref": "#/$defs/WorkflowStepSchema" + }, + "properties": {}, + "required": [], + "type": "object" + } + ] + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "inputs", + "outputs", + "class", + "steps" + ], + "type": "object" + }, + { + "type": "string" + }, + { + "additionalProperties": { + "title": "unknown" + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "Shared1": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/Shared12" + }, + { + "$ref": "#/$defs/Shared5" + }, + { + "$ref": "#/$defs/Shared11" + }, + { + "$ref": "#/$defs/Shared13" + }, + { + "$ref": "#/$defs/Shared8" + }, + { + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" + } + ] + }, + "type": "array" + }, + { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/$defs/Shared12" + }, + { + "$ref": "#/$defs/Shared5" + }, + { + "$ref": "#/$defs/Shared11" + }, + { + "$ref": "#/$defs/Shared13" + }, + { + "$ref": "#/$defs/Shared8" + }, + { + "$ref": "#/$defs/Shared14" + }, + { + "$ref": "#/$defs/Shared15" + }, + { + "$ref": "#/$defs/Shared9" + }, + { + "type": "string" + } + ] + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "additionalProperties": { + "title": "unknown" + }, + "properties": {}, + "required": [], + "type": "object" + } + ] + }, + "Shared2": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "bold": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "italic": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "position": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "text_size": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "type": { + "enum": [ + "text" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "position": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "text": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "enum": [ + "markdown" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "contains_comments": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "type": "array" + } + ] + }, + "contains_steps": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "type": "array" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "position": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "title": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "enum": [ + "frame" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "color": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "line": { + "anyOf": [ + { + "items": { + "items": { + "type": "number" + }, + "type": "array" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "size": { + "anyOf": [ + { + "items": { + "type": "number" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "thickness": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "number" + } + ] + }, + "type": { + "enum": [ + "freehand" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + } + ] + }, + "Shared3": { + "additionalProperties": false, + "properties": { + "_plan_context": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "_plan_in": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "_plan_out": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "_plan_state": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "errors": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "in": { + "$ref": "#/$defs/Shared19" + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "out": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Shared4" + }, + "type": "array" + }, + { + "additionalProperties": { + "$ref": "#/$defs/Shared4" + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "post_job_actions": { + "anyOf": [ + { + "additionalProperties": { + "title": "unknown" + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "run": { + "$ref": "#/$defs/Shared0" + }, + "runtime_inputs": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "state": { + "anyOf": [ + { + "additionalProperties": { + "title": "unknown" + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "tool_id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "tool_shed_repository": { + "$ref": "#/$defs/Shared18" + }, + "tool_state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": { + "title": "unknown" + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "tool_version": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "enum": [ + "tool", + "subworkflow", + "pause", + "pick_value" + ], + "type": "string" + } + ] + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "when": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "required": [], + "type": "object" + }, + "Shared4": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "add_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "change_datatype": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "delete_intermediate_datasets": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "hide": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "remove_tags": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "rename": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "set_columns": { + "anyOf": [ + { + "additionalProperties": { + "title": "unknown" + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "required": [], + "type": "object" + }, + { + "type": "string" + } + ] + }, + "Shared5": { + "additionalProperties": false, + "properties": { + "collection_type": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "column_definitions": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "default_value": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "name": { + "type": "string" + }, + "optional": { + "type": "boolean" + }, + "restrictions": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "type": "array" + } + ] + }, + "suggestions": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "type": "array" + } + ] + }, + "type": { + "enum": [ + "string", + "int", + "float", + "boolean", + "element_identifier" + ], + "type": "string" + }, + "validators": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "title": "unknown" + }, + "type": "array" + } + ] + } + }, + "required": [ + "name", + "type", + "optional" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "fields": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "name": { + "type": "string" + }, + "type": { + "anyOf": [ + { + "enum": [ + "File", + "null", + "boolean", + "int", + "float", + "string" + ], + "type": "string" + }, + { + "items": { + "enum": [ + "File", + "null", + "boolean", + "int", + "float", + "string" + ], + "type": "string" + }, + "type": "array" + } + ] + } + }, + "required": [ + "name", + "type" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "collection" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared6": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "enum": [ + "Person" + ], + "type": "string" + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "familyName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "givenName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificPrefix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "honorificSuffix": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "jobTitle": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "class" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "address": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "alternateName": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "enum": [ + "Organization" + ], + "type": "string" + }, + "email": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "faxNumber": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "identifier": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "image": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "name": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "telephone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "class" + ], + "type": "object" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Shared7": { + "additionalProperties": false, + "properties": { + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "outputSource": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "type": { + "anyOf": [ + { + "type": "null" + }, + { + "enum": [ + "null", + "boolean", + "int", + "long", + "float", + "double", + "string", + "integer", + "text", + "File", + "data", + "collection", + "color", + "directory_uri" + ], + "type": "string" + } + ] + } + }, + "required": [], + "type": "object" + }, + "Shared8": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "restrictOnConnections": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "restrictions": { + "$ref": "#/$defs/Shared20" + }, + "suggestions": { + "$ref": "#/$defs/Shared20" + }, + "type": { + "enum": [ + "text", + "string" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared9": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "directory_uri" + ], + "type": "string" + }, + "validators": { + "$ref": "#/$defs/Shared16" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared10": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "source": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + } + }, + "required": [], + "type": "object" + }, + "Shared11": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "max": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "min": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "integer", + "int" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared12": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "format": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "anyOf": [ + { + "enum": [ + "data", + "File" + ], + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [], + "type": "object" + }, + "Shared13": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "max": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "min": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "float" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared14": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "boolean" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared15": { + "additionalProperties": false, + "properties": { + "default": { + "title": "unknown" + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "optional": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "position": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "left": { + "type": "number" + }, + "top": { + "type": "number" + } + }, + "required": [ + "top", + "left" + ], + "type": "object" + } + ] + }, + "type": { + "enum": [ + "color" + ], + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "Shared16": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "expression": { + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "negate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + } + ] + }, + "type": { + "enum": [ + "regex" + ], + "type": "string" + } + }, + "required": [ + "type", + "expression" + ], + "type": "object" + }, + "type": "array" + } + ] + }, + "Shared17": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Shared7" + }, + "type": "array" + }, + { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/$defs/Shared7" + }, + { + "type": "string" + } + ] + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "additionalProperties": { + "title": "unknown" + }, + "properties": {}, + "required": [], + "type": "object" + } + ] + }, + "Shared18": { + "anyOf": [ + { + "type": "null" + }, + { + "additionalProperties": false, + "properties": { + "changeset_revision": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "tool_shed": { + "type": "string" + } + }, + "required": [ + "name", + "changeset_revision", + "owner", + "tool_shed" + ], + "type": "object" + } + ] + }, + "Shared19": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Shared10" + }, + "type": "array" + }, + { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/$defs/Shared10" + }, + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "properties": {}, + "required": [], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "Shared20": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "type": "object" + } + ] + }, + "type": "array" + } + ] + } + }, + "type": "object", + "required": [ + "inputs", + "outputs", + "class", + "steps" + ], + "properties": { + "id": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "label": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "doc": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "inputs": { + "$ref": "#/$defs/Shared1" + }, + "outputs": { + "$ref": "#/$defs/Shared17" + }, + "uuid": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "class": { + "type": "string", + "enum": [ + "GalaxyWorkflowDraft" + ] + }, + "steps": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/Shared3" + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$ref": "#/$defs/Shared3" + } + } + ] + }, + "report": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "required": [ + "markdown" + ], + "properties": { + "markdown": { + "type": "string" + } + }, + "additionalProperties": false + } + ] + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "comments": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/Shared2" + } + }, + { + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": { + "$ref": "#/$defs/Shared2" + } + }, + { + "type": "null" + } + ] + }, + "creator": { + "$ref": "#/$defs/Shared6" + }, + "license": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + }, + "release": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/schemas/workflow-brief-schema.schema.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/schemas/workflow-brief-schema.schema.json new file mode 100644 index 0000000..92c431e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-template/references/schemas/workflow-brief-schema.schema.json @@ -0,0 +1,103 @@ +{ + "name": "Workflow Brief", + "title": "required", + "sections": [ + { + "heading": "Workflow", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Objective", + "min": 1, + "max": 1, + "content": true + }, + { + "heading": "Sources", + "max": 1, + "content": true + }, + { + "heading": "Scope", + "max": 1, + "content": true, + "sections": [ + { + "heading": "Included", + "max": 1, + "content": true + }, + { + "heading": "Excluded", + "max": 1, + "content": true + } + ] + }, + { + "heading": "Inputs and outputs", + "max": 1, + "content": true + }, + { + "heading": "Requirements and preferences", + "max": 1, + "content": true + }, + { + "heading": "Acceptance criteria", + "max": 1, + "content": true + }, + { + "heading": "Open questions", + "max": 1, + "content": true + }, + { + "heading": "Decisions", + "max": 1, + "content": true + }, + { + "heading": "Related artifacts", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + }, + { + "heading": "Agent Environment", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Tooling", + "max": 1, + "content": true + }, + { + "heading": "Constraints", + "max": 1, + "content": true + }, + { + "heading": "Containerization", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/SKILL.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/SKILL.md new file mode 100644 index 0000000..7c6aefd --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/SKILL.md @@ -0,0 +1,94 @@ +--- +name: workflow-brief-to-galaxy-test-plan +description: "Synthesize a Galaxy workflow test plan from a reviewed Workflow Brief, its Galaxy design briefs, and available source evidence." +--- + +# workflow-brief-to-galaxy-test-plan + +Follow the procedure below and use the artifact/reference sections as the runtime contract. + +## When To Use + +- Synthesize a Galaxy workflow test plan from a reviewed Workflow Brief, its Galaxy design briefs, and available source evidence. + +## Inputs + +- Read artifact `workflow-brief`. Schema: workflow-brief-schema. Produced by `freeform-summary-to-workflow-brief`, `nextflow-summary-to-workflow-brief`. Expert-reviewed implementation contract whose acceptance criteria and scope govern test intent. +- Read artifact `freeform-summary`. Optional; absence is allowed and must be reported honestly. Produced by `interview-to-freeform-summary`, `summarize-paper`. Optional narrative source evidence carrying methods, sample data, parameters, and expected outputs within the brief's selected scope. +- Read artifact `summary-nextflow`. Optional; absence is allowed and must be reported honestly. Schema: summary-nextflow. Produced by `summarize-nextflow`. Optional structured Nextflow source evidence carrying parameters, tests, fixtures, and expected outputs within the brief's selected scope. +- Read artifact `workflow-brief-galaxy-interface`. Produced by `workflow-brief-to-galaxy-interface`. Galaxy interface brief from workflow-brief-to-galaxy-interface that pins workflow inputs, outputs, and labels the plan binds assertions to. +- Read artifact `workflow-brief-galaxy-data-flow`. Produced by `workflow-brief-to-galaxy-data-flow`. Galaxy data-flow brief from workflow-brief-to-galaxy-data-flow that pins abstract operations and collection choices the plan must respect. +- Read artifact `iwc-comparison-notes`. Produced by `compare-against-iwc-exemplar`. Structural diff guidance from compare-against-iwc-exemplar; steers the plan toward IWC-aligned testable checkpoints and fixture shapes. +- Read artifact `iwc-exemplar-gxformat2`. Produced by `compare-against-iwc-exemplar`. Cleaned gxformat2 view of the nearest IWC exemplar from compare-against-iwc-exemplar; pattern-match its labels, collection identifiers, and tested checkpoints. Absent when no nearest exemplar was found. +- Read artifact `open-requirements-ledger`. Optional; absence is allowed and must be reported honestly. Produced by `advance-galaxy-draft-step`, `apply-galaxy-workflow-changeset`, `compare-against-iwc-exemplar`, `cwl-summary-to-galaxy-data-flow`, `cwl-summary-to-galaxy-interface`, `cwl-summary-to-galaxy-template`, `freeform-summary-to-galaxy-data-flow`, `freeform-summary-to-galaxy-interface`, `freeform-summary-to-galaxy-template`, `implement-galaxy-tool-step`, `interview-to-galaxy-workflow-changeset`, `mature-galaxy-workflow-for-iwc`, `nextflow-summary-to-galaxy-data-flow`, `nextflow-summary-to-galaxy-interface`, `nextflow-summary-to-galaxy-reference-data`, `nextflow-summary-to-galaxy-template`, `repair-galaxy-draft-topology`, `workflow-brief-to-galaxy-data-flow`, `workflow-brief-to-galaxy-interface`, `workflow-brief-to-galaxy-template`. Carried workflow-run knowledge open-requirements-ledger when the run supplied one; read-only here, used to follow the current answers to the Workflow Brief's open questions. + +## Outputs + +- Write artifact `galaxy-test-plan` as `galaxy-test-plan.yml`. Format: `yaml`. Schema: galaxy-workflow-test-plan. Reviewable Galaxy workflow test plan (see galaxy-workflow-test-plan): synthesized test cases with job inputs, expected outputs, assertion intent, fixture provenance, label assumptions, unresolved mappings, and omissions. + +## Required Tools + +- None declared. Procedure should not assume external CLIs are present. + +## Load Upfront + +- `references/notes/workflow-brief-design.md`: Research note copied verbatim into the bundle. Keep reviewed acceptance criteria authoritative while treating source summaries as supporting evidence. +- `references/schemas/galaxy-workflow-test-plan.schema.json`: Schema file copied verbatim into the bundle. Output contract: the emitted plan conforms to galaxy-workflow-test-plan. Cast bundles the JSON Schema so the skill carries its output shape; validate with `foundry validate-galaxy-workflow-test-plan`. +- `references/schemas/workflow-brief-schema.schema.json`: Schema file copied verbatim into the bundle. Carry the required implementation contract and its acceptance criteria into test planning. + +## Load On Demand + +- `references/notes/galaxy-workflow-testability-design.md`: Research note copied verbatim into the bundle. Choose which workflow outputs and promoted checkpoints make meaningful assertions when no upstream test evidence exists. Use when: deciding which outputs to assert, which checkpoints to promote, and which labels the plan should assume. +- `references/notes/iwc-shortcuts-anti-patterns.md`: Research note copied verbatim into the bundle. Distinguish accepted IWC-style test shortcuts from assertion smells while synthesizing assertions from intent. Use when: considering existence-only, size-only, image-dimension, or tolerant output checks, or recording an omission. +- `references/notes/iwc-test-data-conventions.md`: Research note copied verbatim into the bundle. Describe job-input fixtures, remote-URL-first locations, hashes, and collection input shapes the plan should record as fixture provenance. Use when: recording job inputs or fixture provenance, or deciding whether a fixture belongs in test-data, Zenodo, ENA/SRA, or CVMFS. +- `references/notes/open-requirements-ledger.md`: Research note copied verbatim into the bundle. Follow the current open-question decisions, including answers the interface Mold made for questions the brief left without a proposal, instead of answering them again. Use when: the run supplied an open-requirements ledger whose decisions carry `open_question`. +- `references/notes/planemo-asserts-idioms.md`: Research note copied verbatim into the bundle. Pick the assertion family and tolerance that fit each synthesized expected output by output type. Use when: turning an expected output into assertion intent and a tolerance. +- `references/notes/planemo-workflow-test-architecture.md`: Research note copied verbatim into the bundle. Keep the plan addressable by stable labels and artifacts Planemo can connect back to invocations, jobs, and outputs. Use when: recording the labels and checkpoints the downstream test must address. +- `references/schemas/tests-format.schema.json`: Schema file copied verbatim into the bundle. Use the Galaxy workflow tests schema as the assertion-family vocabulary referenced by the plan's assertion intent. Use when: naming an assertion family or compare operator for a synthesized expected output. + +## Validation + +- Before starting, check that the installed validator carries the galaxy-workflow-test-plan schema this bundle was cast with: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95` with no artifact. Exit 4, or an `unknown option '--schema-sha256'` error from a validator older than this check, means the validator and this bundle are out of step; stop and report that instead of producing an artifact the validator would misjudge. +- Validate `galaxy-test-plan.yml` before returning it: run `foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml` from `@galaxy-foundry/gxwf-foundry`. If the command is not on PATH, run `npx --package @galaxy-foundry/gxwf-foundry foundry validate-galaxy-workflow-test-plan --schema-sha256 03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95 galaxy-test-plan.yml`. This checks artifact `galaxy-test-plan` against the galaxy-workflow-test-plan schema. + +## Procedure + +Synthesize a Galaxy workflow test plan from the reviewed Workflow Brief and the Galaxy interface and data-flow briefs. Consult retained source evidence when available. The output is a reviewable handoff conforming to galaxy-workflow-test-plan, not a concrete `tests-format` file: it records test cases, job inputs, expected outputs, assertion intent, fixture provenance, label assumptions, unresolved mappings, and intentional omissions so implement-galaxy-workflow-test can author the final Galaxy test artifact. + +When the run supplies an open-requirements ledger, follow its current open-question decisions, such as a chosen test dataset, rather than answering those questions again. The ledger is read-only here. A decision that looks wrong for testing goes into the plan's `warnings[]` for the expert, not into a new answer. + +### Synthesized, not translated + +This is the structural difference from nextflow-test-to-galaxy-test-plan and cwl-test-to-galaxy-test-plan. Those skills **translate** existing upstream test evidence into a Galaxy plan. This skill **synthesizes** the common brief-driven plan from reviewed acceptance criteria and the testable structure the interface/data-flow briefs and nearest IWC exemplar imply. When retained evidence contains concrete fixtures, snapshots, parameters, or expected results, cite and use them within the selected scope; their presence does not let evidence override the brief. + +Set the plan's `source.derived_from` to `intent` (not `test-evidence`) and each synthesized assertion's `evidence` to `intent`; most synthesized assertions are `confidence: medium` or `low`. Where the brief or retained evidence carries a concrete expected value (a known count, a named output token, a published figure), record it as `expected_value` and raise the confidence. + +### Labels and fixtures are assumed, not bound + +This skill consumes the template-era briefs, not the concrete workflow draft or the resolved test-data refs — those exist in the harness run-state by the time the plan is authored, but they are reconciled downstream rather than here. So: + +- Bind assertions to the labels the interface brief pins, and set `label_status` to `assumed` (or `unresolved` when even the brief is silent) and the plan's `workflow.label_source` to `interface-brief`. implement-galaxy-workflow-test reconciles these against the real draft labels and the workflow-label cross-check. +- Record fixtures with `storage: unresolved` and `location: null` when the brief or evidence names test data only by description; capture what is known (a dataset name, an accession, a Zenodo DOI, a rough size) in `fixture.provenance` so test-data resolution and the implement skill can act on it. Use iwc-test-data-conventions for the storage classes and the remote-URL-first convention. + +When a binding or fixture cannot be settled from the briefs, add an `unresolved[]` entry (with `blocking: true` when the final test cannot be authored without it) rather than inventing a label or a URL. + +### Choosing assertions from intent + +For each expected output the briefs expose, pick the assertion family by output type per planemo-asserts-idioms and a defensible tolerance. When the only exposed output is weakly assertable (a stochastic plot, an opaque binary), consult galaxy-workflow-testability-design: prefer recording assertion intent against a stronger promoted checkpoint, and when you settle for a weak check, record the weaker outputs in `omissions[]` with a rationale rather than asserting around them. Check each shortcut against iwc-shortcuts-anti-patterns so an existence-only or size-only intent is a deliberate choice. + +Keep the plan addressable by stable labels and artifacts (planemo-workflow-test-architecture) so the downstream test, run, and debug skills can connect assertions back to invocations and outputs. + +## Feedback Mode + +- Feedback mode is off unless the caller explicitly enables `--feedback` or supplies a feedback-ledger path. +- When enabled, read `_feedback.md` before doing the work and use its registered `foundry-feedback.ledger.yml` protocol. +- Preserve harness-owned run and phase state. Append only concrete observations about a canonical Foundry source asset or a related project that this run showed to be at fault; do not put ordinary workflow requirements in this ledger. +- Before reporting completion, make one explicit pass over the work you just did. Do not ask yourself whether anything was unclear — recall what happened: where you guessed at something the instructions should have settled, needed information this bundle does not carry, hit an instruction that contradicted another or contradicted the artifacts in front of you, used a packaged reference that did not cover your case, or did something the procedure never describes. +- Append an entry for each such event that clears the protocol's bar. If none do, append nothing and report `no feedback` explicitly. Silence and a clean pass are not the same thing, and nothing downstream can tell them apart unless you say which one it was. +- Pass the same ledger path to any subagent used for this work, and merge updates serially so one writer cannot overwrite another. + +## Runtime Notes + +- Do not read Foundry source files at runtime; use only files packaged in this skill bundle and user-supplied artifacts. +- Preserve declared artifact filenames unless the user or harness supplies explicit paths. +- Carry unresolved assumptions into the output artifact instead of silently inventing missing source evidence. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_feedback.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_feedback.md new file mode 100644 index 0000000..61a4bd4 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_feedback.md @@ -0,0 +1,143 @@ +# Foundry feedback ledger + +The `foundry-feedback-ledger` is an opt-in runtime artifact for observations about the Foundry +assets used during a real cast-skill or pipeline run. It records actionable gaps, defects, +friction, and wishes about Molds and their authored references. It does not record unmet +requirements in the workflow being built; those belong in the open-requirements ledger. + +Every entry is destined for `galaxyproject/foundry`. When a run shows that a related project is +at fault — Galaxy, a tool wrapper, the IWC corpus, a harness CLI — that observation still belongs +here, recorded with the analysis that points at the upstream. The Foundry is where the run +context lives, so it is where the issue is tracked; a maintainer forwards from there. Never tell +a user to take an observation to another tracker instead. + +The registered filename is `foundry-feedback.ledger.yml`. Feedback-aware callers pass the same +file through every skill invocation in a run. + +## Ledger shape + +```yaml +run: + pipeline: nextflow-to-galaxy + run_slug: sarek-galaxy + status: running + phases: + - { n: 1, kind: mold, skill: summarize-nextflow, status: done, feedback_checked: true } + - { n: 2, kind: mold, skill: nextflow-summary-to-galaxy-interface, status: running } +entries: + - id: channel-shape-note-silent-on-optional-outputs + raised_by: nextflow-summary-to-galaxy-interface + observed_in: + mold: nextflow-summary-to-galaxy-interface + path: content/molds/nextflow-summary-to-galaxy-interface/index.md + revision: 6 + content_hash: 82d8... + foundry_head: d9e0b09 + subject: + kind: research + label: nextflow-to-galaxy-channel-shape-mapping + locator: content/research/nextflow-to-galaxy-channel-shape-mapping/index.md + content_hash: a18c... + kind: gap + severity: major + what: "The note does not cover optional outputs." + expected: "State how an absent optional channel maps to the Galaxy interface." + evidence: "Interface design stopped at the optional output decision." + status: open + issue: null +``` + +Entry `kind` is `gap`, `defect`, `friction`, or `wish`. Severity is `blocker`, `major`, or +`minor`. Status is `open`, `filed`, `duplicate`, or `wontfix`. + +## Run lifecycle + +For a pipeline run, the harness creates the file before phase 1 with `run.status: running`, an +empty `entries` list, and the complete top-level phase roster from its assembly record. All +phases begin `pending`. The harness changes a phase to `running` immediately before invocation +and to `done` on success. A phase that terminates the run and the run itself become `failed`; a +graceful user stop makes the run `cancelled`. A hard interruption naturally leaves the run +`running`, which is distinguishable from success without a recovery write. + +A phase also carries `feedback_checked`. The harness sets it true only when that phase reported a +feedback outcome — entry ids it appended, or an explicit `no feedback`. It is absent or false +otherwise, including for a phase that simply never said. This is what separates a run that looked +for friction and found none from a run that never looked. + +Loop phases remain one row and increment `iterations` after each successful iteration. Branch +phases remain one row and record the chosen path in `selected`. After every intended phase is +done, the harness sets `run.status: complete`. + +For a standalone skill invocation, the first feedback-aware skill creates a ledger with +`pipeline: null`, the caller's run slug when one exists, and one phase row for itself. It owns +that row's lifecycle as well as any entry it appends. + +No ledger means feedback mode was off. An empty `entries` list is evidence of a clean run only +when `run.status` is `complete` **and** every phase is `feedback_checked`. A complete run with +unchecked phases is complete but unreviewed; an empty incomplete ledger is neither. + +## When to append + +Appending is a deliberate pass, not a background reflex. Before a skill reports completion it +recalls what actually happened during its work — where it guessed at something the instructions +should have settled, needed information its bundle does not carry, hit an instruction that +contradicted another or contradicted the artifacts in front of it, used a reference that did not +cover the case, or did something its procedure never describes. Those are recallable events. "Was +anything unclear?" is not a useful question to ask at that moment and reliably answers itself no. + +The pass ends in one of two outcomes, and the outcome is always stated: entries appended, or +`no feedback`. A skill that says nothing has not reported a clean pass. + +Two entry shapes are admissible. Both require that `what` names a concrete problem, `expected` +states a proposed correction, and the entry is not already present in this ledger for the same +subject and correction. + +**A Foundry source asset.** The observation concerns Foundry-authored content, implementation, or +a registry that could be changed directly, and `subject.locator` canonically identifies it. This +is the ordinary case and the one the clustering and hash checks are built for. + +**A related project.** The run showed that a project the Foundry drives or cites is at fault — +Galaxy, a Tool Shed wrapper, the IWC corpus, `gxwf`, planemo. Use `subject.kind: related-project`, +name the project and the specific component in `label`, and put its public identity in +`locator` (a repository URL, a tool id, an IWC workflow id). There is no `content_hash`. State +in `what` and `evidence` what the run actually observed, and in `expected` what the upstream +would have to do. The entry is still filed against the Foundry. + +Do not use feedback entries for ordinary workflow requirements, tool-selection uncertainty, or +an unsupported preference with nothing concrete to change. + +## Canonical identity and provenance + +For a Foundry subject, use a repository-relative authored path as `subject.locator`; for a +package-export schema, use `package://#`. Generated skills and harnesses are +never subjects; identify the Mold, Pipeline, caster, assembler, or registry that must change. + +Initial subject kinds are `mold`, `pipeline`, `pattern`, `source-pattern`, `research`, `schema`, +`prompt`, `cli-tool`, `cli-command`, `meta`, `implementation`, and `registry`, plus +`related-project` for the upstream case above. `label` is for display only; the locator is the +identity and clustering key. + +A `related-project` subject is the one kind whose locator is not a Foundry path: it is public +upstream identity, carries no content hash, and is never checked against Foundry main. + +For the running Mold, copy `mold.name`, `mold.path`, `mold.revision`, and `mold.content_hash` +from `_provenance.json` into `observed_in`. Record its `mold.commit` value as `foundry_head`, but +never use that commit as exact content identity: it can be null or predate the bytes in the cast. +For a packaged reference, copy its `refs[].src` and `refs[].src_hash` into the subject locator and +content hash. When the subject is the Mold itself, use its authored path and +`observed_in.content_hash`. + +## Safe updates + +Read the complete ledger immediately before changing it. Preserve run state and all entries you +do not own. Append a complete entry in one rewrite; do not leave a partial YAML document. The +harness is the sole writer of pipeline lifecycle fields, while each skill writes only entries. + +Keep evidence concise and safe to upstream. Never copy credentials, tokens, private repository +URLs, proprietary source text, patient or participant data, or user-identifying filesystem +paths into the ledger. Replace a run-directory prefix with `/` and summarize sensitive +evidence instead of quoting it. + +The reporting skill may set an open entry to `filed`, `duplicate`, or `wontfix` and record the +resolved issue URL after the corresponding disposition is confirmed. It never deletes an entry; +the ledger remains the run audit trail. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_provenance.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_provenance.json new file mode 100644 index 0000000..65c7135 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_provenance.json @@ -0,0 +1,263 @@ +{ + "provenance_schema_version": 4, + "cast_target": "claude", + "mold": { + "name": "workflow-brief-to-galaxy-test-plan", + "path": "content/molds/workflow-brief-to-galaxy-test-plan/index.md", + "revision": 4, + "content_hash": "df02aa501cbc18fdf5c36ddc3f2ca87e17f4db51c054a52cbbef6eb0a66bd4e3", + "commit": "94ce4ea20107680b467d03e8889b63689774e325" + }, + "cast_at": "2026-11-10T10:05:13.204Z", + "refs": [ + { + "kind": "research", + "mode": "verbatim", + "ref": "[[galaxy-workflow-testability-design]]", + "src": "content/research/galaxy-workflow-testability-design/index.md", + "dst": "references/notes/galaxy-workflow-testability-design.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Choose which workflow outputs and promoted checkpoints make meaningful assertions when no upstream test evidence exists.", + "trigger": "When deciding which outputs to assert, which checkpoints to promote, and which labels the plan should assume.", + "src_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "dst_hash": "c7dc5421bdd35af8000e61e618c813e162889416a0f322f8d0cb24a66ccad153", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[iwc-shortcuts-anti-patterns]]", + "src": "content/research/iwc-shortcuts-anti-patterns/index.md", + "dst": "references/notes/iwc-shortcuts-anti-patterns.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Distinguish accepted IWC-style test shortcuts from assertion smells while synthesizing assertions from intent.", + "trigger": "When considering existence-only, size-only, image-dimension, or tolerant output checks, or recording an omission.", + "src_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "dst_hash": "b3f691a59751114ab67588832ba978052ab2be2ab5ef775737177ad45d7bdc45", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[iwc-test-data-conventions]]", + "src": "content/research/iwc-test-data-conventions/index.md", + "dst": "references/notes/iwc-test-data-conventions.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Describe job-input fixtures, remote-URL-first locations, hashes, and collection input shapes the plan should record as fixture provenance.", + "trigger": "When recording job inputs or fixture provenance, or deciding whether a fixture belongs in test-data, Zenodo, ENA/SRA, or CVMFS.", + "src_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "dst_hash": "b7ef14132a0b40f626fff98fde44234ce5c848b6a3c255f4748b803c9a44b074", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[open-requirements-ledger]]", + "src": "content/research/open-requirements-ledger/index.md", + "dst": "references/notes/open-requirements-ledger.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "hypothesis", + "purpose": "Follow the current open-question decisions, including answers the interface Mold made for questions the brief left without a proposal, instead of answering them again.", + "trigger": "When the run supplied an open-requirements ledger whose decisions carry `open_question`.", + "verification": "Run a brief whose open question about test data has no proposal; confirm the test plan follows the interface Mold's recorded answer.", + "src_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "dst_hash": "d3d36d558ae5a58de46a0eb869a5f5966be6b02fdb569476e4f77ce1cc1ddf7d", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[planemo-asserts-idioms]]", + "src": "content/research/planemo-asserts-idioms/index.md", + "dst": "references/notes/planemo-asserts-idioms.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Pick the assertion family and tolerance that fit each synthesized expected output by output type.", + "trigger": "When turning an expected output into assertion intent and a tolerance.", + "src_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "dst_hash": "9a6363854dcbeb01e1e590dc5bc0f5d7d0ec9675c4828ad2a8be8fa8022a61df", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[planemo-workflow-test-architecture]]", + "src": "content/research/planemo-workflow-test-architecture/index.md", + "dst": "references/notes/planemo-workflow-test-architecture.md", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Keep the plan addressable by stable labels and artifacts Planemo can connect back to invocations, jobs, and outputs.", + "trigger": "When recording the labels and checkpoints the downstream test must address.", + "src_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "dst_hash": "3b688f5a39951ad6f937335bf1217731aefc59fef7472fcbe9f85002019a2caf", + "source": "deterministic" + }, + { + "kind": "research", + "mode": "verbatim", + "ref": "[[workflow-brief-design]]", + "src": "content/research/workflow-brief-design/index.md", + "dst": "references/notes/workflow-brief-design.md", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Keep reviewed acceptance criteria authoritative while treating source summaries as supporting evidence.", + "verification": "Confirm synthesized tests do not assert excluded analyses or silently alter expert acceptance criteria.", + "src_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "dst_hash": "88237d668f962ca4d81a802ab4e975034eb40442a87f1fc77a27bd5688878b35", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[galaxy-workflow-test-plan]]", + "src": "package://@galaxy-foundry/gxwf-foundry#galaxyWorkflowTestPlanSchema", + "dst": "references/schemas/galaxy-workflow-test-plan.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "hypothesis", + "purpose": "Output contract: the emitted plan conforms to [[galaxy-workflow-test-plan]]. Cast bundles the JSON Schema so the skill carries its output shape; validate with `foundry validate-galaxy-workflow-test-plan`.", + "verification": "Cast the skill, run on a representative reviewed brief with source evidence, confirm the emitted YAML validates and downstream [[implement-galaxy-workflow-test]] consumes it without re-deriving acceptance criteria.", + "license": "MIT", + "src_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "dst_hash": "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "source": "deterministic" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[tests-format]]", + "src": "package://@galaxy-foundry/gxwf-foundry#testsFormatSchema", + "dst": "references/schemas/tests-format.schema.json", + "used_at": "runtime", + "load": "on-demand", + "evidence": "corpus-observed", + "purpose": "Use the Galaxy workflow tests schema as the assertion-family vocabulary referenced by the plan's assertion intent.", + "trigger": "When naming an assertion family or compare operator for a synthesized expected output.", + "license": "MIT", + "license_file": "LICENSES/galaxy-tool-util-ts.LICENSE", + "src_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "dst_hash": "f86dd6a8111919264030609bab6ab970b015f6f66d324b5e2f721d4f62ab3cbc", + "source": "deterministic", + "license_file_hash": "383450c494c01c466bcf22df102540a39ac89da50563fd7be686ba06772f96b3" + }, + { + "kind": "schema", + "mode": "verbatim", + "ref": "[[workflow-brief-schema]]", + "src": "package://@galaxy-foundry/gxwf-foundry#workflowBriefSchema", + "dst": "references/schemas/workflow-brief-schema.schema.json", + "used_at": "runtime", + "load": "upfront", + "evidence": "cast-validated", + "purpose": "Carry the required implementation contract and its acceptance criteria into test planning.", + "src_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "dst_hash": "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "source": "deterministic" + } + ], + "artifacts": { + "produces": [ + { + "id": "galaxy-test-plan", + "kind": "yaml", + "default_filename": "galaxy-test-plan.yml", + "schema": "[[galaxy-workflow-test-plan]]", + "description": "Reviewable Galaxy workflow test plan (see [[galaxy-workflow-test-plan]]): synthesized test cases with job inputs, expected outputs, assertion intent, fixture provenance, label assumptions, unresolved mappings, and omissions." + } + ], + "consumes": [ + { + "id": "workflow-brief", + "description": "Expert-reviewed implementation contract whose acceptance criteria and scope govern test intent.", + "inherited_schema": "[[workflow-brief-schema]]", + "producers": [ + "freeform-summary-to-workflow-brief", + "nextflow-summary-to-workflow-brief" + ] + }, + { + "id": "freeform-summary", + "description": "Optional narrative source evidence carrying methods, sample data, parameters, and expected outputs within the brief's selected scope.", + "optional": true, + "producers": [ + "interview-to-freeform-summary", + "summarize-paper" + ] + }, + { + "id": "summary-nextflow", + "description": "Optional structured Nextflow source evidence carrying parameters, tests, fixtures, and expected outputs within the brief's selected scope.", + "optional": true, + "inherited_schema": "[[summary-nextflow]]", + "producers": [ + "summarize-nextflow" + ] + }, + { + "id": "workflow-brief-galaxy-interface", + "description": "Galaxy interface brief from [[workflow-brief-to-galaxy-interface]] that pins workflow inputs, outputs, and labels the plan binds assertions to.", + "producers": [ + "workflow-brief-to-galaxy-interface" + ] + }, + { + "id": "workflow-brief-galaxy-data-flow", + "description": "Galaxy data-flow brief from [[workflow-brief-to-galaxy-data-flow]] that pins abstract operations and collection choices the plan must respect.", + "producers": [ + "workflow-brief-to-galaxy-data-flow" + ] + }, + { + "id": "iwc-comparison-notes", + "description": "Structural diff guidance from [[compare-against-iwc-exemplar]]; steers the plan toward IWC-aligned testable checkpoints and fixture shapes.", + "producers": [ + "compare-against-iwc-exemplar" + ] + }, + { + "id": "iwc-exemplar-gxformat2", + "description": "Cleaned gxformat2 view of the nearest IWC exemplar from [[compare-against-iwc-exemplar]]; pattern-match its labels, collection identifiers, and tested checkpoints. Absent when no nearest exemplar was found.", + "producers": [ + "compare-against-iwc-exemplar" + ] + }, + { + "id": "open-requirements-ledger", + "description": "Carried workflow-run knowledge [[open-requirements-ledger]] when the run supplied one; read-only here, used to follow the current answers to the Workflow Brief's open questions.", + "optional": true, + "producers": [ + "advance-galaxy-draft-step", + "apply-galaxy-workflow-changeset", + "compare-against-iwc-exemplar", + "cwl-summary-to-galaxy-data-flow", + "cwl-summary-to-galaxy-interface", + "cwl-summary-to-galaxy-template", + "freeform-summary-to-galaxy-data-flow", + "freeform-summary-to-galaxy-interface", + "freeform-summary-to-galaxy-template", + "implement-galaxy-tool-step", + "interview-to-galaxy-workflow-changeset", + "mature-galaxy-workflow-for-iwc", + "nextflow-summary-to-galaxy-data-flow", + "nextflow-summary-to-galaxy-interface", + "nextflow-summary-to-galaxy-reference-data", + "nextflow-summary-to-galaxy-template", + "repair-galaxy-draft-topology", + "workflow-brief-to-galaxy-data-flow", + "workflow-brief-to-galaxy-interface", + "workflow-brief-to-galaxy-template" + ] + } + ] + } +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_verify.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_verify.json new file mode 100644 index 0000000..d6f953f --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/_verify.json @@ -0,0 +1,45 @@ +{ + "verify_schema_version": 1, + "entries": [ + { + "artifact_id": "summary-nextflow", + "direction": "input", + "kind": "json", + "default_filename": "summary-nextflow.json", + "schema": "[[summary-nextflow]]", + "validator_bin": "foundry", + "args": [ + "validate-summary-nextflow", + "{artifact_path}" + ] + }, + { + "artifact_id": "workflow-brief", + "direction": "input", + "kind": "markdown", + "default_filename": "workflow-brief.md", + "schema": "[[workflow-brief-schema]]", + "validator_bin": "foundry", + "args": [ + "validate-workflow-brief", + "--schema-sha256", + "8e65acb789057c59f2fe3a2e7920682ad272a82e0caa92de69b62cc56d5036e0", + "{artifact_path}" + ] + }, + { + "artifact_id": "galaxy-test-plan", + "direction": "output", + "kind": "yaml", + "default_filename": "galaxy-test-plan.yml", + "schema": "[[galaxy-workflow-test-plan]]", + "validator_bin": "foundry", + "args": [ + "validate-galaxy-workflow-test-plan", + "--schema-sha256", + "03fc1d58499c4bd5f3fa9556d771a8f24bcc8a387f120bd6eb28fe5f103ffc95", + "{artifact_path}" + ] + } + ] +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md new file mode 100644 index 0000000..3ba5b73 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/galaxy-workflow-testability-design.md @@ -0,0 +1,62 @@ +--- +type: research +tags: + - target/galaxy +status: draft +created: 2026-05-03 +revised: 2026-09-23 +revision: 3 +related_notes: + - "[[iwc-workflow-testability-survey]]" + - "[[iwc-test-data-conventions]]" + - "[[planemo-asserts-idioms]]" + - "[[iwc-shortcuts-anti-patterns]]" + - "[[planemo-workflow-test-architecture]]" + - "[[implement-galaxy-workflow-test]]" + - "[[gxformat2-schema]]" + - "[[gxformat2-workflow-inputs]]" + - "[[galaxy-datatypes-conf]]" +summary: "Design guidance for Galaxy workflow inputs, outputs, and checkpoints that make IWC-style workflow tests possible." +--- + +# Design a Galaxy workflow that can be tested + +Choose the workflow's public inputs and outputs while designing the analysis, before writing its `-tests.yml`. A workflow test supplies `job:` values by input name and checks `outputs:` by output name. It can only check results the workflow exposes. A good interface gives the test a reproducible input, an addressable result, and at least one assertion that would catch a meaningful wrong result. + +This note concerns workflow design. [[iwc-test-data-conventions]] covers the test file's input fixtures, [[planemo-asserts-idioms]] covers assertion syntax, and [[iwc-shortcuts-anti-patterns]] explains when a smoke check is enough. [[iwc-workflow-testability-survey]] holds the broader corpus evidence. + +## Name the public interface before writing tests + +Give each input and promoted output a stable, descriptive public name. In an exported gxformat2 workflow this is commonly the entry's `id`. The structural [[gxformat2-schema]] also permits `label`. Check the **effective name in the workflow being tested** instead of assuming a step label, generated dataset name, or array position will become the test key. In the pinned [Scanpy workflow](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy.ga), for example, the test's `Initial Anndata General Info` output key addresses that named workflow output, not its producing tool step. + +Treat a public input or output rename as a test-interface change. Update the sibling test and any saved job mapping when it changes. Avoid leaving a useful result behind an anonymous or generated name. A workflow test may still find such an output, but the binding is harder to review and preserve. + +Before committing a test, compare every `job:` and `outputs:` key with the actual workflow interface. [[validate-tests]] can check those bindings statically. A passing schema check alone does not establish that the keys exist or that the chosen assertions are meaningful. + +## Expose a checkpoint that can catch the wrong result + +Start with the failure the test should detect. If a final image, report, or binary artifact varies across runs, its file size or dimensions may only prove that something was produced. Look for a stable result already made along the same path: a count table, summary line, representative sequence, structured property, or other value tied to the analysis. Promote it as a workflow output when it gives the test a stronger check. Keep the final artifact as an output when users need it, and let its smoke check serve that limited purpose. + +The [IWC Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) illustrates the combination. It checks plot dimensions and sizes, but it also checks AnnData structure, a stable summary, and a ranked-gene line exposed as separate outputs. Those checks cover different failure modes. A structural HDF5 key alone does not prove the cluster calculation is right, just as an image width does not prove the plotted values are right. + +Promote selectively. Add a checkpoint as a public output when it serves a clear user or test purpose and an assertion can verify a property that matters. Exposing every intermediate dataset can obscure the workflow interface without improving confidence. If no stable content is available, record the limit in the test plan and use an honest smoke check rather than claiming it validates the scientific result. + +## Make collection members addressable + +A collection output needs predictable element identifiers at every level the test will inspect. Preserve sample or domain identifiers through mapping and collection reshaping when possible. When a tool generates identifiers, inspect an actual run and document the derivation before writing element assertions. Do not infer them from input order or a display name that may change. + +The [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml) addresses samples in `paired_output`, then their `forward` and `reverse` members with nested `element_tests:`. The outer sample identifiers are unusual strings, which is fine when they are stable. Collection type and element count can check shape. Assertions on selected members must check their content or a meaningful property if a scientifically wrong member could otherwise pass. See [[planemo-asserts-idioms]] for the output-test shape and [[galaxy-collection-semantics]] for Galaxy collection types. + +## Choose inputs that real fixtures can supply + +An input's public name and type determine the test's `job:` entry. Check that each required input has a small, reproducible fixture or a documented way to construct one. A `list:paired` input needs the corresponding nested collection of files. A typed `int`, `boolean`, or `string` input can be set directly in the job. A reference input may need a remote file, a local test fixture, or an instance data-table value according to how the workflow actually consumes it. Choose the representation by the runtime contract and test environment, not by a blanket preference for remote files or CVMFS. + +The [IWC CellPlex test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml) combines nested FASTQ collections, reference files, a sample/CMO collection, and a numeric parameter. Its `job:` structure follows the workflow's declared input shapes. When translating another pipeline, resolve sample grouping, collection identifiers, datatype, and reference availability before treating a fixture as ready. [[iwc-test-data-conventions]] gives the supported YAML shapes and provenance checks. + +## Express the choice in gxformat2 + +Top-level gxformat2 `inputs:` and `outputs:` form the workflow interface. A public `outputs:` entry points to the producing step output through `outputSource`. In converted IWC workflows, an entry such as `id: paired_output` with `outputSource: some_step/paired_output` makes the collection addressable as `paired_output` in a test. An optional `doc` can explain an otherwise terse name. The exact accepted fields are in [[gxformat2-schema]]. + +The step's `out:` entry has a different job. Its post-job actions can rename a dataset, change a datatype, or manage history presentation. They do not replace a stable public output name. Apply `change_datatype` only when the produced data truly has that Galaxy datatype, using [[galaxy-datatypes-conf]] to choose the extension. Do not add a post-job action merely to make a test pass. A promoted checkpoint needs a valid source connection, the right datatype, and a name the test can address. + +After authoring, check the actual imported or converted workflow interface, validate the test bindings with [[validate-tests]], and run the test. Static validation will catch some structural and naming mistakes. An execution against the fixture shows whether the assertion passes. Review the failure it is meant to catch to judge whether a wrong result could also pass. [[planemo-workflow-test-architecture]] explains the runtime evidence and failure surfaces. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md new file mode 100644 index 0000000..8dc1aa5 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/iwc-shortcuts-anti-patterns.md @@ -0,0 +1,52 @@ +--- +type: research +tags: + - target/galaxy +status: draft +created: 2026-04-30 +revised: 2026-09-23 +revision: 3 +related_notes: + - "[[galaxy-workflow-testability-design]]" + - "[[iwc-test-data-conventions]]" + - "[[planemo-asserts-idioms]]" + - "[[implement-galaxy-workflow-test]]" + - "[[tests-format]]" + - "[[iwc-conditionals-survey]]" + - "[[iwc-map-over-lifecycle-survey]]" + - "[[iwc-tabular-operations-survey]]" + - "[[iwc-transformations-survey]]" +summary: "Judgments for when weak Galaxy workflow-test assertions are useful smoke checks and when they conceal untested results." +--- + +# IWC test-suite shortcuts and anti-patterns + +A workflow test should fail when a meaningful result is wrong. Start by naming that failure, then choose an assertion that would catch it. A size, shape, or existence check can establish that an output was produced, but it cannot establish that the analysis is correct. Keep such a **smoke check** when the result is genuinely variable or opaque, and add a stable content check or workflow checkpoint whenever one is available. Using a weak check merely because another IWC test uses it is an anti-pattern. + +These authoring judgments guide [[implement-galaxy-workflow-test]]. [[planemo-asserts-idioms]] covers assertion forms, [[iwc-test-data-conventions]] covers fixtures, and [[galaxy-workflow-testability-design]] covers which intermediate results to expose as workflow outputs. + +## Choose by the failure the test must catch + +| Output | Useful check | Anti-pattern | +| --- | --- | --- | +| Small, byte-stable text or data | Compare with an expected `file:`. A `checksum:` can cover a large byte-stable output. | Replacing an exact comparison with size alone when exact content is stable. | +| Variable JSON or HDF5 | Assert a stable property, group, key, or count while leaving variable values unconstrained. | Checking only for `{` or file existence when a stable domain property is available. | +| Stochastic or opaque binary | Compare size with a known expected artifact, then check a stable report or summary if the workflow can expose one. | Setting a tolerance so wide that an empty or substantially wrong result passes. | +| Plot or image | Check relevant image properties and assert on the data or summary behind the plot when possible. | Treating dimensions and file size as proof of the plotted result. | +| Output with mutable headers or provenance | Assert stable records or fields, or compare an expected file with a measured difference allowance. | Assuming `lines_diff:` ignores only headers: it permits differences anywhere in the file. | + +An output assertion can serve as a smoke test even when it is too weak to verify the scientific claim by itself. Record that limit in the test plan, and choose a stronger workflow-level output when the final artifact cannot expose the needed property. This is why the [Scanpy test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks plot dimensions **and** separate AnnData or table outputs. Conversely, the [HyPhy core test](https://github.com/galaxyproject/iwc/blob/main/workflows/comparative_genomics/hyphy/hyphy-core-tests.yml) checks some JSON collection elements only for `{`. That establishes a limited smoke test. It is not a model for validating stable JSON properties that a new workflow could check. + +## Size and difference tolerances need a reason + +`compare: sim_size` compares an output against a referenced expected artifact (`file:` or `location:`). `delta:` is an absolute byte allowance, and `delta_frac:` is fractional. The [RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size bands for variable outputs. Its choice shows one way to smoke-test that workflow, not a reusable tolerance for another dataset. Measure the result across representative runs and ask whether the allowed range still rejects the failure you named. If the lower bound reaches zero, a size comparison alone cannot catch an empty output. + +For `compare: diff`, `lines_diff:` allows a **count** of differing lines. A changed line counts as two, one removal and one addition, under the [[tests-format|vendored test-format schema]]. The allowance does not identify which lines may differ. Copying `lines_diff: 6` from a VCF example because six header lines seem mutable could also permit changed variant records. Inspect the actual diff and add a targeted content assertion if those records matter. + +## Keep format and contribution rules separate + +Planemo [supports exact files, output checksums, and tolerant assertions](https://planemo.readthedocs.io/en/stable/test_format.html#outputs). A checksum is a good fit for a large result known to be byte-stable. Input `hashes:` verify fixture integrity. Choose an output assertion from the result's stability and the failure the test should detect. + +Do not ban a failure-path test, a hand-written test, or a non-Zenodo fixture on the basis of corpus habit. Planemo's [[tests-format|test format]] includes `expect_failure:`, and the [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits tests written by hand as well as tests generated from an invocation. A negative case is useful when the workflow itself promises to reject an invalid input. A tool-specific validation failure usually belongs in that tool's tests. Choose a small, reproducible fixture and verify a remote file's digest when one is recorded. [[iwc-test-data-conventions]] gives the input shapes and provenance details. + +Workflow tests address outputs by label and collection members by identifier. A label or identifier mismatch is a broken test, not an assertion-strength tradeoff. Check those names against the workflow before running Planemo, then run the test against a real invocation. A green smoke check says only that the property it asserted held. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md new file mode 100644 index 0000000..842a914 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/iwc-test-data-conventions.md @@ -0,0 +1,144 @@ +--- +type: research +tags: + - target/galaxy +status: draft +created: 2026-04-30 +revised: 2026-10-21 +revision: 5 +related_notes: + - "[[galaxy-workflow-testability-design]]" + - "[[iwc-shortcuts-anti-patterns]]" + - "[[planemo-asserts-idioms]]" + - "[[implement-galaxy-workflow-test]]" + - "[[tests-format]]" + - "[[iwc-tabular-operations-survey]]" +summary: "IWC and Planemo workflow-test fixture shapes, file provenance, collection identifiers, input hashes, and built-in index values." +--- + +# IWC test data conventions + +An IWC workflow test pairs each workflow with a sibling `-tests.yml`. Each YAML test case supplies `job:` inputs under the workflow's input labels and `outputs:` assertions under its output labels. The optional `test-data/` directory holds local fixtures. These are the [IWC contribution layout](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test-format](https://planemo.readthedocs.io/en/latest/test_format.html) contracts. For decisions about the workflow interface before writing fixtures, see [[galaxy-workflow-testability-design]]. For the exact supported YAML vocabulary and static validation, see [[tests-format]]. + +Examples below come from the cited IWC files or Planemo documentation. Corpus observations describe the workflows sampled for this note, not a rule that every IWC workflow follows. + +## Fixture locations and provenance + +A `class: File` input can use `path:` for a local fixture or `location:` for a URI. Planemo resolves a relative `path:` from the workflow and test directory. IWC's contribution guide recommends small inputs and suggests publishing a toy dataset to Zenodo for a permanent URL. The repository also permits an optional local `test-data/` directory. It gives no universal size cutoff or mandatory host. [Planemo input format](https://planemo.readthedocs.io/en/latest/test_format.html#job), [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#find-input-datasets). + +| Source | Useful when | Example | +| --- | --- | --- | +| Local `path:` | A small fixture belongs beside the test and can be reviewed in the repository | `path: test-data/input_accession_single_end.txt` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | +| Remote `location:` | A published dataset or accession is the source of the test input | Zenodo paired reads in the [short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) | +| Local expected `file:` | An output is small and deterministic enough for comparison | `file: test-data/SRR044777_head.fastq` in the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow) | + +Large or variable outputs are better checked with targeted assertions than committed as full expected files, as the [IWC guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-test-from-a-workflow-invocation) recommends. [[planemo-asserts-idioms]] covers assertion choice. A test may mix local paths, remote locations, and scalar parameters in one `job:`. + +```yaml +- doc: Test with a published input + job: + Reference FASTA: + class: File + location: https://www.ebi.ac.uk/ena/browser/api/fasta/AF325528.1?download=true + filetype: fasta + hashes: + - hash_function: SHA-1 + hash_value: 927d0d00b7db6ad60524bb9e50d3ab41c4ac5ecf + outputs: + result: + asserts: + has_text: + text: expected-marker +``` + +The `Reference FASTA` file block is from the sampled IWC `virology/pox-virus-amplicon/pox-virus-half-genome-tests.yml`. The surrounding test case and output are illustrative. `filetype:` names a Galaxy datatype. The `hashes:` list records input integrity, including the algorithm and digest. SHA-1 is common in the sampled IWC inputs, but the [[tests-format|vendored test-format schema]] also permits MD5, SHA-256, and SHA-512. Record a digest only after computing or verifying it against the actual file. Local paths may also carry hashes. Do not infer a hash merely from a URL or assume invocation generation was its source. + +Input `hashes:` and output `checksum:` serve different purposes. Planemo [documents output checksums](https://planemo.readthedocs.io/en/latest/test_format.html#outputs), as well as `file:` comparisons and content assertions. The sampled IWC tests favored the latter forms, but that observation does not remove `checksum:` from the format. + +## Collection inputs + +An explicit collection names its shape with `collection_type:` and gives each element an `identifier:`. Match the workflow's declared input type and use identifiers that the workflow and output assertions can address. The [Planemo collection examples](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) show both a `list` and a nested `list:paired`. + +```yaml +job: + samples: + class: Collection + collection_type: list + elements: + - identifier: sample_a + class: File + path: test-data/sample_a.fastq.gz + filetype: fastqsanger.gz + - identifier: sample_b + class: File + path: test-data/sample_b.fastq.gz + filetype: fastqsanger.gz +``` + +The [IWC short-read QC test](https://github.com/galaxyproject/iwc/blob/main/workflows/read-preprocessing/short-read-qc-trimming/short-read-quality-control-and-trimming-tests.yml) uses the nested form. The outer collection is `list:paired`, each outer element is a `Collection` with a sample `identifier:`, and the two files are identified as `forward` and `reverse`: + +```yaml +job: + Raw reads: + class: Collection + collection_type: list:paired + elements: + - class: Collection + identifier: pair + elements: + - class: File + identifier: forward + location: https://zenodo.org/records/11484215/files/paired_r1.fastq.gz + filetype: fastqsanger.gz + - class: File + identifier: reverse + location: https://zenodo.org/records/11484215/files/paired_r2.fastq.gz + filetype: fastqsanger.gz +``` + +Only the outer collection declares `collection_type:`. Galaxy derives each inner element's type by splitting it on the first `:`, so an inner `collection_type:` (the IWC file carries a redundant `collection_type: paired`) is ignored, and `type:` is rejected by [[tests-format]]. A bare `paired` collection uses the inner shape without the outer list. For deeper collections, keep each level's identifiers aligned with the workflow input. Check the resulting YAML against [[tests-format]] and the concrete workflow. The sampled IWC tests also contain nested output assertions: an output collection's `element_tests:` selects an outer identifier, then a nested `element_tests:` selects inner identifiers. See the sampled `scRNAseq/fastq-to-matrix-10x/scrna-seq-fastq-to-matrix-10x-cellplex-tests.yml` and [[planemo-asserts-idioms]]. + +Planemo also [accepts a plain YAML list of File objects](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) as a simple Galaxy `list`. That shorthand gives no control over element identifiers. Use the explicit collection form when names or a non-list shape matter. Do not treat an absence from a sample of IWC tests as a parser restriction. + +A collection element's `identifier:` is distinct from the workflow input label used as the `job:` key. Preserve meaningful, stable identifiers when downstream outputs or tests refer to them. Quote YAML mapping keys where required by YAML syntax. In a `paired` collection, the member identifiers are `forward` and `reverse`. An outer `list:paired` element can have a sample identifier such as `pair`. The [Planemo nested example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-collection-inputs) shows this distinction. + +## Other documented input forms + +The [Planemo composite-input example](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-composite-inputs) specifies `composite_data:` as a **list** of local `path:` objects, for example an imzML file plus its `.ibd` companion. The documentation says URI components are not tested there. The sampled IWC tests did not provide a composite example, so validate an actual composite case before relying on it. + +```yaml +job: + imaging input: + class: File + filetype: imzml + composite_data: + - path: test-data/Example_Continuous.imzML + - path: test-data/Example_Continuous.ibd +``` + +Planemo also [documents `tags:` on collection elements](https://planemo.readthedocs.io/en/latest/test_format.html#galaxy-tags). File attributes such as `dbkey:` and `decompress:` appear in the [[tests-format|vendored test-format schema]]. These are available vocabulary, not requirements for every fixture. Use them when the workflow or data import needs them, then validate the concrete test. + +## Data-table values and built-in indexes + +A workflow input selecting an installed reference index takes a scalar value from the relevant Galaxy data table. For example, the [IWC contribution guide's Bowtie2 example](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#use-build-in-indexes) maps the displayed hg38 index to the plain value `hg38` from `bowtie2_indices.loc`: + +```yaml +job: + reference genome: hg38 +``` + +The value column depends on the tool and table definition. Check that tool's `tool_data_table_conf.xml` and the target instance's available table entries. This input is a parameter, not a `class: File` fixture. IWC says its default CI Galaxy has CVMFS for built-in indexes. A local or other Galaxy instance must expose the same table value and underlying reference data for this test to run. A portable file input is possible only if the workflow interface and tool accept a file in place of the built-in index. Do not replace the scalar with `class: File` solely in the test YAML. + +## Authoring and checking an IWC test + +IWC [permits hand-written cases](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) and documents `planemo workflow_test_init ` for a blank template. A successful Galaxy invocation can seed a more complete test with `planemo workflow_test_init --from_invocation ...`. Inspect generated paths, labels, hashes, and output comparisons before retaining them. IWC recommends assertions in place of large or unstable exact-output files. It also documents `planemo workflow_lint`, `planemo test`, and `planemo workflow_test_on_invocation` for checking a test against a saved invocation. [IWC testing instructions](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#manually-write-test-for-workflow), [Planemo workflow best practices](https://planemo.readthedocs.io/en/stable/best_practices_workflows.html#tests). + +IWC's preferred PR test runs on a Galaxy instance started by the GitHub Actions worker. Its [contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#request-testing-against-an-external-galaxy-instance) describes an external instance via `.wt_instance` as a last resort when required infrastructure or reference data cannot reasonably be made available in CI. That is a workflow-specific exception, not an implication of every CVMFS-backed test. Before running a test elsewhere, verify that the instance has the required tools, datatypes, data tables, and reference data. + +## Related references + +- [[galaxy-workflow-testability-design]] — choose stable workflow labels, collection types, and output checkpoints. +- [[planemo-asserts-idioms]] — choose output checks and tolerances. +- [[implement-galaxy-workflow-test]] — turn a test plan and resolved data references into a test file. +- [[tests-format]] — exact Foundry schema and validator for the YAML format. +- [IWC workflow contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md) and [Planemo test format](https://planemo.readthedocs.io/en/latest/test_format.html) — upstream contracts. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/open-requirements-ledger.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/open-requirements-ledger.md new file mode 100644 index 0000000..761183e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/open-requirements-ledger.md @@ -0,0 +1,231 @@ +--- +type: research +title: "Open-requirements ledger" +tags: + - target/galaxy +status: draft +created: 2026-06-16 +revised: 2026-11-10 +revision: 7 +related_notes: + - "[[galaxy-workflow-draft-format]]" +related_molds: + - "[[advance-galaxy-draft-step]]" + - "[[repair-galaxy-draft-topology]]" + - "[[implement-galaxy-tool-step]]" +summary: "Carried workflow-run knowledge artifact for obligations, discoveries, decisions, brief-change recommendations, and convergence state." +--- + +# Open-requirements ledger + +The `open-requirements-ledger` is the carried workflow-run knowledge artifact threaded through the source→Galaxy pipeline. It records **obligations the pipeline has taken on but not yet met** — a declared output with no producer, a parameter whose value the source never pinned, a tool with no corpus exemplar — and **source work it decided not to carry**. It also records evidence discovered during implementation, decisions made within the approved scope, and recommended changes to an immutable Workflow Brief. Each Mold that surfaces an obligation **appends** it; each Mold whose decision closes one **marks it resolved**; the terminal path **surrenders** whatever remains open, explicitly, into the final artifact. + +This ledger is about the workflow being built. It is distinct from the opt-in `foundry-feedback-ledger`, which records actionable defects or friction in the Foundry assets building it. It is also not the harness's phase-progress tracker: execution status belongs to harness state, checkpoints, and reports rather than requirement entries. + +## Framing: obligations the pipeline discharges, not questions a human answers + +This is deliberately *not* an "open questions for the user" list. The pipeline is autonomous — no human-in-the-loop gate is assumed. The ledger's consumers are **Molds and the loop's convergence gate**, with human readout a secondary affordance. An entry is closed by a downstream Mold doing work (wiring a producer, picking a wrapper, settling a value), or — when nothing can close it — surrendered: written into the final draft as a known, labelled gap rather than silently dropped or fabricated around. + +The distinction matters because a "questions for a human" framing leaks an operator's personal interaction style into a tool meant to run inside anyone's harness. The ledger must behave identically cast into a fresh harness with no ambient configuration; that holds only because every obligation is **materialized in the artifact**, never carried as operator habit. + +## Entry shape (v1, loose) + +Like the `_plan_*` family in [[galaxy-workflow-draft-format]], ledger entries are intentionally low-ceremony for v1 — enough structure to count and close, no contract pretending to be machine-parameterizable yet. A reasonable per-entry shape: + +```yaml +- id: sccmec-evidence-missing + status: open # open | resolved | surrendered + kind: gap # gap | dropped — see below; omit for gap + blocking: true # true only when a structural edit closes it; the convergence gate counts these + raised_by: implement-galaxy-tool-step # Mold that appended it + step: classify_context # draft step the obligation attaches to (if any) + unmet: "SCCmec-region candidate output category" + missing: "no wired input carries SCCmec cassette evidence" + resolved_by: null # Mold that closed it, once closed + supersedes: null # id whose justification this entry refutes, once resolved + note: "" # how it was closed or why surrendered +``` + +`kind` separates the two things an open entry can mean, which v1 conflated. A **`gap`** is +an obligation the pipeline cannot discharge: no producer is discoverable, Galaxy has no +datatype for the format, the source pinned no value. A **`dropped`** entry is source work +the pipeline *declined to carry* — a plane cut from the design, a parameter surface not +exposed, an output the repair removed. Both are honest; they are not the same admission, +and a reader cannot tell them apart from `unmet` / `missing` alone. Omit `kind` for a +plain gap. + +A `dropped` entry carries two more fields: + +```yaml +- id: annotation-planes-not-built + status: open + kind: dropped + raised_by: nextflow-summary-to-galaxy-data-flow + units: "25 of 31 designed nodes (rnaseq_short, gnomon, annot_proc, convert)" + because: "`gxwf tool-search asn1` returns no wrapper reading seq-submit; searched 2026-08-29" + unmet: "the pipeline's actual product — an annotated genome" + missing: "the carried spine masks an assembly; it does not annotate a genome" + note: "" +``` + +`units` names what was dropped in the source's own terms, so the drops can be totalled +without re-reading the design. + +`because` says why, and **must carry its citation** — the source construct that is not +there, the target limitation named specifically enough that someone could check it, the +search actually run. A drop's reason is a claim about the world, and the first real run is +why the burden exists: 25 of 31 planes were cut because "each needs an authored Galaxy +wrapper for an undocumented binary available only inside the pipeline's container," and a +Tool Shed search one phase later found the whole pipeline already wrapped at the exact +pinned version. The claim was not dishonest. It was never checked, and nothing in the +entry showed that it had not been. + +A reason you can cite, cite. A reason you cannot cite is still recorded — write it plainly +and leave it uncited rather than dressing it as evidence — but an uncited drop is not a +finding the run may reason from afterwards. It is a debt, and it should read as one. + +Provenance (`raised_by`, `resolved_by`) is the audit trail for *when* each obligation entered and left — the traceability #281 asked for, and the evidence the convergence gate reads. + +`blocking` separates the two grades of obligation the pipeline carries. A plain entry is an unmet need the chain can keep working around — an unpinned parameter, a missing exemplar. A **blocking** entry means the settled draft can't give a correct run and a structural edit closes it. The common case is a computability gap: a declared step output no wired input can supply, which `gxwf` validation cannot see because the connection graph knows ports connect, not what they carry. Others are a wrapper confirmed absent, a chosen wrapper that can't produce a declared output, a step another tool subsumes, a value that must vary per run, and a validation failure on an edge an earlier step settled. Omit `blocking` (or set it `false`) for a plain obligation. Only blocking entries drive topology-repair escalation, and only they are counted by the decreasing-blocker invariant. + +## Workflow-run knowledge beside obligations + +Not every fact learned during a run is an unmet requirement. Keep the existing `entries` list for obligations and dropped work whose `open | resolved | surrendered` lifecycle downstream Molds can act on. Record other workflow-side knowledge in separate top-level collections so it cannot accidentally participate in the blocking count or obligation lifecycle: + +```yaml +discoveries: + - id: source-requires-paired-reads + raised_by: workflow-brief-to-galaxy-interface + evidence: "The selected source method consumes paired reads." + impact: "The interface design must preserve pairing." + +decisions: + - id: expose-pairing-as-collection + decided_by: workflow-brief-to-galaxy-interface + decision: "Represent each sample as a paired collection." + rationale: "This implements the approved sample-attributable objective." + evidence: + - source-requires-paired-reads + +brief_change_recommendations: + - id: require-paired-inputs + raised_by: workflow-brief-to-galaxy-interface + proposed_change: "State that inputs must be paired-end reads." + reason: "The selected method cannot meet the accepted outcome from single-end reads." + evidence: + - source-requires-paired-reads + status: proposed +``` + +A **discovery** is source or implementation evidence a later phase needs. A **decision** is a choice the run is authorized to make within the reviewed brief; it records rationale and supporting evidence. A **brief-change recommendation** identifies a change outside that authority. The implementation run may only create it with `status: proposed`; it must never mark the recommendation accepted or edit the brief. If correctness depends on the proposed change, stop the run and return it to the separate expert editing step. + +Do not use these collections as an undifferentiated activity journal. Classify each durable learning by its effect: evidence about the workflow is a discovery, an authorized choice is a decision, an unmet actionable need is an obligation entry, and a required change of expert intent is a brief-change recommendation. Ordinary phase progress is not workflow knowledge. A problem in a Mold, packaged reference, harness, or related project belongs in `foundry-feedback.ledger.yml` when feedback mode is enabled. + +### Workflow Brief open questions + +A reviewed Workflow Brief can leave choices in `### Open questions`, each with a proposed answer and its evidence ([[workflow-brief-design]]). The expert accepted each proposal by leaving it unedited, so it is reviewed intent rather than an agent guess. The first design Mold records every open question as a decision, so later Molds and the person reading the run can see what was assumed and why: + +```yaml +decisions: + - id: keep-html-reports-only + decided_by: workflow-brief-to-galaxy-interface + open_question: "Should the FastQC zip archives be kept alongside the HTML reports?" + basis: brief-proposal # brief-proposal | agent + decision: "Expose the HTML reports only." + rationale: "The brief's answer, accepted at review." +``` + +- `open_question` quotes the brief's question, so each question traces to exactly one current decision. +- `basis: brief-proposal` adopts the brief's proposal. `basis: agent` records the Mold's own answer to a question with no proposal, as in a hand-authored brief. +- `confidence: low` copies a proposal the brief marked low confidence, or marks an `agent` answer resting on thin evidence. Omit it otherwise. Never drop a low mark. +- A Mold that finds an accepted proposal wrong, including the first design Mold while recording it, records a brief-change recommendation, because the expert accepted that answer. It keeps using the accepted answer. If correctness depends on the change, stop as described above. +- A later Mold that finds an `agent` decision wrong records its own decision. That decision repeats `open_question` and sets `supersedes` to the earlier decision's `id`. The earlier decision stays as history, unlike a superseded obligation entry, which reopens. The current answer to a question is the decision that no other decision supersedes. + +## How to use the ledger + +This section is the runtime protocol for every Mold that carries the ledger. A Mold's own page states only what is local to it — what raises an entry there, and what it does when it reads one; the mechanics are here. + +**Read it before you decide.** Ahead of the decisions this step owns, read the `open` entries and the decisions that carry `open_question`. Open entries bearing on those decisions are the ones you may be able to close. A current open-question decision is the answer to follow. Never re-derive an obligation the chain already recorded — that re-derivation is the failure this artifact exists to end. + +**Start one when none is supplied.** Every carrying Mold declares the ledger as an input, but the first Mold in a run receives none. Its absence is not an error: start an empty ledger at the declared filename and proceed. + +**Preserve every collection.** Carry obligation entries, discoveries, decisions, brief-change recommendations, and run-level headers untouched unless this Mold is explicitly adding to or updating the relevant collection. Never reinterpret one category as another merely to close it. + +**Append what you newly surface.** Each new obligation gets a stable kebab-case `id`, `status: open`, `raised_by` set to your own Mold name, and enough in `unmet` / `missing` for a later Mold to act without re-reading the source. Attach it to a draft step via `step` where one applies. Set `blocking: true` only when the settled draft can't give a correct run and a structural edit closes it. + +**Mark resolved what you close.** When a decision you make discharges an open entry, set `status: resolved`, `resolved_by` to your own Mold name, and `note` to how it closed. Resolving is not deleting — a resolved entry stays in the ledger as the audit trail. + +**Carry the rest untouched.** Entries you neither raise nor close pass through unchanged, statuses and provenance intact. Never renumber, re-word, or drop another Mold's entry. + +**Never fabricate around an entry.** An open obligation is the honest state. Inventing a connection, a tool id, or a value so the artifact looks complete converts a tracked gap into a silent defect — the specific failure the ledger replaces. + +**Record a drop as a drop.** When a decision removes source work from what the run will +carry — narrowing a design, cutting a plane, dropping an output during repair — append a +`kind: dropped` entry naming the `units` and a cited `because`. A cut recorded only as +prose in a brief is invisible to every later Mold; a cut recorded as a `gap` reads as +something the pipeline could not do rather than something it chose not to do. Recording +the drop is not the same as discharging it, and a well-written entry is not a substitute +for the work — an entry makes a cut *legible*, never *justified*. + +**A refuted justification reopens its decision.** When work you do contradicts the +`because` another entry rests on — discovery finds the wrapper a drop assumed absent, a +later brief supplies the evidence a narrowing assumed missing — set `supersedes` to that +entry's `id` on your own resolved entry, and return the superseded entry to `status: open` +with its `because` struck and the contradiction quoted in its `note`. Do not leave a +refuted rationale standing: the terminal writes these into the final artifact, and a +surrendered gap whose stated reason is known to be false is worse than an unexplained one. +Reopening does not oblige *you* to do the dropped work — it obliges the run to stop +claiming a settled reason it no longer has. + +**Record a surrender note; leave the status to the terminal.** When you cannot close an entry and nothing downstream can either, leave it `open` and say so in `note`. A drafty step never leaves the loop, so [[repair-galaxy-draft-topology]] removes one it can't close, when it can cite why, and records the drop. The dropped entry stays `open`, and the terminal writes it into the final artifact. `surrendered` is a terminal status, set at the end of a run: the escalation cap reached, or the final artifact emitted with the obligation still unmet. Either way the entry stays visible and is written into the final artifact as a labelled gap, never dropped. + +**Maintain the escalation budget** — loop Molds only. The `topology_repair` header is loop-level state, described under **Escalation budget (loop-level state)** below. On each escalation the orchestrator increments `escalations`, appends the post-repair open blocking-entry count to `open_history`, and surrenders the still-open blocking entries once `escalations` reaches `cap`. + +## Role in topology repair + +In the Galaxy per-step loop, the ledger is the substrate the topology-repair escalation rides on (see [[galaxy-workflow-draft-format]] and [[repair-galaxy-draft-topology]]): + +- **[[implement-galaxy-tool-step]]** detects, mid-implementation, that a declared step output cannot be computed from its wired inputs, or that the chosen wrapper can't produce it. Rather than fabricate or author around the wrapper, it appends a blocking entry and falls through to repair. Earlier Molds and the loop raise blocking entries too. +- **[[repair-galaxy-draft-topology]]** reads the open blocking entries, makes the structural edits that close them (template-tier authoring), and marks each entry it closes `resolved`; the existing discover-or-author → implement machinery then realizes any new steps. +- Once no drafty step is left, [[advance-galaxy-draft-step]] sends any open blocking entry to repair before extracting. +- The loop's **decreasing-blocker invariant** counts open blocking entries: each repair must leave fewer than it found, under a hard cap on escalations. When the cap is hit with entries still open, those entries are **surrendered** into the final draft — the clean terminal that replaces spin-or-fabricate. + +So the ledger is not a convenience here; it is the countable state the termination guard depends on. + +## Escalation budget (loop-level state) + +The decreasing-blocker invariant needs more than the entries themselves: "lower the open count, under a hard cap" is *loop-level* state — it spans iterations and belongs to no single entry. The ledger carries it in a small `topology_repair` header beside the `entries:` list: + +```yaml +topology_repair: + escalations: 2 # repairs invoked this run + cap: 5 # hard ceiling; at cap, still-open entries are surrendered + open_history: [4, 3, 2] # open blocking-entry count after each escalation +entries: + - id: sccmec-evidence-missing + status: open + blocking: true + # … per-entry shape above +``` + +- `escalations` — how many times [[advance-galaxy-draft-step]] has escalated to [[repair-galaxy-draft-topology]] this run. The orchestrator creates the header at the first escalation, with `cap: 5` unless the run sets another, and increments it on each escalation. +- `cap` — the hard ceiling. When `escalations` reaches `cap`, the loop stops escalating and surrenders any still-`open` blocking entries into the final draft as labelled gaps rather than retrying forever. +- `open_history` — the open blocking-entry count recorded after each escalation. Entries raised between escalations can raise it again, so the convergence gate compares each repair with the count it started from: a repair must leave fewer open blocking entries than it found. One that doesn't is non-convergent and trips the gate immediately: set `escalations` to `cap`, so later calls read the same stop the cap gives. + +This block lives in the ledger as a v1 expedient: the ledger is already the carried, cast-visible state the loop reads each iteration. Its better long-term home is the **draft itself** — a workflow-level annotation that travels with the artifact it bounds, so the budget survives even when the ledger is stripped at the terminal. Recorded here for now; migrate when the draft grows a home for it. + +## Threaded through the whole design tier + +The ledger is not loop-only. Every design-tier Mold in the source→Galaxy pipelines **consumes and re-emits** it — the interface, reference-data, data-flow, IWC-comparison, and template Molds (`*-summary-to-galaxy-interface`, `*-summary-to-galaxy-data-flow`, `*-summary-to-galaxy-reference-data`, `compare-against-iwc-exemplar`, `*-summary-to-galaxy-template`), alongside the change-set Molds on the update path. This is the direct answer to #281: the chain stops re-deriving the same unknowns because each Mold inherits them. + +The template Mold's computability review pass is the notable appender ahead of the loop — `*-summary-to-galaxy-template` re-reads each settled step and, where an output needs evidence no input carries, wires the producer or records the gap as a blocking entry. Source-summary Molds upstream of the design tier keep their own free-text open-questions; the design tier is where those formalize into the ledger. + +## Open work + +- Decide whether the drops should be **counted**, not just labelled. `kind: dropped` and `units` make each cut legible one entry at a time; nothing totals them, so a run that drops a plane four times over still reports four reasonable-looking entries. A conservation rule — every source unit ends the run carried, subsumed, or surrendered, and the buckets sum to the source total — would make the aggregate visible the way `open_history` makes escalation visible. Deliberately not attempted here: labelling is cheap and may prove sufficient, counting needs a definition of "source unit" per source kind. +- Decide whether entry structure should harden further (typed `unmet` / `missing`, links back to source-summary evidence, machine-checkable `status` transitions) once two or three worked runs exercise it. Hardening it would also give the producing Molds an `output_artifacts[].schema` to cite, which none carry today. +- Assign terminal surrender of *non-blocking* entries. [[advance-galaxy-draft-step]] surrenders open blocking entries at the escalation cap, but an unpinned-parameter entry the design tier appended and nobody closed currently rides out the run without ever being marked `surrendered`. Nothing owns that pass today. +- Reconcile the design-tier briefs' free-text "open questions" sections with the ledger. A Workflow Brief's open questions now become ledger decisions (see **Workflow Brief open questions**), but every design Mold still emits its own open questions and ledger entries, with no rule for which destination an unresolved choice belongs in. +- Decide whether the source-summary Molds should also emit structured entries, or keep their free-text open-questions until the design tier formalizes them. +- Specify how surrendered entries surface in the runnable gxformat2 (a workflow-level annotation, a report output, or a sidecar) when the draft is stripped of `_plan_*` and TODO sentinels. +- Move the `topology_repair` escalation budget out of the ledger and into the draft (a workflow-level annotation) so it travels with the artifact it bounds; the ledger home is a v1 expedient. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md new file mode 100644 index 0000000..bd1160e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/planemo-asserts-idioms.md @@ -0,0 +1,124 @@ +--- +type: research +tags: + - target/galaxy +status: draft +created: 2026-04-30 +revised: 2026-10-21 +revision: 10 +related_notes: + - "[[galaxy-workflow-testability-design]]" + - "[[iwc-test-data-conventions]]" + - "[[iwc-shortcuts-anti-patterns]]" + - "[[implement-galaxy-workflow-test]]" + - "[[tests-format]]" + - "[[planemo-workflow-test-architecture]]" + - "[[validate-tests]]" + - "[[iwc-tabular-operations-survey]]" + - "[[galaxy-discover-datasets]]" +summary: "Choose Galaxy workflow-test output assertions by the failure they should catch, the output's stability, and the available checkpoints." +--- + +# Choosing Planemo workflow-test assertions + +Name the wrong result a test must catch before choosing YAML syntax. For a fixed input and byte-stable output, compare the result with an expected file or checksum. For a variable result, assert the stable properties that would fail if the analysis were wrong. A size, shape, or existence check is a useful smoke test when content is genuinely variable, ideally alongside a stable checkpoint. It is an anti-pattern when it replaces a more meaningful check that is available. [[iwc-shortcuts-anti-patterns]] develops that judgment, while [[iwc-test-data-conventions]] covers input fixtures and [[tests-format]] supplies the exact assertion vocabulary. + +## 1. Choose by the property, then the output type + +| Output | Meaningful check to try | If the result varies | +| --- | --- | --- | +| Text report or HTML | [[tests-format#has_text_model|has_text]] for a stable result, section, or value. Use [[tests-format#has_line_model|has_line]] for a complete stable line. | Use [[tests-format#has_text_matching_model|has_text_matching]] or [[tests-format#has_line_matching_model|has_line_matching]] for a bounded variable field. Line count or size can supplement the content check. | +| TSV, CSV, BED, GFF, or GTF | Compare a small deterministic file, or check a stable header and representative data row. [[tests-format#has_n_columns_model|has_n_columns]] checks the first line's width (see section 4). | Check a stable row or field, with a justified [[tests-format#has_n_lines_model|has_n_lines]] range if row count varies. Do not infer correct rows from column count alone. | +| VCF | Compare expected calls exactly when reproducible. Otherwise assert a known variant record or a bounded record pattern. | Header changes may justify a measured `lines_diff:`, but the allowance applies to the whole file. Pair it with a record-level check. | +| FASTA or FASTQ | Compare exact sequence output when reproducible, or assert a known sequence or read identifier. | Line or size checks only establish rough shape. Check a stable sequence statistic or expose one as another output when content varies. | +| JSON | [[tests-format#has_json_property_with_value_model|has_json_property_with_value]] for a stable JSON value, or [[tests-format#has_json_property_with_text_model|has_json_property_with_text]] for a stable string. | Choose a stable property or companion summary. Searching for `{` proves little beyond text presence. | +| HDF5 or AnnData | [[tests-format#has_h5_keys_model|has_h5_keys]] for required groups or datasets, and [[tests-format#has_h5_attribute_model|has_h5_attribute]] where an attribute value matters. | Keys prove structure, not cell counts or biological results. Assert a stable table or report checkpoint too. | +| XML | [[tests-format#is_valid_xml_model|is_valid_xml]] plus [[tests-format#has_element_with_path_model|has_element_with_path]], element text, or element count for the property at issue. | Ignore irrelevant formatting while keeping a stable semantic assertion. | +| Images and plots | Image dimensions or channel/frame checks confirm rendering and shape. | Check the table, matrix, or summary behind the plot if available. Size and dimensions alone do not verify plotted values. | +| ZIP, tar, or tar.gz archives | [[tests-format#has_archive_member_model|has_archive_member]] for required members, optionally with nested assertions on member content. | Compare stable members or a companion report when the archive bytes vary. This assertion is for archive formats, not BAM or HDF5. | +| BAM or other opaque binary | Exact file or checksum comparison if byte-stable. | A measured size band can be a smoke check. Expose a flagstat, coverage table, or other stable summary to check the analysis. | +| Output collection | Assert collection type or count and address significant members by identifier with `element_tests:`. | Member existence alone cannot verify member content. Give representative members meaningful assertions. | + +If only a weak final artifact is exposed, consider adding a workflow output that carries a stable result before settling on the smoke check. The [IWC Scanpy workflow test](https://github.com/galaxyproject/iwc/blob/main/workflows/scRNAseq/scanpy-clustering/Preprocessing-and-Clustering-of-single-cell-RNA-seq-data-with-Scanpy-tests.yml) checks image properties and also asserts on AnnData and text or table outputs. [[galaxy-workflow-testability-design]] explains how to choose those checkpoints. An assertion on a checkpoint should test the relevant result, not merely that its file exists. + +## 2. Compare operators and expected artifacts + +`file:` or `location:` supplies an expected artifact for a comparison. `compare: diff` is the default and is appropriate for byte-stable results. An output `checksum:` can cover a large byte-stable result without storing the whole expected file. Use `compare: re_match` or `re_match_multiline` when the expected artifact expresses bounded regex variation, `contains` when an expected block must occur in the output, and `sim_size` when only approximate byte size is dependable. [Planemo's output-format guide](https://planemo.readthedocs.io/en/latest/test_format.html#outputs) documents these forms. Exact comparison gives a useful diff on failure, so keep it when the output really is stable. + +`lines_diff:` on `diff` is a count of allowed differing lines, not an instruction to ignore a header. A changed line counts as a removal and an addition under the [[tests-format|vendored schema]]. For a VCF with mutable provenance, inspect actual comparisons to set any allowance, then ensure the allowed count cannot let a changed call pass unnoticed. A fixed `lines_diff: 6` is not a VCF default. If a record is stable, assert that record separately or compare a normalized output. + +## 3. Set tolerances from the failure boundary + +For `compare: sim_size`, `delta:` is an absolute byte allowance and `delta_frac:` a fractional allowance relative to the expected artifact. For [[tests-format#has_size_model|has_size]], `delta:` is an absolute byte allowance around `size:`. In count and image assertions, `delta:` uses that assertion's unit, such as lines or pixels. Do not transfer a number between assertion families without checking its unit in [[tests-format]]. + +Run representative inputs across the environments the test must support. Record the observed range and choose the smallest allowance that includes expected variation while rejecting the wrong result named at the start. In particular, check whether the lower bound admits an empty output. If it does, add a positive minimum or a stronger assertion. The [IWC RepeatMasking test](https://github.com/galaxyproject/iwc/blob/main/workflows/repeatmasking/RepeatMasking-Workflow-tests.yml) uses broad size comparisons for variable outputs, but its byte allowances are measurements for that case, not reusable defaults. There is no general 10% tolerance for new outputs. + +## 4. Text assertions and repeated checks + +Use `has_text` for a literal substring, `has_line` for a complete line, and their `*_matching` forms when a regex is needed. `has_text_matching` is an unanchored search, but `has_line_matching` is not: Galaxy wraps the expression as `^$` in multiline mode, so it must match a whole line. `^I\t` matches only a line that is exactly `I` and a tab; write `I\t.*` for lines that start with `I` and a tab. A leading `^` or trailing `$` is redundant but harmless. `has_n_lines` checks count and can detect truncation, but a plausible count does not establish correct content. A test can combine count with a stable token or row. `has_n_columns` counts the `sep:`-delimited fields of the first non-comment line only, and `comment:` is the only control over which line that is. When header and data rows differ in width, it measures the header. R's `write.table(x, row.names = TRUE, col.names = TRUE)` writes N header fields above N+1-field data rows, so a five-sample matrix has `has_n_columns` 5. Check data-row width with a `has_line_matching` that fixes the field count, e.g. `([^\t]*\t){5}[^\t]*` for six fields. The [[tests-format|schema]] accepts `asserts:` as a mapping for one assertion of each kind or as a list. Use the list form to repeat a kind: repeated YAML mapping keys can silently overwrite earlier checks. + +```yaml +- doc: Stable report fields + job: + input: + class: File + path: test-data/input.txt + outputs: + report: + asserts: + - has_text: + text: "Total Sequences" + - has_text: + text: "Passed" + - has_n_lines: + n: 12 +``` + +The names and count above are illustrative. Replace them with values observed from the fixture being tested. A `has_text` assertion with `text: "ERROR"` and `negate: true` is useful when absence of that marker is part of the output contract. A workflow-level `expect_failure:` case is useful when the workflow itself promises to reject an invalid input. Neither is an automatic requirement for every workflow. + +## 5. Collection and structured-output examples + +For a collection output, `element_tests:` maps produced element identifiers to assertions, at each level of a nested output collection, as the [IWC SRA manifest test](https://github.com/galaxyproject/iwc/blob/main/workflows/data-fetching/sra-manifest-to-concatenated-fastqs/sra-manifest-to-concatenated-fastqs-tests.yml#L12-L27) does for samples and their forward/reverse files. The output's `collection_type:` and `element_count:` check shape, while element assertions check results. A nested sub-collection under `element_tests:` takes `count:` (or `min:`/`max:`) instead of `element_count:`, and no `collection_type:`, since the output's `collection_type:` already fixes every rank. Naming members only proves they exist, so add `count:` when exact membership matters, e.g. `count: 1` under an unpaired sample in a `list:paired_or_unpaired` output so a spurious second member fails. See [[tests-format]] and [Galaxy's collection-test parser](https://github.com/galaxyproject/galaxy/blob/a63da1dfd1960360f4aa2fddc6a75396954d750a/lib/galaxy/tool_util/parser/interface.py#L860-L870). + +```yaml +- doc: Check selected collection members + job: + input: + class: File + path: test-data/input.txt + outputs: + results: + class: Collection + collection_type: list + element_tests: + sample_a: + asserts: + has_json_property_with_text: + property: status + text: complete + sample_b: + asserts: + has_json_property_with_value: + property: count + value: "3" +``` + +For `has_json_property_with_value`, `value:` is a **JSON-encoded string**, so a numeric three is written `"3"` in YAML. For HDF5, `has_h5_keys` takes one comma-separated `keys:` string, for example `keys: "obs/louvain,var/highly_variable"`. Use an additional result assertion if these structural keys could survive a scientifically wrong computation. Check the syntax against the vendored schema rather than copying YAML with repeated keys from an existing test. + +## 6. Validate labels and schema before execution + +A schema-valid test can still name an output or input label that the workflow does not expose. Run [[validate-tests]] with the workflow to check both the test format and the label/type match before a Galaxy run: + +```sh +gxwf validate-tests workflow-tests.yml --workflow workflow.ga +``` + +That static check cannot prove an assertion passes or that a chosen property is scientifically meaningful. Follow it with `planemo workflow_test_on_invocation` against a saved successful invocation when available, then run the full [[planemo-test|Planemo test]] for integration. If an assertion fails, inspect the actual output and the named failure before widening a tolerance. See [[planemo-workflow-test-architecture]] for execution and failure evidence. + +## 7. Create and refine the test + +The [IWC contribution guide](https://github.com/galaxyproject/iwc/blob/main/workflows/README.md#generate-tests) permits writing a test by hand or generating a starting file. `planemo workflow_test_init` can create a template, and `--from_invocation` can use an existing successful invocation. Inspect generated input references, labels, expected files, and comparisons. Keep exact comparisons that are stable. Replace those made brittle by known variation with targeted assertions that still catch wrong results. [[planemo-workflow_test_on_invocation]] can rerun assertion checks against that invocation without executing the workflow again. + +## 8. Source of the exact syntax + +[[tests-format]] points to the vendored JSON Schema for accepted fields, types, defaults, and assertion arguments. This note supplies the choice and its limits, not a second schema. Consult the [Planemo test-format documentation](https://planemo.readthedocs.io/en/latest/test_format.html) for the upstream test-file structure and [[iwc-shortcuts-anti-patterns]] for when a smoke check conceals an available stronger test. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md new file mode 100644 index 0000000..72583a5 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/planemo-workflow-test-architecture.md @@ -0,0 +1,168 @@ +--- +type: research +title: "Planemo workflow-test architecture" +tags: + - tool/planemo + - target/galaxy +status: draft +created: 2026-05-02 +revised: 2026-10-26 +revision: 4 +related_notes: + - "[[galaxy-workflow-testability-design]]" + - "[[galaxy-tool-job-failure-reference]]" + - "[[galaxy-workflow-invocation-failure-reference]]" + - "[[planemo-asserts-idioms]]" +related_molds: + - "[[run-workflow-test]]" + - "[[debug-galaxy-workflow-output]]" + - "[[implement-galaxy-workflow-test]]" +sources: + - "~/projects/repositories/planemo/planemo/commands/cmd_test.py" + - "~/projects/repositories/planemo/planemo/commands/cmd_run.py" + - "~/projects/repositories/planemo/planemo/galaxy/activity.py" + - "~/projects/repositories/planemo/planemo/galaxy/invocations" + - "~/projects/repositories/planemo/planemo/galaxy/config.py" + - "~/projects/repositories/planemo/planemo/galaxy/serve.py" + - "~/projects/repositories/planemo/planemo/engine/galaxy.py" +summary: "Reference for Planemo workflow test/run architecture, Galaxy modes, API polling, and noisy failure boundaries." +--- + +# Planemo Workflow-Test Architecture + +This note describes Planemo architecture relevant to workflow tests and workflow runs. It is reference material for Molds that need to run tests or interpret Planemo artifacts, not a command-selection recipe. + +## Main Commands + +| User action | Command | Core behavior | +|---|---|---| +| Full workflow test | `planemo test ` ([[planemo-test]]) | Finds test definitions, starts or targets Galaxy, stages inputs, invokes workflow, checks assertions, writes reports. | +| Direct run | `planemo run ` | Runs one workflow/job pair and can download outputs without assertion checks. | +| Recheck assertions | `planemo workflow_test_on_invocation ` ([[planemo-workflow_test_on_invocation]]) | Runs test assertions against an existing invocation without rerunning the workflow. | +| Track invocation | `planemo workflow_track ` | Polls an existing invocation and displays progress. | +| Generate test from invocation | `planemo workflow_test_init --from_invocation ` ([[planemo-workflow_test_init]]) | Builds a test template from a completed invocation and its outputs. | +| Generate job template | `planemo workflow_job_init ` | Creates a job-input template for a workflow. | + +Observed code paths live under `~/projects/repositories/planemo/planemo/commands/`, `planemo/engine/`, and `planemo/galaxy/`. + +## Engine Selection + +Planemo chooses between engines early: + +- CWL runnables can use CWL-specific engines. +- Supplying `--galaxy_url` implies an external running Galaxy unless an engine is explicitly selected. +- Default Galaxy workflow testing uses a managed local Galaxy engine. +- `--engine docker_galaxy` uses a managed Dockerized Galaxy. +- `--engine external_galaxy` uses an existing running Galaxy. + +“Existing Galaxy” can mean two different things: + +- Existing local Galaxy source tree, still managed by Planemo for this run. +- Existing running Galaxy server, addressed through URL and API keys. + +Mold output and eval logs should record which meaning applies. + +## Managed Galaxy Configuration + +For managed Galaxy, Planemo builds a temporary configuration directory and generated Galaxy config. It can configure tool config, shed tool config, job config, file sources, object store paths, dependency config, SQLite database by default, admin users, API keys, and environment overrides. + +Important managed-mode behavior: + +- Planemo may find a local Galaxy root or install/use a cached Galaxy source tree. +- `--galaxy_branch` selects which Galaxy branch is cloned when no Galaxy root is supplied; it defaults to `master`. Omitting it therefore tests against the newest Galaxy. Naming an older `release_*` branch narrows what workflow syntax Galaxy will accept, and the resulting rejection surfaces as an invocation-time error rather than as a version mismatch. +- Managed Galaxy defaults are convenient but may not match production Galaxy behavior. +- Planemo can install workflow tool dependencies through Tool Shed metadata when configured. +- Test histories are created automatically unless a history id is supplied. + +## External Galaxy Configuration + +For external Galaxy, Planemo uses supplied Galaxy URL and API keys. A user key runs workflows; an admin key may be needed for installing missing repositories or creating a user key. + +External Galaxy mode is useful when the target environment matters, but failure surfaces can include server configuration, installed tools, permissions, and existing history state. Eval logs should preserve URL/key mode without storing secrets. + +## API Interaction + +Planemo primarily talks to Galaxy through BioBlend-backed clients. + +Important operations: + +- Import workflows into Galaxy. +- Stage datasets and collections into histories. +- Invoke workflows by input names. +- Poll invocations and jobs. +- Fetch job details with full detail when needed. +- Download outputs and output collections. +- Run assertion checks and write structured reports. + +For workflows, Planemo invokes Galaxy using input labels/names. Stable generated labels are therefore important for both test execution and debugging. + +### Workflow Import Failures + +Planemo imports the workflow while it brings Galaxy up for the run (`config.install_workflows()`, called from `_serve()` in `planemo/galaxy/serve.py` for managed Galaxy and from `ExternalGalaxyEngine.ensure_runnables_served` in `planemo/engine/galaxy.py` for external Galaxy, as of planemo 0.75.47). For gxformat2 it converts the workflow to a native dict client-side first, so Galaxy's own gxformat2 conversion never runs. That is outside the per-test `execute()` wrapper that turns exceptions into an `ErrorRunResponse`. When Galaxy rejects the import request, BioBlend raises `ConnectionError`, the test command exits non-zero, and Planemo writes neither the `--test_output_json` report nor the HTML report. No workflow, history, or invocation id exists. + +The evidence then lives in three places: + +| Evidence | What it holds | +|---|---| +| Upload response | HTTP status and body. A 400 usually names the failing field or validation rule. A 500 body rarely explains the cause. | +| Galaxy server log | The traceback for the failing request, the only place a 500's cause appears. Managed Galaxy streams its log into the Planemo terminal output and keeps it on disk. External Galaxy keeps it on the server. | +| Planemo terminal log | The BioBlend exception and the point in the run where it was raised. | + +Re-importing the same converted dict as the same user on the same server reproduces the failure without the rest of the Planemo run, which makes it the cheap loop for isolating a trigger. + +Workflow testability design guidance lives in [[galaxy-workflow-testability-design]]. This architecture note only records why Planemo makes labels and workflow-level outputs operationally important. + +## Structured Artifacts + +Useful Planemo artifacts and fields: + +| Artifact or field | Use | +|---|---| +| `tool_test_output.json` | Primary structured result artifact for tests. | +| invocation id | Key for Galaxy invocation API follow-up. | +| history id | Key for history contents and output inspection. | +| workflow id | Key for workflow invocation aliases and reruns. | +| output problems | Assertion and missing-output failures. | +| execution problem | Staging, invocation, API, or other execution-level failure. | +| invocation details | Step/job/output details Planemo collected from Galaxy. | +| job details | Tool id, state, exit code, command, stdout/stderr when collected. | + +Terminal output is not enough for durable failure analysis. Preserve structured output whenever possible. + +## Noisy Boundaries + +Planemo transforms and summarizes Galaxy failure information. These are likely information-loss boundaries: + +- Exceptions during execution can become stringified `ErrorRunResponse` values. +- Polling may summarize “at least one job is in error” while detailed job evidence is printed elsewhere or stored separately. +- Missing outputs become assertion/output problems, which can conflate label mismatch, workflow output omission, optional output absence, failed invocation, or output download issue. +- Output download failures may be logged but not always propagated as the most specific failure. +- A rejected workflow import aborts the run before any structured report is written, so terminal and server logs are the only record. +- External Galaxy without an admin key may defer missing-tool problems until runtime. +- `allow_tool_state_corrections=True` can let Galaxy adjust tool state during invocation, which is useful but can mask definition drift. +- `--no_wait` is not suitable for debug conclusions because invocation creation can succeed before jobs fail. + +## Reference Use For Molds + +For [[run-workflow-test]]: + +- Preserve structured Planemo result output, invocation id, history id, workflow id, and Galaxy mode. +- Record whether the run used managed local Galaxy, Docker Galaxy, or external Galaxy. +- Do not treat terminal output as the primary failure artifact. + +For [[debug-galaxy-workflow-output]]: + +- Start from structured Planemo output. When there is none because the workflow import failed, start from the upload response and the Galaxy server log. +- For missing-output failures, check label drift and omitted workflow-level outputs before assuming tool failure. +- If the failure is job-level, inspect Galaxy job APIs described in [[galaxy-tool-job-failure-reference]]. +- If the failure is invocation-level, inspect Galaxy invocation APIs described in [[galaxy-workflow-invocation-failure-reference]]. +- If only assertions failed, use [[planemo-asserts-idioms]] before deciding to rerun. + +## Verification Gaps + +Actual runs should verify: + +- Which invocation messages Planemo exposes directly. +- Whether target Planemo/Galaxy versions populate `completed`, `/completion`, and step job summaries. +- How `workflow_test_on_invocation` behaves for failed, warning-only, mapped collection, and subworkflow invocations. +- Whether generated test cases from invocation preserve nested output collection structure correctly. diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/workflow-brief-design.md b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/workflow-brief-design.md new file mode 100644 index 0000000..8bc1e6d --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/notes/workflow-brief-design.md @@ -0,0 +1,96 @@ +--- +type: research +title: "Workflow Brief" +tags: + - meta +status: draft +created: 2026-09-17 +revised: 2026-10-19 +revision: 6 +summary: "Markdown Workflow Brief definition and section contract for scope, constraints, agent environment, and explicit blockers." +--- + +# Workflow Brief + +A Workflow Brief is an expert-editable document describing what a workflow should achieve, what it should leave out, and the constraints under which an agent will build and run it. It is an input to design and implementation. The draft workflow captures the proposed steps, wiring, and tool state. + +## Document shape + +The brief is Markdown with a title and two sections: `## Workflow` and `## Agent Environment`. The workflow needs an objective; most details are optional. The agent environment must contain a description, which can explicitly record what still needs to be checked. + +The `workflowBriefSchema` declaration uses the shared engine described in [[markdown-document-contract]] and lives in `packages/gxwf-foundry/src/workflow-brief.ts`. An illustrative document lives in `packages/gxwf-foundry/test/fixtures/workflow-brief/read-alignment.md`. + +| Heading | Required | Content | +| --- | --- | --- | +| `## Workflow` | Yes | Scientific intent and scope. | +| `### Objective` | Yes, under Workflow | Scientific goal and desired outcome. | +| `### Sources` | No | Papers, interviews, repositories, relevant source pins, and evidence availability. | +| `### Scope` | No | Optional `#### Included` and `#### Excluded` describe selected work and boundaries. | +| `### Inputs and outputs` | No | Scientific roles and source-supported formats or organization. | +| `### Requirements and preferences` | No | Expert or source requirements, their strength, rationale, and evidence. | +| `### Acceptance criteria` | No | High-level behaviors that establish success. | +| `### Open questions` | No | Unresolved choices that do not prevent work, each with a proposed answer and its evidence. | +| `### Decisions` | No | Expert decisions and rationale. | +| `### Related artifacts` | No | Links to source evidence and other relevant inputs. | +| `### Blockers` | No, under Workflow | Unresolved issues that prevent design or implementation. Any content blocks work. | +| `## Agent Environment` | Yes | Agent tooling and operating conditions. | +| `### Tooling` | No | Expected Foundry tooling and versions, and what is known about availability. | +| `### Constraints` | No | Agent workspace, network, installation, resource, and access restrictions. | +| `### Containerization` | No | Container preference and Docker, Singularity, or Apptainer availability. | +| `### Blockers` | No, under Agent Environment | Environment issues that prevent work. Any content blocks work. | + +Headings are case-insensitive; section order is flexible and additional sections are allowed. Recognized sections are unique within their parent. If present, ordinary sections need content beyond headings, comments, or separators. Blockers may be empty. Scope's Included and Excluded subsections are optional. + +## Workflow content + +The writing agent must record evidence and expert intent without making Galaxy design choices. It must not assume Galaxy datatypes, dataset collection structures, tool availability, interface labels, or concrete tool versions. Source formats and explicitly named source tools can be cited as evidence; they do not establish a Galaxy mapping or an installed wrapper. An expert may supply specific Galaxy requirements here, and the agent must identify them as expert-provided rather than inferred. + +The agent must not write a step graph, concrete tool state, or final test declarations into the brief. Interface design, data-flow design, tool discovery, and test development belong to the downstream design and implementation Molds. Inputs and outputs stay at scientific roles and source-supported descriptions. Acceptance criteria stay high-level; detailed fixtures and assertions belong to test development. + +Requirements and preferences describe what the expert or source asks for. Agent Environment Constraints describe restrictions on the agent's work. Uncertainty that prevents proceeding belongs in Blockers. Record every other unresolved choice in Open questions with a proposed answer and the evidence for it, such as source behavior, the expert's wording, or the available test data. Propose the answer the evidence best supports. Where expert intent and source behavior conflict, propose the expert's intent and cite the conflict. When the evidence is thin, still propose an answer, mark it low confidence, and say what evidence is missing. The expert accepts a proposal by leaving it unedited and overrides it by editing the answer. + +Write each open question as a bullet with `Proposed:` and `Evidence:` sub-bullets. Add `Confidence: low` and `Missing:` when the evidence is thin: + +```md +- Should runs of one sample be merged before trimming? + - Proposed: yes, merge them as the source does. + - Evidence: the source merges sample-sheet rows sharing a sample ID. The selection does not mention merging. + - Confidence: low + - Missing: whether the expert's samples ever span several runs. +``` + +## Agent environment + +Tooling records whether expected tools such as `gxwf`, `foundry`, and `planemo` are available at the required versions. Distinguish intended requirements from observed availability. Containerization records the preference and whether Docker, Singularity, or Apptainer is actually usable, rather than assuming that an executable name proves a working engine. + +A separate preflight result should record observed versions, container availability, source/reference access, writable paths, timestamps, and diagnostics. A requested installation strategy is not evidence of an installed executable. + +## Validation and blockers + +```sh +foundry validate-workflow-brief workflow-brief.md +foundry check-workflow-brief workflow-brief.md --json +``` + +Validation checks the title, required sections, unique recognized headings, content, and parent-scoped subsections. Example headings in fences, quotes, lists, or comments do not satisfy requirements. Success exits `0`, structural failures exit `3`, and input read failures exit `1`. + +The static readiness check adds a check for `### Blockers` under both Workflow and Agent Environment. A structurally valid document with content in either section exits `4`; a structurally valid document without blockers exits `0`. Omit or leave Blockers empty when there are no blockers. Do not write “none” there: any content, including that word, is blocking. Comments and separators alone are empty. The JSON result includes `valid`, `ready`, `errors`, and blocker bodies with their parent and source line. + +A clear static check only establishes structure and the absence of declared blockers. It does not establish semantic completeness, expert approval, or successful environment preflight. The harness must stop on declared blockers and apply its own review and preflight checks before implementation. + +The exported `WorkflowBrief` and `WorkflowBriefSection` types describe the parsed document. `WorkflowBriefReadinessResult` describes the static check. These checks read the document without changing it. + +## Lifecycle + +1. A source-to-brief producer assembles evidence, proposed scope, and unanswered questions without Galaxy design assumptions. +2. The expert edits the brief and settles blockers needed before the next stage. +3. The harness checks structure, declared blockers, review, and environment readiness. +4. Brief-to-Galaxy consumes the brief and referenced evidence, then runs design, draft, implementation, and test development. + +Brief-to-Galaxy and every design or implementation step must leave the brief unchanged. Record durable workflow knowledge in `open-requirements.ledger.yml`: evidence as discoveries, authorized choices as decisions, unmet needs as obligations, and changes to expert intent as proposed brief-change recommendations. Ordinary phase progress remains harness state. Updating the brief belongs to a separate expert editing step; an implementation agent must not resolve a blocker by rewriting its input. + +A hand-authored brief is a first-class input. Implementation should work in a fresh session using the document and its referenced artifacts. + +## Open design questions + +- Should expert questions live here or link to a separate decision record? diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json new file mode 100644 index 0000000..82e4529 --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/galaxy-workflow-test-plan.schema.json @@ -0,0 +1,553 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://galaxyproject.org/foundry/schemas/galaxy-workflow-test-plan.schema.json", + "$comment": "Canonical source: packages/gxwf-foundry/src/schemas/galaxy-workflow-test-plan/galaxy-workflow-test-plan.schema.json in galaxyproject/foundry. Mold frontmatter cites this schema via [[galaxy-workflow-test-plan]] wiki-links; the cast pipeline imports the `galaxyWorkflowTestPlanSchema` runtime export and serializes it into cast bundles. The on-disk artifact is YAML; validate it with `foundry validate-galaxy-workflow-test-plan`.", + "title": "Galaxy Workflow Test Plan", + "description": "Intermediate, reviewable Galaxy workflow test-plan handoff produced by a *-test-to-galaxy-test-plan Mold (nextflow, cwl, freeform, or galaxy) and consumed by implement-galaxy-workflow-test. It preserves test intent, fixture provenance, assertion intent, tolerances, label assumptions, unresolved mappings, and intentional omissions before any concrete tests-format `*-tests.yml` is authored. It is NOT the final test artifact: assertion vocabulary is referenced by family name from the tests-format schema, not duplicated here.", + "type": "object", + "additionalProperties": false, + "required": [ + "plan_version", + "source", + "workflow", + "test_cases", + "unresolved", + "omissions", + "warnings" + ], + "properties": { + "plan_version": { + "type": "string", + "enum": [ + "1" + ], + "description": "Test-plan schema major version." + }, + "source": { + "$ref": "#/$defs/SourceRecord" + }, + "workflow": { + "$ref": "#/$defs/WorkflowRef" + }, + "test_cases": { + "type": "array", + "description": "One entry per planned Galaxy workflow test (each becomes one tests-format test entry downstream).", + "items": { + "$ref": "#/$defs/TestCase" + } + }, + "unresolved": { + "type": "array", + "description": "Mappings the plan could not resolve and that implement-galaxy-workflow-test (or a reviewer) must settle before authoring the final test file.", + "items": { + "$ref": "#/$defs/UnresolvedItem" + } + }, + "omissions": { + "type": "array", + "description": "Outputs or behaviors deliberately left unasserted, with rationale, so the gap is a recorded decision rather than an oversight.", + "items": { + "$ref": "#/$defs/Omission" + } + }, + "warnings": { + "type": "array", + "description": "Plan-construction warnings: source-test evidence that could not be translated, expression-derived shapes, missing fixtures, and similar.", + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "$defs": { + "SourceRecord": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "name", + "derived_from", + "notes" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "nextflow", + "cwl", + "freeform", + "galaxy" + ], + "description": "Source family of the Galaxy-targeting translator that produced this plan. galaxy denotes a Galaxy→Galaxy update plan carried forward from an existing Galaxy workflow's own tests (the changeset-to-galaxy-test-plan case)." + }, + "name": { + "type": "string", + "description": "Human-oriented workflow or source name the plan was derived from." + }, + "derived_from": { + "type": "string", + "enum": [ + "test-evidence", + "intent" + ], + "description": "Dominant basis for the plan: test-evidence means it was translated from existing upstream test fixtures/snapshots/job files (nf-test, CWL jobs); intent means it was synthesized from workflow intent and scenario-level expected outputs without upstream test evidence (freeform paper/interview sources)." + }, + "notes": { + "type": [ + "string", + "null" + ], + "description": "Free-text provenance about the source and how test evidence was (or was not) available." + } + } + }, + "WorkflowRef": { + "type": "object", + "additionalProperties": false, + "description": "How the plan relates to the Galaxy workflow it tests, and where its input/output labels came from.", + "required": [ + "title", + "label_source", + "notes" + ], + "properties": { + "title": { + "type": [ + "string", + "null" + ], + "description": "Galaxy workflow title/name when known." + }, + "label_source": { + "type": "string", + "enum": [ + "draft", + "interface-brief", + "assumed" + ], + "description": "Provenance of the input/output labels this plan references: draft means they were read from a concrete gxformat2 workflow draft (verifiable); interface-brief means they came from an upstream Galaxy interface brief; assumed means they were inferred and are unconfirmed against any workflow." + }, + "notes": { + "type": [ + "string", + "null" + ], + "description": "Free-text notes about label confidence and workflow-binding assumptions." + } + } + }, + "TestCase": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "doc", + "derived_from", + "provenance", + "job_inputs", + "expected_outputs" + ], + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for this planned test case." + }, + "doc": { + "type": "string", + "description": "What this test exercises, in plain language; becomes the tests-format test `doc`." + }, + "derived_from": { + "type": "string", + "enum": [ + "test-evidence", + "intent", + "mixed" + ], + "description": "Basis for this case: translated from upstream test evidence, synthesized from intent, or a mix." + }, + "provenance": { + "type": [ + "string", + "null" + ], + "description": "Where this case came from, e.g. an nf-test name, a CWL job file, a paper section, or an interview answer." + }, + "job_inputs": { + "type": "array", + "description": "Planned workflow job inputs for this case.", + "items": { + "$ref": "#/$defs/JobInput" + } + }, + "expected_outputs": { + "type": "array", + "description": "Planned workflow outputs to assert for this case.", + "items": { + "$ref": "#/$defs/ExpectedOutput" + } + } + } + }, + "JobInput": { + "type": "object", + "additionalProperties": false, + "required": [ + "workflow_label", + "label_status", + "description", + "collection_shape", + "datatype", + "fixture" + ], + "properties": { + "workflow_label": { + "type": [ + "string", + "null" + ], + "description": "Workflow input label this job entry binds to; null when the label is not yet known." + }, + "label_status": { + "type": "string", + "enum": [ + "resolved", + "assumed", + "unresolved" + ], + "description": "resolved means the label matches a concrete workflow draft input; assumed means it is inferred from a brief; unresolved means the binding label is not yet decided." + }, + "description": { + "type": "string", + "description": "What this input is and the role it plays in the test." + }, + "collection_shape": { + "type": [ + "string", + "null" + ], + "description": "Galaxy collection shape when the input is a collection, e.g. list, paired, list:paired, list:list; null for a single dataset or when unknown." + }, + "datatype": { + "type": [ + "string", + "null" + ], + "description": "Galaxy datatype/extension hint, e.g. fastqsanger.gz, fasta, tabular; null when undecided." + }, + "fixture": { + "$ref": "#/$defs/Fixture" + } + } + }, + "Fixture": { + "type": "object", + "additionalProperties": false, + "required": [ + "storage", + "location", + "checksum", + "provenance" + ], + "properties": { + "storage": { + "type": [ + "string", + "null" + ], + "enum": [ + "remote-url", + "in-repo", + "cvmfs-string", + "generated-toy", + "unresolved", + null + ], + "description": "Where the fixture data lives or should live: remote-url (Zenodo/EBI/SRA location), in-repo (committed test-data path), cvmfs-string (data-table/built-in-index bare string), generated-toy (to be synthesized), or unresolved (not yet sourced)." + }, + "location": { + "type": [ + "string", + "null" + ], + "description": "Concrete URL, repo-relative path, or data-table string when known; null when the fixture still needs resolution." + }, + "checksum": { + "type": [ + "string", + "null" + ], + "description": "Integrity hash when available, e.g. SHA-1:; null when unknown." + }, + "provenance": { + "type": "string", + "description": "Where this fixture comes from or how it was chosen, e.g. a translated source test-data reference, a Zenodo record, or a note that test-data-resolution must supply it." + } + } + }, + "ExpectedOutput": { + "type": "object", + "additionalProperties": false, + "required": [ + "workflow_label", + "label_status", + "description", + "output_kind", + "collection_shape", + "assertion_intent" + ], + "properties": { + "workflow_label": { + "type": [ + "string", + "null" + ], + "description": "Workflow output label to assert against; null when the output label is not yet known." + }, + "label_status": { + "type": "string", + "enum": [ + "resolved", + "assumed", + "unresolved" + ], + "description": "resolved means the label matches a concrete workflow draft output; assumed means it is inferred from a brief; unresolved means the output label is not yet decided." + }, + "description": { + "type": "string", + "description": "What this output is and why it is worth asserting." + }, + "output_kind": { + "type": [ + "string", + "null" + ], + "enum": [ + "dataset", + "collection", + null + ], + "description": "Whether the output is a single dataset or a collection; null when undecided. Collection outputs carry element-level assertion intent via assertion_intent[].element_identifier." + }, + "collection_shape": { + "type": [ + "string", + "null" + ], + "description": "Galaxy collection shape for collection outputs, e.g. list, list:paired, list:list; null otherwise." + }, + "assertion_intent": { + "type": "array", + "description": "Assertion intent for this output. May be empty when the output is intentionally unasserted (record the reason in omissions). Each entry names a tests-format assertion family by name rather than duplicating its full parameter object.", + "items": { + "$ref": "#/$defs/AssertionIntent" + } + } + } + }, + "AssertionIntent": { + "type": "object", + "additionalProperties": false, + "required": [ + "family", + "intent", + "expected_value", + "tolerance", + "element_identifier", + "evidence", + "confidence" + ], + "properties": { + "family": { + "type": "string", + "description": "tests-format assertion family, compare operator, or output-attribute field this intent maps to, by name. Assertion families go inside the output's `asserts:`, e.g. has_text, has_n_lines, has_n_columns, has_size, has_image_width, has_h5_keys. Compare operators and output-attribute fields sit beside `asserts:`: file, compare:diff, compare:sim_size, element_count (collection outputs), count (nested sub-collections), ftype, checksum, metadata. Do not use collection_type (state it in ExpectedOutput.collection_shape) or element_tests (a non-null element_identifier implies it). Names reference the tests-format schema vocabulary; this plan does not restate parameter shapes." + }, + "intent": { + "type": "string", + "description": "What this assertion verifies, in plain language, so implement-galaxy-workflow-test can materialize the concrete assertion without re-reading the source tests." + }, + "expected_value": { + "type": [ + "string", + "number", + "integer", + "boolean", + "null" + ], + "description": "Concrete expected token, count, size, or comparison fixture when known; null when synthesized from intent without a concrete value." + }, + "tolerance": { + "anyOf": [ + { + "$ref": "#/$defs/Tolerance" + }, + { + "type": "null" + } + ], + "description": "Tolerance for the expected value when applicable; null for exact or existence-only assertions." + }, + "element_identifier": { + "type": [ + "string", + "null" + ], + "description": "Collection element identifier this assertion targets, for element-level (element_tests) assertions; null for top-level dataset assertions. Address an element of a nested collection by joining identifiers outermost first with `/`, e.g. WT_REP1/forward in a list:paired." + }, + "evidence": { + "type": "string", + "enum": [ + "test-evidence", + "fixture-derived", + "intent" + ], + "description": "Basis of the expected value: test-evidence means translated from an upstream assertion, checksum, or expected file; fixture-derived means measured from upstream test fixture bytes where no upstream assertion exists (e.g. snapshot-only nf-tests); intent means synthesized from workflow intent, wrapper XML, or step state with no upstream anchor." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ], + "description": "How firmly the expected value and family are believed correct; synthesized assertions are typically medium or low." + } + } + }, + "Tolerance": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "magnitude", + "rationale" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "delta", + "delta_frac", + "lines_diff", + "none" + ], + "description": "Tolerance operator: delta (absolute count/bytes, suffixes like 1K/1M allowed as strings), delta_frac (fraction such as 0.1), lines_diff (number of mutable header lines for compare:diff), or none." + }, + "magnitude": { + "type": [ + "string", + "number", + "integer", + "null" + ], + "description": "Tolerance magnitude, e.g. 6, \"10K\", or 0.1; null when kind is none or the magnitude is undecided." + }, + "rationale": { + "type": [ + "string", + "null" + ], + "description": "Why this magnitude was chosen, e.g. mutable VCF header line count, percentage of typical file size, or nondeterministic output." + } + } + }, + "UnresolvedItem": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "description", + "blocking", + "suggested_resolution" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "input-label", + "output-label", + "fixture", + "assertion", + "datatype", + "collection-shape", + "other" + ], + "description": "Category of the unresolved mapping." + }, + "description": { + "type": "string", + "description": "What could not be resolved and why." + }, + "blocking": { + "type": "boolean", + "description": "Whether implement-galaxy-workflow-test must resolve this before authoring a runnable test." + }, + "suggested_resolution": { + "type": [ + "string", + "null" + ], + "description": "A proposed way to resolve it when one is apparent; null otherwise." + } + } + }, + "Omission": { + "type": "object", + "additionalProperties": false, + "required": [ + "target", + "reason", + "category" + ], + "properties": { + "target": { + "type": "string", + "description": "What is intentionally left unasserted, e.g. a workflow output label or a class of outputs." + }, + "reason": { + "type": "string", + "description": "Why it is omitted, e.g. nondeterministic content, no stable checkpoint, or asserted only weakly elsewhere." + }, + "category": { + "type": [ + "string", + "null" + ], + "enum": [ + "nondeterministic", + "no-stable-checkpoint", + "weak-output", + "out-of-scope", + "other", + null + ], + "description": "Coarse omission category; null when none fits." + } + } + }, + "Warning": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "message", + "path" + ], + "properties": { + "code": { + "type": "string", + "description": "Stable warning code, e.g. untranslatable-snapshot, missing-fixture, expression-derived-shape." + }, + "message": { + "type": "string", + "description": "Human-readable warning detail." + }, + "path": { + "type": [ + "string", + "null" + ], + "description": "Pointer into the source or plan the warning refers to; null when not applicable." + } + } + } + } +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/tests-format.schema.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/tests-format.schema.json new file mode 100644 index 0000000..99d551f --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/tests-format.schema.json @@ -0,0 +1,7390 @@ +{ + "$defs": { + "Collection": { + "additionalProperties": false, + "properties": { + "class": { + "const": "Collection", + "title": "Class", + "type": "string" + }, + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Collection Type" + }, + "elements": { + "anyOf": [ + { + "items": { + "oneOf": [ + { + "oneOf": [ + { + "$ref": "#/$defs/LocationFile" + }, + { + "$ref": "#/$defs/PathFile" + }, + { + "$ref": "#/$defs/ContentsFile" + }, + { + "$ref": "#/$defs/CompositeDataFile" + } + ] + }, + { + "$ref": "#/$defs/Collection" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Elements" + }, + "identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Identifier" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "rows": { + "anyOf": [ + { + "additionalProperties": { + "items": {}, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rows" + } + }, + "required": [ + "class" + ], + "title": "Collection", + "type": "object" + }, + "CollectionAttributes": { + "additionalProperties": false, + "properties": { + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Collection Type" + } + }, + "title": "CollectionAttributes", + "type": "object" + }, + "CompositeDataFile": { + "additionalProperties": false, + "properties": { + "class": { + "const": "File", + "title": "Class", + "type": "string" + }, + "composite_data": { + "items": { + "type": "string" + }, + "title": "Composite Data", + "type": "array" + }, + "contents": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Contents" + }, + "dbkey": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dbkey" + }, + "decompress": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Decompress" + }, + "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Deferred" + }, + "filetype": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "File Type" + }, + "hashes": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/HashEntry" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hashes" + }, + "identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Identifier" + }, + "info": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Info" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "space_to_tab": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Space To Tab" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + }, + "to_posix_lines": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "To POSIX Lines" + } + }, + "required": [ + "class", + "composite_data" + ], + "title": "CompositeDataFile", + "type": "object" + }, + "ContentsFile": { + "additionalProperties": false, + "description": "CWL File literal — content inlined as a string.", + "properties": { + "class": { + "const": "File", + "title": "Class", + "type": "string" + }, + "composite_data": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Composite Data" + }, + "contents": { + "title": "Contents", + "type": "string" + }, + "dbkey": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dbkey" + }, + "decompress": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Decompress" + }, + "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Deferred" + }, + "filetype": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "File Type" + }, + "hashes": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/HashEntry" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hashes" + }, + "identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Identifier" + }, + "info": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Info" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "space_to_tab": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Space To Tab" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + }, + "to_posix_lines": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "To POSIX Lines" + } + }, + "required": [ + "class", + "contents" + ], + "title": "ContentsFile", + "type": "object" + }, + "Directory": { + "additionalProperties": false, + "description": "CWL-style directory input. Supported by stage_inputs for directory-typed\ndatasets (e.g. bwa_mem2_index test fixtures). Rare in workflow tests; IWC\ndoes not use it.", + "properties": { + "class": { + "const": "Directory", + "title": "Class", + "type": "string" + }, + "filetype": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "File Type" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + } + }, + "required": [ + "class" + ], + "title": "Directory", + "type": "object" + }, + "HashEntry": { + "additionalProperties": false, + "properties": { + "hash_function": { + "enum": [ + "MD5", + "SHA-1", + "SHA-256", + "SHA-512" + ], + "title": "Hash Function", + "type": "string" + }, + "hash_value": { + "title": "Hash Value", + "type": "string" + } + }, + "required": [ + "hash_function", + "hash_value" + ], + "title": "HashEntry", + "type": "object" + }, + "Job": { + "additionalProperties": { + "anyOf": [ + { + "oneOf": [ + { + "$ref": "#/$defs/LocationFile" + }, + { + "$ref": "#/$defs/PathFile" + }, + { + "$ref": "#/$defs/ContentsFile" + }, + { + "$ref": "#/$defs/CompositeDataFile" + } + ] + }, + { + "$ref": "#/$defs/Collection" + }, + { + "$ref": "#/$defs/Directory" + }, + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "items": { + "anyOf": [ + { + "oneOf": [ + { + "$ref": "#/$defs/LocationFile" + }, + { + "$ref": "#/$defs/PathFile" + }, + { + "$ref": "#/$defs/ContentsFile" + }, + { + "$ref": "#/$defs/CompositeDataFile" + } + ] + }, + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "title": "Job", + "type": "object" + }, + "LocationFile": { + "additionalProperties": false, + "properties": { + "class": { + "const": "File", + "title": "Class", + "type": "string" + }, + "composite_data": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Composite Data" + }, + "contents": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Contents" + }, + "dbkey": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dbkey" + }, + "decompress": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Decompress" + }, + "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Deferred" + }, + "filetype": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "File Type" + }, + "hashes": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/HashEntry" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hashes" + }, + "identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Identifier" + }, + "info": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Info" + }, + "location": { + "title": "Location", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "space_to_tab": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Space To Tab" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + }, + "to_posix_lines": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "To POSIX Lines" + } + }, + "required": [ + "class", + "location" + ], + "title": "LocationFile", + "type": "object" + }, + "OutputCompareType": { + "enum": [ + "diff", + "re_match", + "sim_size", + "re_match_multiline", + "contains", + "image_diff" + ], + "title": "OutputCompareType", + "type": "string" + }, + "PathFile": { + "additionalProperties": false, + "properties": { + "class": { + "const": "File", + "title": "Class", + "type": "string" + }, + "composite_data": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Composite Data" + }, + "contents": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Contents" + }, + "dbkey": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dbkey" + }, + "decompress": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Decompress" + }, + "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Deferred" + }, + "filetype": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "File Type" + }, + "hashes": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/HashEntry" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hashes" + }, + "identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Identifier" + }, + "info": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Info" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "path": { + "title": "Path", + "type": "string" + }, + "space_to_tab": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Space To Tab" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + }, + "to_posix_lines": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "To POSIX Lines" + } + }, + "required": [ + "class", + "path" + ], + "title": "PathFile", + "type": "object" + }, + "TestCollectionCollectionElementAssertions": { + "additionalProperties": false, + "properties": { + "class": { + "anyOf": [ + { + "const": "Collection", + "type": "string" + }, + { + "type": "null" + } + ], + "default": "Collection", + "title": "Class" + }, + "count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Count" + }, + "element_tests": { + "anyOf": [ + { + "additionalProperties": { + "else": { + "$ref": "#/$defs/TestCollectionDatasetElementAssertions" + }, + "if": { + "properties": { + "class": { + "const": "Collection" + } + }, + "required": [ + "class" + ], + "type": "object" + }, + "then": { + "$ref": "#/$defs/TestCollectionCollectionElementAssertions" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Element Tests" + }, + "elements": { + "anyOf": [ + { + "additionalProperties": { + "else": { + "$ref": "#/$defs/TestCollectionDatasetElementAssertions" + }, + "if": { + "properties": { + "class": { + "const": "Collection" + } + }, + "required": [ + "class" + ], + "type": "object" + }, + "then": { + "$ref": "#/$defs/TestCollectionCollectionElementAssertions" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Elements" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Maximum Count" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimum Count" + } + }, + "title": "TestCollectionCollectionElementAssertions", + "type": "object" + }, + "TestCollectionDatasetElementAssertions": { + "additionalProperties": false, + "properties": { + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "$ref": "#/$defs/assertion_dict" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Assertions about the content of the output.", + "title": "Asserts" + }, + "checksum": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The target output's checksum should match the value specified here, in the form `hash_type$hash_value` (e.g. `sha1$8156d7ca0f46ed7abac98f82e36cfaddb2aca041`). Useful for large static files where uploading the whole file is inconvenient.", + "title": "Checksum" + }, + "class": { + "anyOf": [ + { + "const": "File", + "type": "string" + }, + { + "type": "null" + } + ], + "default": "File", + "title": "Class" + }, + "compare": { + "anyOf": [ + { + "$ref": "#/$defs/OutputCompareType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Comparison mode used when matching the output against the reference file.", + "title": "Compare" + }, + "decompress": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If true, decompress files before comparison. Applies to assertions expressed with `assert_contents` or `compare` set to anything but `sim_size`. Useful for testing compressed outputs that are non-deterministic despite having deterministic decompressed contents. By default, only files compressed with bz2, gzip and zip are automatically decompressed.", + "title": "Decompress" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If `compare` is set to `sim_size`, the maximum allowed absolute size difference (in bytes) between the generated data set and the reference file in `test-data/`. Default is 10000 bytes. Can be combined with `delta_frac`.", + "title": "Delta" + }, + "delta_frac": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If `compare` is set to `sim_size`, the maximum allowed relative size difference between the generated data set and the reference file in `test-data/`. 0.1 means the generated file can differ by at most 10%. Default is not to check for relative size difference. Can be combined with `delta`.", + "title": "Delta Frac" + }, + "file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the output file stored in the target `test-data` directory that will be used to compare against the results of executing the tool via the functional test framework.", + "title": "File" + }, + "ftype": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against the corresponding output's data type. If these do not match, the test will fail.", + "title": "File Type" + }, + "lines_diff": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Applies when `compare` is set to `diff`, `re_match`, or `contains`. For `diff`, the number of lines of difference to allow (a modified line counts as two: one added, one removed).", + "title": "Lines Diff" + }, + "location": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL that points to a remote output file that will be downloaded and used for output comparison. Use only when the file cannot be included in the `test-data` folder. May be combined with `file` (downloads when missing on disk) or used alone (filename inferred from the URL). A `checksum` is also used to verify the download when provided.", + "title": "Location" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mapping of metadata keys to expected values for this output.", + "title": "Metadata" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filesystem path to a local output file used for comparison.", + "title": "Path" + }, + "sort": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", + "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" + } + }, + "title": "TestCollectionDatasetElementAssertions", + "type": "object" + }, + "TestCollectionOutputAssertions": { + "additionalProperties": false, + "properties": { + "attributes": { + "anyOf": [ + { + "$ref": "#/$defs/CollectionAttributes" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Attributes" + }, + "class": { + "anyOf": [ + { + "const": "Collection", + "type": "string" + }, + { + "type": "null" + } + ], + "default": "Collection", + "title": "Class" + }, + "collection_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Collection Type" + }, + "element_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Element Count" + }, + "element_tests": { + "anyOf": [ + { + "additionalProperties": { + "else": { + "$ref": "#/$defs/TestCollectionDatasetElementAssertions" + }, + "if": { + "properties": { + "class": { + "const": "Collection" + } + }, + "required": [ + "class" + ], + "type": "object" + }, + "then": { + "$ref": "#/$defs/TestCollectionCollectionElementAssertions" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Element Tests" + }, + "elements": { + "anyOf": [ + { + "additionalProperties": { + "else": { + "$ref": "#/$defs/TestCollectionDatasetElementAssertions" + }, + "if": { + "properties": { + "class": { + "const": "Collection" + } + }, + "required": [ + "class" + ], + "type": "object" + }, + "then": { + "$ref": "#/$defs/TestCollectionCollectionElementAssertions" + } + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Elements" + } + }, + "title": "TestCollectionOutputAssertions", + "type": "object" + }, + "TestDataOutputAssertions": { + "additionalProperties": false, + "properties": { + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "$ref": "#/$defs/assertion_dict" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Assertions about the content of the output.", + "title": "Asserts" + }, + "checksum": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The target output's checksum should match the value specified here, in the form `hash_type$hash_value` (e.g. `sha1$8156d7ca0f46ed7abac98f82e36cfaddb2aca041`). Useful for large static files where uploading the whole file is inconvenient.", + "title": "Checksum" + }, + "class": { + "anyOf": [ + { + "const": "File", + "type": "string" + }, + { + "type": "null" + } + ], + "default": "File", + "title": "Class" + }, + "compare": { + "anyOf": [ + { + "$ref": "#/$defs/OutputCompareType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Comparison mode used when matching the output against the reference file.", + "title": "Compare" + }, + "decompress": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If true, decompress files before comparison. Applies to assertions expressed with `assert_contents` or `compare` set to anything but `sim_size`. Useful for testing compressed outputs that are non-deterministic despite having deterministic decompressed contents. By default, only files compressed with bz2, gzip and zip are automatically decompressed.", + "title": "Decompress" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If `compare` is set to `sim_size`, the maximum allowed absolute size difference (in bytes) between the generated data set and the reference file in `test-data/`. Default is 10000 bytes. Can be combined with `delta_frac`.", + "title": "Delta" + }, + "delta_frac": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If `compare` is set to `sim_size`, the maximum allowed relative size difference between the generated data set and the reference file in `test-data/`. 0.1 means the generated file can differ by at most 10%. Default is not to check for relative size difference. Can be combined with `delta`.", + "title": "Delta Frac" + }, + "file": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the output file stored in the target `test-data` directory that will be used to compare against the results of executing the tool via the functional test framework.", + "title": "File" + }, + "ftype": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against the corresponding output's data type. If these do not match, the test will fail.", + "title": "File Type" + }, + "lines_diff": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Applies when `compare` is set to `diff`, `re_match`, or `contains`. For `diff`, the number of lines of difference to allow (a modified line counts as two: one added, one removed).", + "title": "Lines Diff" + }, + "location": { + "anyOf": [ + { + "format": "uri", + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL that points to a remote output file that will be downloaded and used for output comparison. Use only when the file cannot be included in the `test-data` folder. May be combined with `file` (downloads when missing on disk) or used alone (filename inferred from the URL). A `checksum` is also used to verify the download when provided.", + "title": "Location" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mapping of metadata keys to expected values for this output.", + "title": "Metadata" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filesystem path to a local output file used for comparison.", + "title": "Path" + }, + "sort": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Applies only if `compare` is `diff`, `re_match` or `re_match_multiline`. Sorts the lines of the history data set before comparison; for `diff` and `re_match` the local file is also sorted. Useful for non-deterministic output.", + "title": "Sort" + }, + "visible": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, this value is checked against whether the corresponding output is shown in the history. Use to test outputs a tool or workflow is expected to hide, for instance via a `HideDatasetAction` post job action.", + "title": "Visible" + } + }, + "title": "TestDataOutputAssertions", + "type": "object" + }, + "TestJob": { + "additionalProperties": false, + "properties": { + "doc": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Describes the purpose of the test.", + "title": "Doc" + }, + "expect_failure": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If true, the workflow is expected to produce an error.", + "title": "Expect Failure" + }, + "job": { + "$ref": "#/$defs/Job", + "description": "Defines the job to execute. Can be a path to a file or an inline dictionary describing the job inputs." + }, + "outputs": { + "additionalProperties": { + "else": { + "else": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "if": { + "type": "object" + }, + "then": { + "$ref": "#/$defs/TestDataOutputAssertions" + } + }, + "if": { + "properties": { + "class": { + "const": "Collection" + } + }, + "required": [ + "class" + ], + "type": "object" + }, + "then": { + "$ref": "#/$defs/TestCollectionOutputAssertions" + } + }, + "description": "Defines assertions about outputs (datasets, collections or parameters). Each key corresponds to a labeled output; values are dictionaries describing the expected output.", + "title": "Outputs", + "type": "object" + } + }, + "required": [ + "job", + "outputs" + ], + "title": "TestJob", + "type": "object" + }, + "assertion_dict": { + "additionalProperties": false, + "properties": { + "attribute_is": { + "anyOf": [ + { + "$ref": "#/$defs/base_attribute_is_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Attribute Is" + }, + "attribute_matches": { + "anyOf": [ + { + "$ref": "#/$defs/base_attribute_matches_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Attribute Matches" + }, + "element_text": { + "anyOf": [ + { + "$ref": "#/$defs/base_element_text_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Element Text" + }, + "element_text_is": { + "anyOf": [ + { + "$ref": "#/$defs/base_element_text_is_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Element Text Is" + }, + "element_text_matches": { + "anyOf": [ + { + "$ref": "#/$defs/base_element_text_matches_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Element Text Matches" + }, + "has_archive_member": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_archive_member_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Archive Member" + }, + "has_element_with_path": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_element_with_path_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Element With Path" + }, + "has_h5_attribute": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_h5_attribute_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has H5 Attribute" + }, + "has_h5_keys": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_h5_keys_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has H5 Keys" + }, + "has_image_center_of_mass": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_center_of_mass_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Center Of Mass" + }, + "has_image_channels": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_channels_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Channels" + }, + "has_image_depth": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_depth_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Depth" + }, + "has_image_frames": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_frames_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Frames" + }, + "has_image_height": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_height_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Height" + }, + "has_image_mean_intensity": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_mean_intensity_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Mean Intensity" + }, + "has_image_mean_object_size": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_mean_object_size_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Mean Object Size" + }, + "has_image_n_labels": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_n_labels_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image N Labels" + }, + "has_image_width": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_image_width_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Image Width" + }, + "has_json_property_with_text": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_json_property_with_text_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Json Property With Text" + }, + "has_json_property_with_value": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_json_property_with_value_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Json Property With Value" + }, + "has_line": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_line_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Line" + }, + "has_line_matching": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_line_matching_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Line Matching" + }, + "has_n_columns": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_n_columns_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has N Columns" + }, + "has_n_elements_with_path": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_n_elements_with_path_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has N Elements With Path" + }, + "has_n_lines": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_n_lines_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has N Lines" + }, + "has_size": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_size_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Size" + }, + "has_text": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_text_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Text" + }, + "has_text_matching": { + "anyOf": [ + { + "$ref": "#/$defs/base_has_text_matching_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Has Text Matching" + }, + "is_valid_xml": { + "anyOf": [ + { + "$ref": "#/$defs/base_is_valid_xml_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Is Valid Xml" + }, + "not_has_text": { + "anyOf": [ + { + "$ref": "#/$defs/base_not_has_text_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Not Has Text" + }, + "xml_element": { + "anyOf": [ + { + "$ref": "#/$defs/base_xml_element_model" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assert Xml Element" + } + }, + "title": "assertion_dict", + "type": "object" + }, + "assertion_list": { + "items": { + "anyOf": [ + { + "discriminator": { + "mapping": { + "attribute_is": "#/$defs/attribute_is_model", + "attribute_matches": "#/$defs/attribute_matches_model", + "element_text": "#/$defs/element_text_model", + "element_text_is": "#/$defs/element_text_is_model", + "element_text_matches": "#/$defs/element_text_matches_model", + "has_archive_member": "#/$defs/has_archive_member_model", + "has_element_with_path": "#/$defs/has_element_with_path_model", + "has_h5_attribute": "#/$defs/has_h5_attribute_model", + "has_h5_keys": "#/$defs/has_h5_keys_model", + "has_image_center_of_mass": "#/$defs/has_image_center_of_mass_model", + "has_image_channels": "#/$defs/has_image_channels_model", + "has_image_depth": "#/$defs/has_image_depth_model", + "has_image_frames": "#/$defs/has_image_frames_model", + "has_image_height": "#/$defs/has_image_height_model", + "has_image_mean_intensity": "#/$defs/has_image_mean_intensity_model", + "has_image_mean_object_size": "#/$defs/has_image_mean_object_size_model", + "has_image_n_labels": "#/$defs/has_image_n_labels_model", + "has_image_width": "#/$defs/has_image_width_model", + "has_json_property_with_text": "#/$defs/has_json_property_with_text_model", + "has_json_property_with_value": "#/$defs/has_json_property_with_value_model", + "has_line": "#/$defs/has_line_model", + "has_line_matching": "#/$defs/has_line_matching_model", + "has_n_columns": "#/$defs/has_n_columns_model", + "has_n_elements_with_path": "#/$defs/has_n_elements_with_path_model", + "has_n_lines": "#/$defs/has_n_lines_model", + "has_size": "#/$defs/has_size_model", + "has_text": "#/$defs/has_text_model", + "has_text_matching": "#/$defs/has_text_matching_model", + "is_valid_xml": "#/$defs/is_valid_xml_model", + "not_has_text": "#/$defs/not_has_text_model", + "xml_element": "#/$defs/xml_element_model" + }, + "propertyName": "that" + }, + "oneOf": [ + { + "$ref": "#/$defs/has_line_model" + }, + { + "$ref": "#/$defs/has_line_matching_model" + }, + { + "$ref": "#/$defs/has_n_lines_model" + }, + { + "$ref": "#/$defs/has_text_model" + }, + { + "$ref": "#/$defs/has_text_matching_model" + }, + { + "$ref": "#/$defs/not_has_text_model" + }, + { + "$ref": "#/$defs/has_n_columns_model" + }, + { + "$ref": "#/$defs/attribute_is_model" + }, + { + "$ref": "#/$defs/attribute_matches_model" + }, + { + "$ref": "#/$defs/element_text_model" + }, + { + "$ref": "#/$defs/element_text_is_model" + }, + { + "$ref": "#/$defs/element_text_matches_model" + }, + { + "$ref": "#/$defs/has_element_with_path_model" + }, + { + "$ref": "#/$defs/has_n_elements_with_path_model" + }, + { + "$ref": "#/$defs/is_valid_xml_model" + }, + { + "$ref": "#/$defs/xml_element_model" + }, + { + "$ref": "#/$defs/has_json_property_with_text_model" + }, + { + "$ref": "#/$defs/has_json_property_with_value_model" + }, + { + "$ref": "#/$defs/has_h5_attribute_model" + }, + { + "$ref": "#/$defs/has_h5_keys_model" + }, + { + "$ref": "#/$defs/has_archive_member_model" + }, + { + "$ref": "#/$defs/has_size_model" + }, + { + "$ref": "#/$defs/has_image_center_of_mass_model" + }, + { + "$ref": "#/$defs/has_image_channels_model" + }, + { + "$ref": "#/$defs/has_image_depth_model" + }, + { + "$ref": "#/$defs/has_image_frames_model" + }, + { + "$ref": "#/$defs/has_image_height_model" + }, + { + "$ref": "#/$defs/has_image_mean_intensity_model" + }, + { + "$ref": "#/$defs/has_image_mean_object_size_model" + }, + { + "$ref": "#/$defs/has_image_n_labels_model" + }, + { + "$ref": "#/$defs/has_image_width_model" + } + ] + }, + { + "$ref": "#/$defs/has_line_model_nested" + }, + { + "$ref": "#/$defs/has_line_matching_model_nested" + }, + { + "$ref": "#/$defs/has_n_lines_model_nested" + }, + { + "$ref": "#/$defs/has_text_model_nested" + }, + { + "$ref": "#/$defs/has_text_matching_model_nested" + }, + { + "$ref": "#/$defs/not_has_text_model_nested" + }, + { + "$ref": "#/$defs/has_n_columns_model_nested" + }, + { + "$ref": "#/$defs/attribute_is_model_nested" + }, + { + "$ref": "#/$defs/attribute_matches_model_nested" + }, + { + "$ref": "#/$defs/element_text_model_nested" + }, + { + "$ref": "#/$defs/element_text_is_model_nested" + }, + { + "$ref": "#/$defs/element_text_matches_model_nested" + }, + { + "$ref": "#/$defs/has_element_with_path_model_nested" + }, + { + "$ref": "#/$defs/has_n_elements_with_path_model_nested" + }, + { + "$ref": "#/$defs/is_valid_xml_model_nested" + }, + { + "$ref": "#/$defs/xml_element_model_nested" + }, + { + "$ref": "#/$defs/has_json_property_with_text_model_nested" + }, + { + "$ref": "#/$defs/has_json_property_with_value_model_nested" + }, + { + "$ref": "#/$defs/has_h5_attribute_model_nested" + }, + { + "$ref": "#/$defs/has_h5_keys_model_nested" + }, + { + "$ref": "#/$defs/has_archive_member_model_nested" + }, + { + "$ref": "#/$defs/has_size_model_nested" + }, + { + "$ref": "#/$defs/has_image_center_of_mass_model_nested" + }, + { + "$ref": "#/$defs/has_image_channels_model_nested" + }, + { + "$ref": "#/$defs/has_image_depth_model_nested" + }, + { + "$ref": "#/$defs/has_image_frames_model_nested" + }, + { + "$ref": "#/$defs/has_image_height_model_nested" + }, + { + "$ref": "#/$defs/has_image_mean_intensity_model_nested" + }, + { + "$ref": "#/$defs/has_image_mean_object_size_model_nested" + }, + { + "$ref": "#/$defs/has_image_n_labels_model_nested" + }, + { + "$ref": "#/$defs/has_image_width_model_nested" + } + ] + }, + "title": "assertion_list", + "type": "array" + }, + "attribute_is_model": { + "additionalProperties": false, + "description": "Asserts the XML ``attribute`` for the element (or tag) with the specified\nXPath-like ``path`` is the specified ``text``.\n\nFor example:\n\n```xml\n\n```\n\nThe assertion implicitly also asserts that an element matching ``path`` exists.\nWith ``negate`` the result of the assertion (on the equality) can be inverted (the\nimplicit assertion on the existence of the path is not affected).", + "properties": { + "attribute": { + "description": "The XML attribute name to test against from the target XML element.", + "title": "Attribute", + "type": "string" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "text": { + "description": "The expected attribute value to test against on the target XML element", + "title": "Text", + "type": "string" + }, + "that": { + "const": "attribute_is", + "default": "attribute_is", + "title": "That", + "type": "string" + } + }, + "required": [ + "path", + "attribute", + "text" + ], + "title": "Assert Attribute Is", + "type": "object" + }, + "attribute_is_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "attribute_is": { + "$ref": "#/$defs/base_attribute_is_model", + "title": "Assert Attribute Is" + } + }, + "required": [ + "attribute_is" + ], + "title": "Assert Attribute Is (Nested)", + "type": "object" + }, + "attribute_matches_model": { + "additionalProperties": false, + "description": "Asserts the XML ``attribute`` for the element (or tag) with the specified\nXPath-like ``path`` matches the regular expression specified by ``expression``.\n\nFor example:\n\n```xml\n\n```\n\nThe assertion implicitly also asserts that an element matching ``path`` exists.\nWith ``negate`` the result of the assertion (on the matching) can be inverted (the\nimplicit assertion on the existence of the path is not affected).", + "properties": { + "attribute": { + "description": "The XML attribute name to test against from the target XML element.", + "title": "Attribute", + "type": "string" + }, + "expression": { + "description": "The regular expressions to apply against the named attribute on the target XML element.", + "title": "Expression", + "type": "string" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "that": { + "const": "attribute_matches", + "default": "attribute_matches", + "title": "That", + "type": "string" + } + }, + "required": [ + "path", + "attribute", + "expression" + ], + "title": "Assert Attribute Matches", + "type": "object" + }, + "attribute_matches_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "attribute_matches": { + "$ref": "#/$defs/base_attribute_matches_model", + "title": "Assert Attribute Matches" + } + }, + "required": [ + "attribute_matches" + ], + "title": "Assert Attribute Matches (Nested)", + "type": "object" + }, + "base_attribute_is_model": { + "additionalProperties": false, + "description": "base model for attribute_is describing attributes.", + "properties": { + "attribute": { + "description": "The XML attribute name to test against from the target XML element.", + "title": "Attribute", + "type": "string" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "text": { + "description": "The expected attribute value to test against on the target XML element", + "title": "Text", + "type": "string" + } + }, + "required": [ + "path", + "attribute", + "text" + ], + "title": "base_attribute_is_model", + "type": "object" + }, + "base_attribute_matches_model": { + "additionalProperties": false, + "description": "base model for attribute_matches describing attributes.", + "properties": { + "attribute": { + "description": "The XML attribute name to test against from the target XML element.", + "title": "Attribute", + "type": "string" + }, + "expression": { + "description": "The regular expressions to apply against the named attribute on the target XML element.", + "title": "Expression", + "type": "string" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + } + }, + "required": [ + "path", + "attribute", + "expression" + ], + "title": "base_attribute_matches_model", + "type": "object" + }, + "base_element_text_is_model": { + "additionalProperties": false, + "description": "base model for element_text_is describing attributes.", + "properties": { + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "text": { + "description": "The expected element text (body of the XML tag) to test against on the target XML element", + "title": "Text", + "type": "string" + } + }, + "required": [ + "path", + "text" + ], + "title": "base_element_text_is_model", + "type": "object" + }, + "base_element_text_matches_model": { + "additionalProperties": false, + "description": "base model for element_text_matches describing attributes.", + "properties": { + "expression": { + "description": "The regular expressions to apply against the target element.", + "title": "Expression", + "type": "string" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + } + }, + "required": [ + "path", + "expression" + ], + "title": "base_element_text_matches_model", + "type": "object" + }, + "base_element_text_model": { + "additionalProperties": false, + "description": "base model for element_text describing attributes.", + "properties": { + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Asserts" + }, + "children": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Children" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "base_element_text_model", + "type": "object" + }, + "base_has_archive_member_model": { + "additionalProperties": false, + "description": "base model for has_archive_member describing attributes.", + "properties": { + "all": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first", + "title": "All" + }, + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Asserts" + }, + "children": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Children" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The regular expression specifying the archive member.", + "title": "Path", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "base_has_archive_member_model", + "type": "object" + }, + "base_has_element_with_path_model": { + "additionalProperties": false, + "description": "base model for has_element_with_path describing attributes.", + "properties": { + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "base_has_element_with_path_model", + "type": "object" + }, + "base_has_h5_attribute_model": { + "additionalProperties": false, + "description": "base model for has_h5_attribute describing attributes.", + "properties": { + "key": { + "description": "HDF5 attribute to check value of.", + "title": "Key", + "type": "string" + }, + "value": { + "description": "Expected value of HDF5 attribute to check.", + "title": "Value", + "type": "string" + } + }, + "required": [ + "key", + "value" + ], + "title": "base_has_h5_attribute_model", + "type": "object" + }, + "base_has_h5_keys_model": { + "additionalProperties": false, + "description": "base model for has_h5_keys describing attributes.", + "properties": { + "keys": { + "description": "HDF5 attributes to check value of as a comma-separated string.", + "title": "Keys", + "type": "string" + } + }, + "required": [ + "keys" + ], + "title": "base_has_h5_keys_model", + "type": "object" + }, + "base_has_image_center_of_mass_model": { + "additionalProperties": false, + "description": "base model for has_image_center_of_mass describing attributes.", + "properties": { + "center_of_mass": { + "description": "The required center of mass of the image intensities (horizontal and vertical coordinate, separated by a comma).", + "title": "Center Of Mass", + "type": "string" + }, + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "eps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "default": 0.01, + "description": "The maximum allowed Euclidean distance to the required center of mass (defaults to ``0.01``).", + "title": "Eps" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + } + }, + "required": [ + "center_of_mass" + ], + "title": "base_has_image_center_of_mass_model", + "type": "object" + }, + "base_has_image_channels_model": { + "additionalProperties": false, + "description": "base model for has_image_channels describing attributes.", + "properties": { + "channels": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected number of channels of the image.", + "title": "Channels" + }, + "delta": { + "default": 0, + "description": "Maximum allowed difference of the number of channels (default is 0). The observed number of channels has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed number of channels.", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed number of channels.", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "title": "base_has_image_channels_model", + "type": "object" + }, + "base_has_image_depth_model": { + "additionalProperties": false, + "description": "base model for has_image_depth describing attributes.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the image depth (number of slices, default is 0). The observed depth has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "depth": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected depth of the image (number of slices).", + "title": "Depth" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed depth of the image (number of slices).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed depth of the image (number of slices).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "title": "base_has_image_depth_model", + "type": "object" + }, + "base_has_image_frames_model": { + "additionalProperties": false, + "description": "base model for has_image_frames describing attributes.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the number of frames in the image sequence (number of time steps, default is 0). The observed number of frames has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "frames": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected number of frames in the image sequence (number of time steps).", + "title": "Frames" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed number of frames in the image sequence (number of time steps).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed number of frames in the image sequence (number of time steps).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "title": "base_has_image_frames_model", + "type": "object" + }, + "base_has_image_height_model": { + "additionalProperties": false, + "description": "base model for has_image_height describing attributes.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the image height (in pixels, default is 0). The observed height has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "height": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected height of the image (in pixels).", + "title": "Height" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed height of the image (in pixels).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed height of the image (in pixels).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "title": "base_has_image_height_model", + "type": "object" + }, + "base_has_image_mean_intensity_model": { + "additionalProperties": false, + "description": "base model for has_image_mean_intensity describing attributes.", + "properties": { + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "eps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "default": 0.01, + "description": "The absolute tolerance to be used for ``value`` (defaults to ``0.01``). The observed mean value of the image intensities has to be in the range ``value +- eps``.", + "title": "Eps" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An upper bound of the required mean value of the image intensities.", + "title": "Max" + }, + "mean_intensity": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The required mean value of the image intensities.", + "title": "Mean Intensity" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A lower bound of the required mean value of the image intensities.", + "title": "Min" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + } + }, + "title": "base_has_image_mean_intensity_model", + "type": "object" + }, + "base_has_image_mean_object_size_model": { + "additionalProperties": false, + "description": "base model for has_image_mean_object_size describing attributes.", + "properties": { + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "eps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "default": 0.01, + "description": "The absolute tolerance to be used for ``value`` (defaults to ``0.01``). The observed mean size of the uniquely labeled objects has to be in the range ``value +- eps``.", + "title": "Eps" + }, + "exclude_labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels to be excluded from consideration, separated by a comma. The primary usage of this attribute is to exclude the background of a label image. Cannot be used in combination with ``labels``.", + "title": "Exclude Labels" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels, separated by a comma. Labels *not* on this list will be excluded from consideration. Cannot be used in combination with ``exclude_labels``.", + "title": "Labels" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An upper bound of the required mean size of the uniquely labeled objects.", + "title": "Max" + }, + "mean_object_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The required mean size of the uniquely labeled objects.", + "title": "Mean Object Size" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A lower bound of the required mean size of the uniquely labeled objects.", + "title": "Min" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + } + }, + "title": "base_has_image_mean_object_size_model", + "type": "object" + }, + "base_has_image_n_labels_model": { + "additionalProperties": false, + "description": "base model for has_image_n_labels describing attributes.", + "properties": { + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "delta": { + "default": 0, + "description": "Maximum allowed difference of the number of labels (default is 0). The observed number of labels has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "exclude_labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels to be excluded from consideration, separated by a comma. The primary usage of this attribute is to exclude the background of a label image. Cannot be used in combination with ``labels``.", + "title": "Exclude Labels" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels, separated by a comma. Labels *not* on this list will be excluded from consideration. Cannot be used in combination with ``exclude_labels``.", + "title": "Labels" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed number of labels.", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed number of labels.", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected number of labels.", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + } + }, + "title": "base_has_image_n_labels_model", + "type": "object" + }, + "base_has_image_width_model": { + "additionalProperties": false, + "description": "base model for has_image_width describing attributes.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the image width (in pixels, default is 0). The observed width has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed width of the image (in pixels).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed width of the image (in pixels).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "width": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected width of the image (in pixels).", + "title": "Width" + } + }, + "title": "base_has_image_width_model", + "type": "object" + }, + "base_has_json_property_with_text_model": { + "additionalProperties": false, + "description": "base model for has_json_property_with_text describing attributes.", + "properties": { + "property": { + "description": "The property name to search the JSON document for.", + "title": "Property", + "type": "string" + }, + "text": { + "description": "The expected text value of the target JSON attribute.", + "title": "Text", + "type": "string" + } + }, + "required": [ + "property", + "text" + ], + "title": "base_has_json_property_with_text_model", + "type": "object" + }, + "base_has_json_property_with_value_model": { + "additionalProperties": false, + "description": "base model for has_json_property_with_value describing attributes.", + "properties": { + "property": { + "description": "The property name to search the JSON document for.", + "title": "Property", + "type": "string" + }, + "value": { + "description": "The expected JSON value of the target JSON attribute (as a JSON encoded string).", + "title": "Value", + "type": "string" + } + }, + "required": [ + "property", + "value" + ], + "title": "base_has_json_property_with_value_model", + "type": "object" + }, + "base_has_line_matching_model": { + "additionalProperties": false, + "description": "base model for has_line_matching describing attributes.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "expression": { + "description": "The regular expressions to attempt match in the output.", + "title": "Expression", + "type": "string" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "required": [ + "expression" + ], + "title": "base_has_line_matching_model", + "type": "object" + }, + "base_has_line_model": { + "additionalProperties": false, + "description": "base model for has_line describing attributes.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "line": { + "description": "The full line of text to search for in the output.", + "title": "Line", + "type": "string" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "required": [ + "line" + ], + "title": "base_has_line_model", + "type": "object" + }, + "base_has_n_columns_model": { + "additionalProperties": false, + "description": "base model for has_n_columns describing attributes.", + "properties": { + "comment": { + "default": "", + "description": "Comment character(s) used to skip comment lines (which should not be used for counting columns)", + "title": "Comment", + "type": "string" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "sep": { + "default": "\t", + "description": "Separator defining columns, default: tab", + "title": "Sep", + "type": "string" + } + }, + "title": "base_has_n_columns_model", + "type": "object" + }, + "base_has_n_elements_with_path_model": { + "additionalProperties": false, + "description": "base model for has_n_elements_with_path describing attributes.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "base_has_n_elements_with_path_model", + "type": "object" + }, + "base_has_n_lines_model": { + "additionalProperties": false, + "description": "base model for has_n_lines describing attributes.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "title": "base_has_n_lines_model", + "type": "object" + }, + "base_has_size_model": { + "additionalProperties": false, + "description": "base model for has_size describing attributes.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired size of the output (in bytes), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Size" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated alias for `size`", + "title": "Value" + } + }, + "title": "base_has_size_model", + "type": "object" + }, + "base_has_text_matching_model": { + "additionalProperties": false, + "description": "base model for has_text_matching describing attributes.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "expression": { + "description": "The regular expressions to attempt match in the output.", + "title": "Expression", + "type": "string" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + } + }, + "required": [ + "expression" + ], + "title": "base_has_text_matching_model", + "type": "object" + }, + "base_has_text_model": { + "additionalProperties": false, + "description": "base model for has_text describing attributes.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "text": { + "description": "The text to search for in the output.", + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "base_has_text_model", + "type": "object" + }, + "base_is_valid_xml_model": { + "additionalProperties": false, + "description": "base model for is_valid_xml describing attributes.", + "properties": {}, + "title": "base_is_valid_xml_model", + "type": "object" + }, + "base_not_has_text_model": { + "additionalProperties": false, + "description": "base model for not_has_text describing attributes.", + "properties": { + "text": { + "description": "The text to search for in the output.", + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "base_not_has_text_model", + "type": "object" + }, + "base_xml_element_model": { + "additionalProperties": false, + "description": "base model for xml_element describing attributes.", + "properties": { + "all": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first ", + "title": "All" + }, + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Asserts" + }, + "attribute": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The XML attribute name to test against from the target XML element.", + "title": "Attribute" + }, + "children": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Children" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "base_xml_element_model", + "type": "object" + }, + "element_text_is_model": { + "additionalProperties": false, + "description": "Asserts the text of the XML element with the specified XPath-like ``path`` is\nthe specified ``text``.\n\nFor example:\n\n```xml\n\n```\n\nThe assertion implicitly also asserts that an element matching ``path`` exists.\nWith ``negate`` the result of the assertion (on the equality) can be inverted (the\nimplicit assertion on the existence of the path is not affected).", + "properties": { + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "text": { + "description": "The expected element text (body of the XML tag) to test against on the target XML element", + "title": "Text", + "type": "string" + }, + "that": { + "const": "element_text_is", + "default": "element_text_is", + "title": "That", + "type": "string" + } + }, + "required": [ + "path", + "text" + ], + "title": "Assert Element Text Is", + "type": "object" + }, + "element_text_is_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "element_text_is": { + "$ref": "#/$defs/base_element_text_is_model", + "title": "Assert Element Text Is" + } + }, + "required": [ + "element_text_is" + ], + "title": "Assert Element Text Is (Nested)", + "type": "object" + }, + "element_text_matches_model": { + "additionalProperties": false, + "description": "Asserts the text of the XML element with the specified XPath-like ``path``\nmatches the regular expression defined by ``expression``.\n\nFor example:\n\n```xml\n\n```\n\nThe assertion implicitly also asserts that an element matching ``path`` exists.\nWith ``negate`` the result of the assertion (on the matching) can be inverted (the\nimplicit assertion on the existence of the path is not affected).", + "properties": { + "expression": { + "description": "The regular expressions to apply against the target element.", + "title": "Expression", + "type": "string" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "that": { + "const": "element_text_matches", + "default": "element_text_matches", + "title": "That", + "type": "string" + } + }, + "required": [ + "path", + "expression" + ], + "title": "Assert Element Text Matches", + "type": "object" + }, + "element_text_matches_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "element_text_matches": { + "$ref": "#/$defs/base_element_text_matches_model", + "title": "Assert Element Text Matches" + } + }, + "required": [ + "element_text_matches" + ], + "title": "Assert Element Text Matches (Nested)", + "type": "object" + }, + "element_text_model": { + "additionalProperties": false, + "description": "This tag allows the developer to recurisively specify additional assertions as\nchild elements about just the text contained in the element specified by the\nXPath-like ``path``, e.g.\n\n```xml\n\n \n\n```\n\nThe assertion implicitly also asserts that an element matching ``path`` exists.\nWith ``negate`` the result of the implicit assertions can be inverted.\nThe sub-assertions, which have their own ``negate`` attribute, are not affected\nby ``negate``.", + "properties": { + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Asserts" + }, + "children": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Children" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "that": { + "const": "element_text", + "default": "element_text", + "title": "That", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "Assert Element Text", + "type": "object" + }, + "element_text_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "element_text": { + "$ref": "#/$defs/base_element_text_model", + "title": "Assert Element Text" + } + }, + "required": [ + "element_text" + ], + "title": "Assert Element Text (Nested)", + "type": "object" + }, + "has_archive_member_model": { + "additionalProperties": false, + "description": "This tag allows to check if ``path`` is contained in a compressed file.\n\nThe path is a regular expression that is matched against the full paths of the objects in\nthe compressed file (remember that \"matching\" means it is checked if a prefix of\nthe full path of an archive member is described by the regular expression).\nValid archive formats include ``.zip``, ``.tar``, and ``.tar.gz``. Note that\ndepending on the archive creation method:\n\n- full paths of the members may be prefixed with ``./``\n- directories may be treated as empty files\n\n```xml\n\n```\n\nWith ``n`` and ``delta`` (or ``min`` and ``max``) assertions on the number of\narchive members matching ``path`` can be expressed. The following could be used,\ne.g., to assert an archive containing n±1 elements out of which at least\n4 need to have a ``txt`` extension.\n\n```xml\n\n\n```\n\nIn addition the tag can contain additional assertions as child elements about\nthe first member in the archive matching the regular expression ``path``. For\ninstance\n\n```xml\n\n \n\n```\n\nIf the ``all`` attribute is set to ``true`` then all archive members are subject\nto the assertions. Note that, archive members matching the ``path`` are sorted\nalphabetically.\n\nThe ``negate`` attribute of the ``has_archive_member`` assertion only affects\nthe asserts on the presence and number of matching archive members, but not any\nsub-assertions (which can offer the ``negate`` attribute on their own). The\ncheck if the file is an archive at all, which is also done by the function, is\nnot affected.", + "properties": { + "all": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first", + "title": "All" + }, + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Asserts" + }, + "children": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Children" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The regular expression specifying the archive member.", + "title": "Path", + "type": "string" + }, + "that": { + "const": "has_archive_member", + "default": "has_archive_member", + "title": "That", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "Assert Has Archive Member", + "type": "object" + }, + "has_archive_member_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_archive_member": { + "$ref": "#/$defs/base_has_archive_member_model", + "title": "Assert Has Archive Member" + } + }, + "required": [ + "has_archive_member" + ], + "title": "Assert Has Archive Member (Nested)", + "type": "object" + }, + "has_element_with_path_model": { + "additionalProperties": false, + "description": "Asserts the XML output contains at least one element (or tag) with the specified\nXPath-like ``path``, e.g.\n\n```xml\n\n```\n\nWith ``negate`` the result of the assertion can be inverted.", + "properties": { + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "that": { + "const": "has_element_with_path", + "default": "has_element_with_path", + "title": "That", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "Assert Has Element With Path", + "type": "object" + }, + "has_element_with_path_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_element_with_path": { + "$ref": "#/$defs/base_has_element_with_path_model", + "title": "Assert Has Element With Path" + } + }, + "required": [ + "has_element_with_path" + ], + "title": "Assert Has Element With Path (Nested)", + "type": "object" + }, + "has_h5_attribute_model": { + "additionalProperties": false, + "description": "Asserts HDF5 output contains the specified ``value`` for an attribute (``key``), e.g.\n\n```xml\n\n```", + "properties": { + "key": { + "description": "HDF5 attribute to check value of.", + "title": "Key", + "type": "string" + }, + "that": { + "const": "has_h5_attribute", + "default": "has_h5_attribute", + "title": "That", + "type": "string" + }, + "value": { + "description": "Expected value of HDF5 attribute to check.", + "title": "Value", + "type": "string" + } + }, + "required": [ + "key", + "value" + ], + "title": "Assert Has H5 Attribute", + "type": "object" + }, + "has_h5_attribute_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_h5_attribute": { + "$ref": "#/$defs/base_has_h5_attribute_model", + "title": "Assert Has H5 Attribute" + } + }, + "required": [ + "has_h5_attribute" + ], + "title": "Assert Has H5 Attribute (Nested)", + "type": "object" + }, + "has_h5_keys_model": { + "additionalProperties": false, + "description": "Asserts the specified HDF5 output has the given keys.", + "properties": { + "keys": { + "description": "HDF5 attributes to check value of as a comma-separated string.", + "title": "Keys", + "type": "string" + }, + "that": { + "const": "has_h5_keys", + "default": "has_h5_keys", + "title": "That", + "type": "string" + } + }, + "required": [ + "keys" + ], + "title": "Assert Has H5 Keys", + "type": "object" + }, + "has_h5_keys_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_h5_keys": { + "$ref": "#/$defs/base_has_h5_keys_model", + "title": "Assert Has H5 Keys" + } + }, + "required": [ + "has_h5_keys" + ], + "title": "Assert Has H5 Keys (Nested)", + "type": "object" + }, + "has_image_center_of_mass_model": { + "additionalProperties": false, + "description": "Asserts the specified output is an image and has the specified center of mass.\n\nAsserts the output is an image and has a specific center of mass,\nor has an Euclidean distance of ``eps`` or less to that point (e.g.,\n````).", + "properties": { + "center_of_mass": { + "description": "The required center of mass of the image intensities (horizontal and vertical coordinate, separated by a comma).", + "title": "Center Of Mass", + "type": "string" + }, + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "eps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "default": 0.01, + "description": "The maximum allowed Euclidean distance to the required center of mass (defaults to ``0.01``).", + "title": "Eps" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + }, + "that": { + "const": "has_image_center_of_mass", + "default": "has_image_center_of_mass", + "title": "That", + "type": "string" + } + }, + "required": [ + "center_of_mass" + ], + "title": "Assert Has Image Center Of Mass", + "type": "object" + }, + "has_image_center_of_mass_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_center_of_mass": { + "$ref": "#/$defs/base_has_image_center_of_mass_model", + "title": "Assert Has Image Center Of Mass" + } + }, + "required": [ + "has_image_center_of_mass" + ], + "title": "Assert Has Image Center Of Mass (Nested)", + "type": "object" + }, + "has_image_channels_model": { + "additionalProperties": false, + "description": "Asserts the output is an image and has a specific number of channels.\n\nThe number of channels is plus/minus ``delta`` (e.g., ````).\n\nAlternatively the range of the expected number of channels can be specified by ``min`` and/or ``max``.", + "properties": { + "channels": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected number of channels of the image.", + "title": "Channels" + }, + "delta": { + "default": 0, + "description": "Maximum allowed difference of the number of channels (default is 0). The observed number of channels has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed number of channels.", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed number of channels.", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_image_channels", + "default": "has_image_channels", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has Image Channels", + "type": "object" + }, + "has_image_channels_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_channels": { + "$ref": "#/$defs/base_has_image_channels_model", + "title": "Assert Has Image Channels" + } + }, + "required": [ + "has_image_channels" + ], + "title": "Assert Has Image Channels (Nested)", + "type": "object" + }, + "has_image_depth_model": { + "additionalProperties": false, + "description": "Asserts the output is an image and has a specific depth (number of slices).\n\nThe depth is plus/minus ``delta`` (e.g., ````).\nAlternatively the range of the expected depth can be specified by ``min`` and/or ``max``.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the image depth (number of slices, default is 0). The observed depth has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "depth": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected depth of the image (number of slices).", + "title": "Depth" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed depth of the image (number of slices).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed depth of the image (number of slices).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_image_depth", + "default": "has_image_depth", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has Image Depth", + "type": "object" + }, + "has_image_depth_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_depth": { + "$ref": "#/$defs/base_has_image_depth_model", + "title": "Assert Has Image Depth" + } + }, + "required": [ + "has_image_depth" + ], + "title": "Assert Has Image Depth (Nested)", + "type": "object" + }, + "has_image_frames_model": { + "additionalProperties": false, + "description": "Asserts the output is an image and has a specific number of frames (number of time steps).\n\nThe number of frames is plus/minus ``delta`` (e.g., ````).\nAlternatively the range of the expected number of frames can be specified by ``min`` and/or ``max``.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the number of frames in the image sequence (number of time steps, default is 0). The observed number of frames has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "frames": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected number of frames in the image sequence (number of time steps).", + "title": "Frames" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed number of frames in the image sequence (number of time steps).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed number of frames in the image sequence (number of time steps).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_image_frames", + "default": "has_image_frames", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has Image Frames", + "type": "object" + }, + "has_image_frames_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_frames": { + "$ref": "#/$defs/base_has_image_frames_model", + "title": "Assert Has Image Frames" + } + }, + "required": [ + "has_image_frames" + ], + "title": "Assert Has Image Frames (Nested)", + "type": "object" + }, + "has_image_height_model": { + "additionalProperties": false, + "description": "Asserts the output is an image and has a specific height (in pixels).\n\nThe height is plus/minus ``delta`` (e.g., ````).\nAlternatively the range of the expected height can be specified by ``min`` and/or ``max``.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the image height (in pixels, default is 0). The observed height has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "height": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected height of the image (in pixels).", + "title": "Height" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed height of the image (in pixels).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed height of the image (in pixels).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_image_height", + "default": "has_image_height", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has Image Height", + "type": "object" + }, + "has_image_height_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_height": { + "$ref": "#/$defs/base_has_image_height_model", + "title": "Assert Has Image Height" + } + }, + "required": [ + "has_image_height" + ], + "title": "Assert Has Image Height (Nested)", + "type": "object" + }, + "has_image_mean_intensity_model": { + "additionalProperties": false, + "description": "Asserts the output is an image and has a specific mean intensity value.\n\nThe mean intensity value is plus/minus ``eps`` (e.g., ````).\nAlternatively the range of the expected mean intensity value can be specified by ``min`` and/or ``max``.", + "properties": { + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "eps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "default": 0.01, + "description": "The absolute tolerance to be used for ``value`` (defaults to ``0.01``). The observed mean value of the image intensities has to be in the range ``value +- eps``.", + "title": "Eps" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An upper bound of the required mean value of the image intensities.", + "title": "Max" + }, + "mean_intensity": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The required mean value of the image intensities.", + "title": "Mean Intensity" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A lower bound of the required mean value of the image intensities.", + "title": "Min" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + }, + "that": { + "const": "has_image_mean_intensity", + "default": "has_image_mean_intensity", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has Image Mean Intensity", + "type": "object" + }, + "has_image_mean_intensity_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_mean_intensity": { + "$ref": "#/$defs/base_has_image_mean_intensity_model", + "title": "Assert Has Image Mean Intensity" + } + }, + "required": [ + "has_image_mean_intensity" + ], + "title": "Assert Has Image Mean Intensity (Nested)", + "type": "object" + }, + "has_image_mean_object_size_model": { + "additionalProperties": false, + "description": "Asserts the output is an image with labeled objects which have the specified mean size (number of pixels),\n\nThe mean size is plus/minus ``eps`` (e.g., ````).\n\nThe labels must be unique.", + "properties": { + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "eps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "default": 0.01, + "description": "The absolute tolerance to be used for ``value`` (defaults to ``0.01``). The observed mean size of the uniquely labeled objects has to be in the range ``value +- eps``.", + "title": "Eps" + }, + "exclude_labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels to be excluded from consideration, separated by a comma. The primary usage of this attribute is to exclude the background of a label image. Cannot be used in combination with ``labels``.", + "title": "Exclude Labels" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels, separated by a comma. Labels *not* on this list will be excluded from consideration. Cannot be used in combination with ``exclude_labels``.", + "title": "Labels" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An upper bound of the required mean size of the uniquely labeled objects.", + "title": "Max" + }, + "mean_object_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The required mean size of the uniquely labeled objects.", + "title": "Mean Object Size" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A lower bound of the required mean size of the uniquely labeled objects.", + "title": "Min" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + }, + "that": { + "const": "has_image_mean_object_size", + "default": "has_image_mean_object_size", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has Image Mean Object Size", + "type": "object" + }, + "has_image_mean_object_size_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_mean_object_size": { + "$ref": "#/$defs/base_has_image_mean_object_size_model", + "title": "Assert Has Image Mean Object Size" + } + }, + "required": [ + "has_image_mean_object_size" + ], + "title": "Assert Has Image Mean Object Size (Nested)", + "type": "object" + }, + "has_image_n_labels_model": { + "additionalProperties": false, + "description": "Asserts the output is an image and has the specified labels.\n\nLabels can be a number of labels or unique values (e.g.,\n````).\n\nThe primary usage of this assertion is to verify the number of objects in images with uniquely labeled objects.", + "properties": { + "channel": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific channel of the image (where ``0`` corresponds to the first image channel).", + "title": "Channel" + }, + "delta": { + "default": 0, + "description": "Maximum allowed difference of the number of labels (default is 0). The observed number of labels has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "exclude_labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels to be excluded from consideration, separated by a comma. The primary usage of this attribute is to exclude the background of a label image. Cannot be used in combination with ``labels``.", + "title": "Exclude Labels" + }, + "frame": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific frame of the image sequence (where ``0`` corresponds to the first image frame).", + "title": "Frame" + }, + "labels": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of labels, separated by a comma. Labels *not* on this list will be excluded from consideration. Cannot be used in combination with ``exclude_labels``.", + "title": "Labels" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed number of labels.", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed number of labels.", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected number of labels.", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "slice": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Restricts the assertion to a specific slice of the image (where ``0`` corresponds to the first image slice).", + "title": "Slice" + }, + "that": { + "const": "has_image_n_labels", + "default": "has_image_n_labels", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has Image N Labels", + "type": "object" + }, + "has_image_n_labels_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_n_labels": { + "$ref": "#/$defs/base_has_image_n_labels_model", + "title": "Assert Has Image N Labels" + } + }, + "required": [ + "has_image_n_labels" + ], + "title": "Assert Has Image N Labels (Nested)", + "type": "object" + }, + "has_image_width_model": { + "additionalProperties": false, + "description": "Asserts the output is an image and has a specific width (in pixels).\n\nThe width is plus/minus ``delta`` (e.g., ````).\nAlternatively the range of the expected width can be specified by ``min`` and/or ``max``.", + "properties": { + "delta": { + "default": 0, + "description": "Maximum allowed difference of the image width (in pixels, default is 0). The observed width has to be in the range ``value +- delta``.", + "title": "Delta", + "type": "integer" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum allowed width of the image (in pixels).", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum allowed width of the image (in pixels).", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_image_width", + "default": "has_image_width", + "title": "That", + "type": "string" + }, + "width": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Expected width of the image (in pixels).", + "title": "Width" + } + }, + "title": "Assert Has Image Width", + "type": "object" + }, + "has_image_width_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_image_width": { + "$ref": "#/$defs/base_has_image_width_model", + "title": "Assert Has Image Width" + } + }, + "required": [ + "has_image_width" + ], + "title": "Assert Has Image Width (Nested)", + "type": "object" + }, + "has_json_property_with_text_model": { + "additionalProperties": false, + "description": "Asserts the JSON document contains a property or key with the specified text (i.e. string) value.\n\n```xml\n\n```", + "properties": { + "property": { + "description": "The property name to search the JSON document for.", + "title": "Property", + "type": "string" + }, + "text": { + "description": "The expected text value of the target JSON attribute.", + "title": "Text", + "type": "string" + }, + "that": { + "const": "has_json_property_with_text", + "default": "has_json_property_with_text", + "title": "That", + "type": "string" + } + }, + "required": [ + "property", + "text" + ], + "title": "Assert Has Json Property With Text", + "type": "object" + }, + "has_json_property_with_text_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_json_property_with_text": { + "$ref": "#/$defs/base_has_json_property_with_text_model", + "title": "Assert Has Json Property With Text" + } + }, + "required": [ + "has_json_property_with_text" + ], + "title": "Assert Has Json Property With Text (Nested)", + "type": "object" + }, + "has_json_property_with_value_model": { + "additionalProperties": false, + "description": "Asserts the JSON document contains a property or key with the specified JSON value.\n\n```xml\n\n```", + "properties": { + "property": { + "description": "The property name to search the JSON document for.", + "title": "Property", + "type": "string" + }, + "that": { + "const": "has_json_property_with_value", + "default": "has_json_property_with_value", + "title": "That", + "type": "string" + }, + "value": { + "description": "The expected JSON value of the target JSON attribute (as a JSON encoded string).", + "title": "Value", + "type": "string" + } + }, + "required": [ + "property", + "value" + ], + "title": "Assert Has Json Property With Value", + "type": "object" + }, + "has_json_property_with_value_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_json_property_with_value": { + "$ref": "#/$defs/base_has_json_property_with_value_model", + "title": "Assert Has Json Property With Value" + } + }, + "required": [ + "has_json_property_with_value" + ], + "title": "Assert Has Json Property With Value (Nested)", + "type": "object" + }, + "has_line_matching_model": { + "additionalProperties": false, + "description": "Asserts the specified output contains a line matching the\nregular expression specified by the argument expression. If n is given\nthe assertion checks for exactly n occurrences.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "expression": { + "description": "The regular expressions to attempt match in the output.", + "title": "Expression", + "type": "string" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_line_matching", + "default": "has_line_matching", + "title": "That", + "type": "string" + } + }, + "required": [ + "expression" + ], + "title": "Assert Has Line Matching", + "type": "object" + }, + "has_line_matching_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_line_matching": { + "$ref": "#/$defs/base_has_line_matching_model", + "title": "Assert Has Line Matching" + } + }, + "required": [ + "has_line_matching" + ], + "title": "Assert Has Line Matching (Nested)", + "type": "object" + }, + "has_line_model": { + "additionalProperties": false, + "description": "Asserts the specified output contains the line specified by the\nargument line. The exact number of occurrences can be optionally\nspecified by the argument n", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "line": { + "description": "The full line of text to search for in the output.", + "title": "Line", + "type": "string" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_line", + "default": "has_line", + "title": "That", + "type": "string" + } + }, + "required": [ + "line" + ], + "title": "Assert Has Line", + "type": "object" + }, + "has_line_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_line": { + "$ref": "#/$defs/base_has_line_model", + "title": "Assert Has Line" + } + }, + "required": [ + "has_line" + ], + "title": "Assert Has Line (Nested)", + "type": "object" + }, + "has_n_columns_model": { + "additionalProperties": false, + "description": "Asserts tabular output contains the specified\nnumber (``n``) of columns.\n\nFor instance, ````. The assertion tests only the first line.\nNumber of columns can optionally also be specified with ``delta``. Alternatively the\nrange of expected occurrences can be specified by ``min`` and/or ``max``.\n\nOptionally a column separator (``sep``, default is `` ``) `and comment character(s)\ncan be specified (``comment``, default is empty string). The first non-comment\nline is used for determining the number of columns.", + "properties": { + "comment": { + "default": "", + "description": "Comment character(s) used to skip comment lines (which should not be used for counting columns)", + "title": "Comment", + "type": "string" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "sep": { + "default": "\t", + "description": "Separator defining columns, default: tab", + "title": "Sep", + "type": "string" + }, + "that": { + "const": "has_n_columns", + "default": "has_n_columns", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has N Columns", + "type": "object" + }, + "has_n_columns_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_n_columns": { + "$ref": "#/$defs/base_has_n_columns_model", + "title": "Assert Has N Columns" + } + }, + "required": [ + "has_n_columns" + ], + "title": "Assert Has N Columns (Nested)", + "type": "object" + }, + "has_n_elements_with_path_model": { + "additionalProperties": false, + "description": "Asserts the XML output contains the specified number (``n``, optionally with ``delta``) of elements (or\ntags) with the specified XPath-like ``path``.\n\nFor example:\n\n```xml\n\n```\n\nAlternatively to ``n`` and ``delta`` also the ``min`` and ``max`` attributes\ncan be used to specify the range of the expected number of occurrences.\nWith ``negate`` the result of the assertion can be inverted.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "that": { + "const": "has_n_elements_with_path", + "default": "has_n_elements_with_path", + "title": "That", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "Assert Has N Elements With Path", + "type": "object" + }, + "has_n_elements_with_path_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_n_elements_with_path": { + "$ref": "#/$defs/base_has_n_elements_with_path_model", + "title": "Assert Has N Elements With Path" + } + }, + "required": [ + "has_n_elements_with_path" + ], + "title": "Assert Has N Elements With Path (Nested)", + "type": "object" + }, + "has_n_lines_model": { + "additionalProperties": false, + "description": "Asserts the specified output contains ``n`` lines allowing\nfor a difference in the number of lines (delta)\nor relative differebce in the number of lines", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_n_lines", + "default": "has_n_lines", + "title": "That", + "type": "string" + } + }, + "title": "Assert Has N Lines", + "type": "object" + }, + "has_n_lines_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_n_lines": { + "$ref": "#/$defs/base_has_n_lines_model", + "title": "Assert Has N Lines" + } + }, + "required": [ + "has_n_lines" + ], + "title": "Assert Has N Lines (Nested)", + "type": "object" + }, + "has_size_model": { + "additionalProperties": false, + "description": "Asserts the specified output has a size of the specified value\n\nAttributes size and value or synonyms though value is considered deprecated.\nThe size optionally allows for absolute (``delta``) difference.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired size of the output (in bytes), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Size" + }, + "that": { + "const": "has_size", + "default": "has_size", + "title": "That", + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated alias for `size`", + "title": "Value" + } + }, + "title": "Assert Has Size", + "type": "object" + }, + "has_size_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_size": { + "$ref": "#/$defs/base_has_size_model", + "title": "Assert Has Size" + } + }, + "required": [ + "has_size" + ], + "title": "Assert Has Size (Nested)", + "type": "object" + }, + "has_text_matching_model": { + "additionalProperties": false, + "description": "Asserts the specified output contains text matching the\nregular expression specified by the argument expression.\nIf n is given the assertion checks for exactly n (nonoverlapping)\noccurrences.", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "expression": { + "description": "The regular expressions to attempt match in the output.", + "title": "Expression", + "type": "string" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "that": { + "const": "has_text_matching", + "default": "has_text_matching", + "title": "That", + "type": "string" + } + }, + "required": [ + "expression" + ], + "title": "Assert Has Text Matching", + "type": "object" + }, + "has_text_matching_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_text_matching": { + "$ref": "#/$defs/base_has_text_matching_model", + "title": "Assert Has Text Matching" + } + }, + "required": [ + "has_text_matching" + ], + "title": "Assert Has Text Matching (Nested)", + "type": "object" + }, + "has_text_model": { + "additionalProperties": false, + "description": "Asserts specified output contains the substring specified by\nthe argument text. The exact number of occurrences can be\noptionally specified by the argument n", + "properties": { + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "text": { + "description": "The text to search for in the output.", + "title": "Text", + "type": "string" + }, + "that": { + "const": "has_text", + "default": "has_text", + "title": "That", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "Assert Has Text", + "type": "object" + }, + "has_text_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "has_text": { + "$ref": "#/$defs/base_has_text_model", + "title": "Assert Has Text" + } + }, + "required": [ + "has_text" + ], + "title": "Assert Has Text (Nested)", + "type": "object" + }, + "is_valid_xml_model": { + "additionalProperties": false, + "description": "Asserts the output is a valid XML file (e.g. ````).", + "properties": { + "that": { + "const": "is_valid_xml", + "default": "is_valid_xml", + "title": "That", + "type": "string" + } + }, + "title": "Assert Is Valid Xml", + "type": "object" + }, + "is_valid_xml_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "is_valid_xml": { + "$ref": "#/$defs/base_is_valid_xml_model", + "title": "Assert Is Valid Xml" + } + }, + "required": [ + "is_valid_xml" + ], + "title": "Assert Is Valid Xml (Nested)", + "type": "object" + }, + "not_has_text_model": { + "additionalProperties": false, + "description": "Asserts specified output does not contain the substring\nspecified by the argument text", + "properties": { + "text": { + "description": "The text to search for in the output.", + "title": "Text", + "type": "string" + }, + "that": { + "const": "not_has_text", + "default": "not_has_text", + "title": "That", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "Assert Not Has Text", + "type": "object" + }, + "not_has_text_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "not_has_text": { + "$ref": "#/$defs/base_not_has_text_model", + "title": "Assert Not Has Text" + } + }, + "required": [ + "not_has_text" + ], + "title": "Assert Not Has Text (Nested)", + "type": "object" + }, + "xml_element_model": { + "additionalProperties": false, + "description": "Assert if the XML file contains element(s) or tag(s) with the specified\n[XPath-like ``path``](https://lxml.de/xpathxslt.html). If ``n`` and ``delta``\nor ``min`` and ``max`` are given also the number of occurrences is checked.\n\n```xml\n\n \n \n \n\n```\n\nWith ``negate=\"true\"`` the outcome of the assertions wrt the presence and number\nof ``path`` can be negated. If there are any sub assertions then check them against\n\n- the content of the attribute ``attribute``\n- the element's text if no attribute is given\n\n```xml\n\n \n \n \n\n```\n\nSub-assertions are not subject to the ``negate`` attribute of ``xml_element``.\nIf ``all`` is ``true`` then the sub assertions are checked for all occurrences.\n\nNote that all other XML assertions can be expressed by this assertion (Galaxy\nalso implements the other assertions by calling this one).", + "properties": { + "all": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first ", + "title": "All" + }, + "asserts": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Asserts" + }, + "attribute": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The XML attribute name to test against from the target XML element.", + "title": "Attribute" + }, + "children": { + "anyOf": [ + { + "$ref": "#/$defs/assertion_list" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Children" + }, + "delta": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": 0, + "description": "Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Delta" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "Min" + }, + "n": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired number, can be suffixed by ``(k|M|G|T|P|E)i?``", + "title": "N" + }, + "negate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": false, + "description": "A boolean that can be set to true to negate the outcome of the assertion.", + "title": "Negate" + }, + "path": { + "description": "The Python xpath-like expression to find the target element.", + "title": "Path", + "type": "string" + }, + "that": { + "const": "xml_element", + "default": "xml_element", + "title": "That", + "type": "string" + } + }, + "required": [ + "path" + ], + "title": "Assert Xml Element", + "type": "object" + }, + "xml_element_model_nested": { + "additionalProperties": false, + "description": "Nested version of this assertion model.", + "properties": { + "xml_element": { + "$ref": "#/$defs/base_xml_element_model", + "title": "Assert Xml Element" + } + }, + "required": [ + "xml_element" + ], + "title": "Assert Xml Element (Nested)", + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Galaxy workflow tests file — a YAML list of test entries asserting the expected inputs and outputs of a workflow run.", + "items": { + "$ref": "#/$defs/TestJob" + }, + "title": "GalaxyWorkflowTests", + "type": "array" +} diff --git a/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/workflow-brief-schema.schema.json b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/workflow-brief-schema.schema.json new file mode 100644 index 0000000..92c431e --- /dev/null +++ b/plugins/foundry-skills/skills/workflow-brief-to-galaxy-test-plan/references/schemas/workflow-brief-schema.schema.json @@ -0,0 +1,103 @@ +{ + "name": "Workflow Brief", + "title": "required", + "sections": [ + { + "heading": "Workflow", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Objective", + "min": 1, + "max": 1, + "content": true + }, + { + "heading": "Sources", + "max": 1, + "content": true + }, + { + "heading": "Scope", + "max": 1, + "content": true, + "sections": [ + { + "heading": "Included", + "max": 1, + "content": true + }, + { + "heading": "Excluded", + "max": 1, + "content": true + } + ] + }, + { + "heading": "Inputs and outputs", + "max": 1, + "content": true + }, + { + "heading": "Requirements and preferences", + "max": 1, + "content": true + }, + { + "heading": "Acceptance criteria", + "max": 1, + "content": true + }, + { + "heading": "Open questions", + "max": 1, + "content": true + }, + { + "heading": "Decisions", + "max": 1, + "content": true + }, + { + "heading": "Related artifacts", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + }, + { + "heading": "Agent Environment", + "min": 1, + "max": 1, + "content": true, + "sections": [ + { + "heading": "Tooling", + "max": 1, + "content": true + }, + { + "heading": "Constraints", + "max": 1, + "content": true + }, + { + "heading": "Containerization", + "max": 1, + "content": true + }, + { + "heading": "Blockers", + "max": 1 + } + ] + } + ] +} diff --git a/plugins/galaxy-dev-skills/.claude-plugin/plugin.json b/plugins/galaxy-dev-skills/.claude-plugin/plugin.json index 4c518b2..4475bc4 100644 --- a/plugins/galaxy-dev-skills/.claude-plugin/plugin.json +++ b/plugins/galaxy-dev-skills/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "galaxy-dev-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Galaxy development skills: tool XML wrappers, Nextflow-to-Galaxy conversion, ToolShed revisions in usegalaxy-tools, UCSC track hubs and Galaxy Hub news posts. Mirror of galaxyproject/galaxy-skills (dev-skills/).", "author": { "name": "The Galaxy Project", diff --git a/plugins/galaxy-dev-skills/.codex-plugin/plugin.json b/plugins/galaxy-dev-skills/.codex-plugin/plugin.json index 7339e3c..21939c4 100644 --- a/plugins/galaxy-dev-skills/.codex-plugin/plugin.json +++ b/plugins/galaxy-dev-skills/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "galaxy-dev-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Galaxy development skills: tool XML wrappers, Nextflow-to-Galaxy conversion, ToolShed revisions in usegalaxy-tools, UCSC track hubs and Galaxy Hub news posts. Mirror of galaxyproject/galaxy-skills (dev-skills/).", "author": { "name": "The Galaxy Project", diff --git a/plugins/galaxy-dev-skills/.cursor-plugin/plugin.json b/plugins/galaxy-dev-skills/.cursor-plugin/plugin.json index ebae43b..d66d14c 100644 --- a/plugins/galaxy-dev-skills/.cursor-plugin/plugin.json +++ b/plugins/galaxy-dev-skills/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "galaxy-dev-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Galaxy development skills: tool XML wrappers, Nextflow-to-Galaxy conversion, ToolShed revisions in usegalaxy-tools, UCSC track hubs and Galaxy Hub news posts. Mirror of galaxyproject/galaxy-skills (dev-skills/).", "author": { "name": "The Galaxy Project" diff --git a/plugins/galaxy-dev-skills/UPSTREAM.json b/plugins/galaxy-dev-skills/UPSTREAM.json index 43cac4c..b305321 100644 --- a/plugins/galaxy-dev-skills/UPSTREAM.json +++ b/plugins/galaxy-dev-skills/UPSTREAM.json @@ -1,7 +1,7 @@ { "repository": "https://github.com/galaxyproject/galaxy-skills", "ref": "main", - "commit": "7eb5a1dd2e3afed412f0cfebaa39236f36695845", + "commit": "5ec0fea302cae826708e1e198e89f9206886d8f9", "path": "dev-skills", - "synced_at": "2026-09-17T20:49:30Z" + "synced_at": "2026-11-16T06:42:22Z" } diff --git a/plugins/galaxy-dev-skills/skills/tool-dev/SKILL.md b/plugins/galaxy-dev-skills/skills/tool-dev/SKILL.md index 7d5ef4f..97eae2a 100644 --- a/plugins/galaxy-dev-skills/skills/tool-dev/SKILL.md +++ b/plugins/galaxy-dev-skills/skills/tool-dev/SKILL.md @@ -1,11 +1,11 @@ --- name: tool-dev -description: Galaxy tool development reference — creating new tools, testing, IUC review preparation, and updating existing tools. Use when authoring or modifying Galaxy tool wrappers for tools-iuc. +description: How to author, test, and update Galaxy tool wrappers so they meet the IUC standards, with pre-PR passes for security, ToolShed readiness, remote execution, and review flags. Use when authoring or modifying Galaxy tool wrappers for tools-iuc; defers to the IUC standards for the rules. --- # Galaxy Tool Development Reference -Reference for authoring Galaxy tool wrappers that pass tools-iuc review. Derived from real IUC review feedback (25 inline comments, 3 reviewers) on a 5-tool submission. +The [IUC standards][std] say what a correct wrapper is. This skill says how an agent gets there and proves it: worked examples, gotchas the standards do not cover, decisions to make, and the checks to run and record before hand-back. It does not restate the standards. Where a section opens with a **Standard:** line, read that page for the rule; the text below it is examples and practice only. ## When to Use This Skill @@ -17,31 +17,62 @@ Reference for authoring Galaxy tool wrappers that pass tools-iuc review. Derived - Preparing a tools-iuc PR - Reviewing Galaxy XML conventions -## Key References +Data managers are out of scope here. Follow [Data Manager Best Practices][std-dm] (naming `data_manager_$name`, stable identifiers with no wall-clock values, tests with stable expected values). + +## Authority + +Order of precedence: + +1. **The target repository's convention beats everything below.** If sibling tools in the repo you are contributing to do something differently (help format, `detect_errors`, inline tokens vs `macros.xml`, profile), follow the repo and **say so in the hand-back**, naming the convention and the sibling tool you took it from. +2. **The IUC standards beat this skill.** If the two disagree, the standards win; report the disagreement so the skill can be fixed. +3. **This skill** covers how to apply the standards, plus agent practice they omit. + +| Standards page | Covers | +|---|---| +| [tool_xml][std-tool-xml] | Versions, IDs, profile, bio.tools, required files, creator, params, tests, command, help, coding style | +| [security][std-security] | Tool Security Checklist (ten items) -- drives pass 10a | +| [integration_checklist][std-checklist] | ToolShed Readiness Checklist -- drives pass 10b | +| [pulsar][std-pulsar] | Remote Execution Checklist (six items) -- drives pass 10c | +| [package_xml][std-package] | Requirements, Bioconda/conda-forge, multi-requirement (mulled) containers | +| [licensing][std-licensing] | OSI rule, licensing layers, use restrictions | +| [shed_yml][std-shed] | `.shed.yml` fields, naming, auto-generated suites, include/exclude | +| [repositories][std-repos] | Repository layout, README, suites | +| [data_managers][std-dm] | Data managers (out of scope for this skill) | +| [management][std-management] | Publishing to the ToolShed (IUC CI does this; do not upload by hand) | + +Last aligned against galaxy-iuc/standards 3f6039ac (2026-10-21) plus [PR #95][std-pr95]. PR #95 is a proposal (Markdown help preferred for new tools, when `exit_code` is right, the `@PROFILE@` token, the profile ceiling); where this skill cites it, the published standards may not say it yet. + +Other references: -- [IUC Best Practices](https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices.html) — the canonical standards document - [Galaxy Tool XSD Schema](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/tool_util/xsd/galaxy.xsd) — validate XML against this - [Galaxy Core Test Tools](https://github.com/galaxyproject/galaxy/tree/dev/test/functional/tools) — reference implementations and edge cases - [Planemo Documentation](https://planemo.readthedocs.io/) — lint, test, serve, shed workflows - `references/testing.md` — standalone planemo testing reference (also used by other skills) - `references/tool-placement.md` — where to create tools decision guide +- `references/tool-help-diagrams.md` -- SVG diagrams for tool help (RST help only) - `tool-selection-diagram/SKILL.md` — generate tool selection flowchart diagrams for multi-tool suites --- ## 1. Directory Structure +**Standard:** [repositories][std-repos] for layout, [shed_yml][std-shed] for suites. + All tools in a suite live in **one flat directory** under `tools//`. No subdirectories for individual tools. ``` tools/mytool/ -├── macros.xml # Shared tokens, macros, citations +├── macros.xml # Only when two or more tools share something (see Section 3) ├── mytool_align.xml # Tool wrapper A ├── mytool_filter.xml # Tool wrapper B ├── .shed.yml # Tool Shed metadata ├── static/ │ └── images/ -│ └── mytool_tool_selection.png # Optional: tool selection diagram +│ └── mytool_tool_selection.png # Optional: help image (forces RST help, see Section 2) +├── tool-data/ # Only when the tool uses a data table (see Section 5) +│ └── mytool_indexes.loc.sample +├── tool_data_table_conf.xml.sample # Only with a data table +├── tool_data_table_conf.xml.test # Only with a data table └── test-data/ ├── input.fastq.gz # Shared test inputs ├── reference.fasta @@ -52,28 +83,38 @@ tools/mytool/ Add `.py` wrapper scripts only when the upstream CLI can't produce the output Galaxy needs (format conversion, multi-step pipelines, etc.) — most tools don't need them. **Key rules:** -- Tool IDs use underscores: `mytool_align`, not hyphens +- Tool IDs contain only `[a-z0-9_-]`; separate words with underscores or dashes and match what sibling tools in the repo do ([Tool ids][std-ids]) - Test data goes in a single shared `test-data/` directory -- One `macros.xml` per tool suite, not per tool +- A shared `macros.xml` is per suite, not per tool. A single-tool directory can keep its tokens inline instead (Section 3) - One `.shed.yml` per suite +> **Verify:** [repositories][std-repos] says every unrestricted ToolShed repository should contain a `README.rst`. Many tools-iuc tool directories have none, and it is not clear how strictly reviewers apply it. Check sibling tools before adding one. + --- ## 2. XML Wrapper Structure +**Standard:** [tool_xml][std-tool-xml]. Element and attribute order: [Coding Style][std-style]. + ### Element Order (Strict) -Galaxy XML elements must appear in this order. `planemo lint` enforces this. +Galaxy XML elements must appear in the standards' order. `planemo lint` checks it. The elements an agent normally writes, in order: ```xml ... - macros.xml + macros.xml + + mytool + + + + - echo @TOOL_VERSION@ + mytool --version + + ... + ``` +`` and ``, when used, go between `` and ``. Section 13 has the rest. + ### Tool Element Attributes ```xml @@ -108,14 +154,16 @@ Galaxy XML elements must appear in this order. `planemo lint` enforces this. profile="@PROFILE@"> ``` -- `id`: lowercase, `[a-z0-9_-]` only. Should be meaningful in a larger context — prefix with suite name for multi-tool suites (e.g., `bedtools_intersect`). +- `id`: [Tool ids][std-ids]. Prefix with the suite name for multi-tool suites (e.g., `bedtools_intersect`). - `name`: Human-readable, title case. How users and admins find the tool — be specific, not generic. -- `version`: Always use `@TOOL_VERSION@+galaxy@VERSION_SUFFIX@` from macros -- `profile`: Use `@PROFILE@` token — must be recent (not older than ~1 year). Currently `25.0` for new tools. +- `version`: [Tool versions][std-versions]. Use `@TOOL_VERSION@+galaxy@VERSION_SUFFIX@` built from tokens (inline or in `macros.xml`, see Section 3), and reuse `@TOOL_VERSION@` in the ``. Exception: a tool that is not a wrapper for one underlying tool omits the `+galaxy` suffix and uses any PEP 440 version. When to bump is in Section 11. +- `profile`: [Tool profile][std-profile]. Use a recent profile, but not one newer than the major public servers run: Galaxy refuses to load a tool whose profile is newer than the server, so a newer profile is not a goal in itself ([PR #95][std-pr95]). Do not copy a number from this skill. Look at what sibling tools in the target repo use, and go higher only when the wrapper needs a feature from a later profile (for example `format="markdown"` help needs 24.2). In a suite, put it in a `@PROFILE@` token so all tools move together; a single tool can set it inline. ### bio.tools Cross-References -Link to the upstream tool's bio.tools entry with ``. Create the bio.tools entry if none exists (needs at least one EDAM Topic and Operation). Pick specific EDAM terms, not root terms like "Topic" or "Operation". +**Standard:** [Tool cross-references][std-biotools]. + +Link to the upstream tool's bio.tools entry with ``. Reference only the tool being wrapped, not its dependencies. Create the bio.tools entry if none exists (needs at least one EDAM Topic and Operation, specific terms rather than root terms). Improve EDAM annotation in the bio.tools entry, not with `edam_topics`/`edam_operations` elements in the XML -- the standards call that not recommended. ```xml @@ -123,9 +171,46 @@ Link to the upstream tool's bio.tools entry with ``. Create the bio.tools ``` +### `` + +**Standard:** [Required files][std-required-files]; how to get it right: [pulsar item 1][pulsar-1]. + +Every file the wrapper ships and reaches through `$__tool_directory__` needs an ``: the entry script, helpers it imports or sources, config files, files copied in with `cp`. Not macros. This applies to any wrapper-shipped file, not only Python. The check is one include per `$__tool_directory__` reference across ``, ``, and ``, with the same relative path, and no dead includes. + +Files under `test-data/` and `tool-data/` are excluded by default, so a script or fixture the job needs at runtime cannot live there. + +### `` + +**Standard:** [Creator][std-creator]. + +Optional. Reviewers invite it ("you can add yourself as creator") but do not require it. It goes last, after ``. Do not invent identities: add a person only when the human you are working for asks, with their real ORCID or URL, and an email only with their explicit consent. + +```xml + + + + +``` + ### Command Block -Use `detect_errors="aggressive"` (catches non-zero exit codes and "error:"/"exception:" on stderr) and CDATA wrapping. Chain multiple commands with `&&` for proper error propagation. Most tools call the upstream binary directly: +**Standard:** [Command tag][std-command], [Exit Code Detection][std-exit]. + +Tripwire: every text parameter, input path, and output path goes in **single** quotes. Double quotes are not equivalent. See Section 4 and [security item 1][sec-1]. + +**`detect_errors`:** `aggressive` is the default. It fails the job on a non-zero exit code or when `error:` or `exception:` appears in the tool's output (case-insensitive; the standards say stderr, and Galaxy's `stdio.py` checks stdout as well). A line that only says `WARNING` is not a trigger. Use `exit_code` instead in two cases, and put a comment above `` saying which one applies ([PR #95][std-pr95]): + +- the tool prints `error:` or `exception:` text on stderr during normal, successful operation; +- reporting errors is the tool's job (validators, linters, QC reporters). + +"The tool is chatty" is not a reason by itself. Check the actual stderr of a successful run for the two trigger strings before switching, and check what sibling tools for the same upstream do. + +```xml + + ``` -When the CLI can't produce the output Galaxy needs (format conversion, multi-step pipelines), use a Python wrapper script: +When the CLI can't produce the output Galaxy needs (format conversion, multi-step pipelines), use a Python wrapper script and declare it in ``: ```xml ``` +**`version_command`:** use the tool's real version command (`mytool --version`). Reviewers always ask for the real command. `echo @TOOL_VERSION@` is a fallback only for tools with no way to print a version. Wrap it in CDATA when it contains `&` or `<`, for example when the version goes to stderr: + +```xml +&1 | head -n 1]]> +``` + **Parameter parity rule:** Every param in `` must appear in ``, and every flag in `` must trace back to an `` param or a macro token. Orphaned params are a lint warning and a review flag. -**Index generation:** When a tool needs to index input files, create symlinks to the inputs in the working directory — don't try to write indices next to the (read-only) input files. +**Index generation:** [Generating Indices][std-indices]. Symlink inputs into the working directory and index there; inputs are read-only. + +**Composite datatypes:** inputs use `$input.extra_files_path`, outputs use `$output.files_path` ([Command tag][std-command]). ### Output Paths and `from_work_dir` +**Standard:** [pulsar items 5 and 6][pulsar-5]. Outputs are real files inside the job directory. Never `ln -s` an output to an input or to anything outside the job tree, and never write a discovered output to an absolute path. + When a CLI tool writes to a fixed filename or prefix, use a staging directory so Galaxy can find the output predictably: ```xml @@ -186,6 +281,8 @@ Or use `from_work_dir` on the output: ``` +HTML, SVG, and JavaScript outputs are an administrator trust decision, not a free choice. See [security item 10][sec-10] before emitting one. + ### Conditional Outputs (Filters) Use `` to create outputs only when certain params are set: @@ -202,7 +299,7 @@ Set `expect_num_outputs` to the number of outputs actually produced by each test ### Dynamic Output Discovery -For tools producing variable numbers of output files, use `discover_datasets`: +For tools producing variable numbers of output files, use `discover_datasets`. The `directory` must be inside the job working directory: ```xml @@ -212,28 +309,55 @@ For tools producing variable numbers of output files, use `discover_datasets`: ### Help Section -Use `format="markdown"` for new tools (preferred over RST). Structure with bold `**headers**`, horizontal rules `-----` between sections, double backticks for code references, and end with a citation block. Keep it concise and actionable. +**Standard:** [Help tag][std-help], which describes reStructuredText only at the pinned commit; Markdown as the preferred format for new tools is proposed in [PR #95][std-pr95]. Reviewers already offer it ("you can use markdown if this is any easier") and nobody asks for conversion either way. + +Pick the format by this table, then write in that format's idioms only: + +| Situation | Format | +|---|---| +| New tool, no help images | Markdown: ``. Galaxy supports it from 24.2, so the tool's profile must be >= 24.2 | +| New tool that ships help images | RST. Galaxy rewrites `.. image::` paths only for RST help (still true in 26.2), so images do not resolve in Markdown help | +| Editing an existing RST tool | Keep RST. Convert only if you convert the whole help section, and only when the tool has no images. Existing RST tools need not convert | +| Target repo or brief asks for RST | RST, and note it in the hand-back | + +Markdown idioms: `##` / `###` headings, single backticks for inline code, fenced blocks for examples, pipe tables, and a blank line before every list (reviewers flag a missing one). RST idioms: `**bold**` pseudo-headers or underlined titles, `-----` rules between sections, double backticks for code, `.. image::` for images. Do not mix them: `-----` rules and double backticks in a `format="markdown"` section are the usual mistake. + +Either way: wrap in CDATA, say what the tool does, what each input and output is, and keep it concise. Citations go in ``, not the help text. For images and diagrams see [Including Images][std-images] and `references/tool-help-diagrams.md`. --- ## 3. macros.xml Patterns +**Standard:** [Tool versions][std-versions] (tokens), [integration_checklist][std-checklist] (a macros file is for avoiding duplication). + +### Tokens Are Required, the File Is Not + +The standards require the version *tokens*, not a `macros.xml` *file*. Reviewers ask for shared things to go in macros, and question a macro file when nothing is reused. + +| Tool directory | Where tokens and macros go | +|---|---| +| One tool, nothing shared | Inline `......` in the tool XML | +| Two or more tools, or anything reused | Shared `macros.xml`, imported with `macros.xml` | + +Follow the target repo if its single-tool directories do it differently. + ### Tokens vs XML Macros -This is the single most common IUC review comment. Get it right from the start. +Get this right from the start. **``** — For simple text substitution and Cheetah snippets. Expanded inline with `@NAME@` syntax. Use for version strings, Cheetah command fragments. **``** — For structured XML element trees. Expanded with ``. Use for parameters, requirements, citations. -**The rule:** If it contains Cheetah template logic (`#if`, `#for`), it MUST be a ``, never an ``. Cheetah inside `` macros does not work correctly. +**The convention:** anything containing Cheetah template logic (`#if`, `#for`) goes in a ``, not an ``. Reviewers ask for this. (An `` macro body is parsed as XML, so a bare Cheetah fragment there is fragile at best; this skill has not verified exactly how it fails.) ```xml 1.2.3 0 - 25.0 + + 24.2 - + #if str($optional_param).strip() --optional-flag '$optional_param' @@ -309,7 +433,9 @@ Use **token parameterization** on xml macros to pass values into the macro at ex The following macros are only needed when wrapping external APIs (not CLI tools). -**Credentials** — Galaxy's vault-backed secrets system. Place `` inside `` in the macro, not as a separate macro: +**Standard:** [security item 8][sec-8]. Secrets never appear in tool XML, test fixtures, command-line arguments, logs, or a templated ``. The tool reads them from the environment. + +**Credentials** -- Galaxy's vault-backed secrets system (`` arrived in Galaxy 25.1, so it needs a server at 25.1 or later). Place `` inside `` in the macro, not as a separate macro: ```xml @@ -322,7 +448,7 @@ The following macros are only needed when wrapping external APIs (not CLI tools) ``` -Tools then just use `` — no yield or separate credentials expand needed. +Tools then just use `` -- no yield or separate credentials expand needed. Do not expand `\$MYTOOL_API_KEY` onto the command line; let the script read the environment. **Test fixture param** — hidden param for fixture-based CI testing (bypasses API calls): @@ -332,29 +458,62 @@ Tools then just use `` — no yield or separate cr ``` +Three things this pattern needs: + +- Fixtures live in a directory the job can reach (`fixtures/` here), declared in `` (``). Files under `test-data/` are excluded from staging by default, so a fixture read through `$__tool_directory__/test-data/...` is missing under remote execution. +- Recorded fixtures are scrubbed before commit (Section 8). +- The script accepts only a bare filename for `--test-fixture` and rejects path separators. + +> **Verify:** whether a `type="hidden"` param can be set by a user through the API. If it can, `$test_fixture` is a user-controlled value concatenated into a tool-directory path, which is why the script-side filename check above is not optional. This skill has not confirmed it either way. + --- ## 4. Cheetah Templating +**Standard:** [security items 1, 4, 5, 6][std-security]; [Command tag][std-command]. Cheetah should be indented and mainly PEP 8 conformant ([Coding Style][std-style]). + ### Dollar Sign Escaping This causes more bugs than anything else in Galaxy XML. The rules: | Context | Syntax | Example | |---------|--------|---------| -| Galaxy parameter | `'$param'` | `--input '$input_file'` | -| Galaxy parameter (no quotes needed) | `$param` | `--count $max_count` | +| Text param, dataset path, output path, collection value | `'$param'` | `--input '$input_file'` | +| `integer` / `float`, boolean `truevalue`/`falsevalue`, fixed-option `select` only | `$param` | `--count $max_count` | | Shell environment variable | `\${VAR}` | `--workers \${GALAXY_SLOTS:-1}` | | Cheetah loop variable | `'$item'` | `'$item'` inside `#for` | | Literal dollar in shell | `\$` | `awk '{print \$1}'` | -**Critical:** Galaxy parameters are single-quoted (`'$param'`). Environment variables use backslash-escaped dollar with braces (`\${VAR:-default}`). +**Critical:** Galaxy parameters are single-quoted (`'$param'`). Double quotes (`"$param"`) are not a substitute: the shell still expands inside them ([security item 1][sec-1]). An unquoted parameter is acceptable only when its type or finite value set makes it safe, as in row two; a `select` with dynamic options or a `text` param never qualifies. Environment variables use backslash-escaped dollar with braces (`\${VAR:-default}`). + +### Nested Contexts and Configfiles + +**Standard:** [security item 1][sec-1] (nested interpreters), [item 6][sec-6] (data, not code). + +Single quotes protect a value from the outer shell only. A parameter placed inside `bash -c '...'`, an `awk` or `sed` program, `Rscript -e`, `python -c`, or a generated script is parsed again by that inner language, and the outer quoting does nothing there. Do not interpolate parameters into program text. Hand them over as data: + +```xml + + + + +``` + +`` writes every parameter to a JSON file that the script parses, so no user value is ever program text. A templated `` is acceptable only when its content is consumed strictly as data (a key/value table, a config format). A `` that renders R, Python, or shell with `$param` inside it is generated code and has the same problem as `bash -c`. Never offer `eval`, a script-body param, or a free-text "extra options" param. Secrets never go in a configfile. + +### Keep Cheetah Side-Effect Free + +**Standard:** [security item 5][sec-5], [pulsar items 2 and 3][pulsar-2]. + +Cheetah in `` and `` runs on the Galaxy server before the job starts, with Galaxy's privileges and the server's filesystem. In a template: no `#import os` / `subprocess` / `pathlib` / `glob`, no `open(`, no `os.path.*` on data paths, no `$__app__`. Use dataset metadata (`$input.get_size()`, `$input.metadata.*`, `$input.is_of_type(...)`, `$input.element_identifier`). When metadata is not enough, do the work at runtime: a shell glob in the command, or plain script text in a configfile. `#import re` and `#import json` for pure string work are fine. ### Conditional Parameters @@ -366,6 +525,8 @@ For text params, always use `str()` and `.strip()` — never test the raw param #end if ``` +Convert with `str()` **before** calling any method (`.split()`, `.strip()`, joins, formatting), then single-quote each resulting shell argument: method calls on the wrapped value can bypass sanitization ([security item 4][sec-4]). The aim is a value sanitized once for the context that consumes it, not `str()` everywhere. + For select params, use string comparison: ``` @@ -376,19 +537,30 @@ For select params, use string comparison: ### Boolean Parameters -Galaxy booleans render as lowercase strings `"true"` or `"false"`. Use string equality, not Pythonic truthiness: +**Standard:** [Booleans][std-booleans]. + +Put the CLI flag in `truevalue`/`falsevalue` (Section 5) and emit the parameter directly. No `#if` is needed: + +``` +mytool $gzip '$input' +``` + +`str($flag)` is whatever `truevalue`/`falsevalue` say, so it is `"true"`/`"false"` only when those attributes are left at their defaults. Comparing `str($flag) == "true"` on a boolean with `truevalue="--gzip"` is always false. When you need to branch on a boolean, test it directly: ``` -## BAD: Pythonic boolean test — unreliable in Cheetah +## GOOD: works whatever truevalue is #if $my_flag -#if $my_flag is True + --enable-feature --feature-level 3 +#end if -## GOOD: string comparison +## BAD once truevalue/falsevalue are set: the string is the flag, never "true" #if str($my_flag) == "true" - --enable-feature -#end if ``` +The standards use both idioms (the licensing page has `#if not $non_commercial_use` and a string comparison on a default-valued boolean), and neither is declared canonical. Follow sibling tools in the target repo. + +> **Verify:** `#if $flag` is truthy by the checked state in current Galaxy (the parameter wrapper's `__bool__` returns `bool(value)`). This skill has not checked it against old profiles. + ### Numeric Zero Gotcha Integer and float params: `0` is a valid value but falsy in Cheetah/Python. Use `str()` to avoid skipping zero: @@ -439,10 +611,20 @@ In the command block: #end if ``` +### Field Gotchas + +Seen on Galaxy 25.1 during wrapper work; not in the standards. + +- **Generator expressions do not compile in Cheetah.** `#if any(x for $b in $barcodes)` fails. Use an explicit `#for` loop that sets a flag. +- **`find ... -exec ... \;` gets mangled in a command block.** Use a shell glob or a `for f in dir/*; do ...; done` loop instead (which is also the remote-execution-safe way to enumerate files). +- **`re` is not available until you import it.** `#import re` first, or `re.sub(...)` raises `NameError` (seen in a configfile). + --- ## 5. Input Parameter Conventions +**Standard:** [Parameter name, argument and help][std-argument], [Booleans][std-booleans], [Dynamic Options][std-dynamic], [Data parameters][std-data-params], [security item 2][sec-2]. + ### The `argument=` Attribute Prefer `argument="--flag"` over bare `name="flag"`. This auto-generates the `name` attribute (stripping leading dashes, replacing `-` with `_`) and displays the flag in the help text. Always use the long form (`--output` not `-o`). @@ -456,11 +638,7 @@ Prefer `argument="--flag"` over bare `name="flag"`. This auto-generates the `nam ``` -When using `argument=`, the param `name` is derived automatically (e.g., `argument="--min-score"` creates `name="min_score"`). You still need to include the flag in the command block — `argument=` does not inject it for you: - -``` ---min-score $min_score -``` +`argument=` does not inject the flag. You still write `--min-score $min_score` in the command block. **IUC standard attribute order:** `name, argument, type, format, min|truevalue, max|falsevalue, value|checked, optional, label, help`. @@ -483,9 +661,19 @@ Group related parameters into logical sections for complex tools: ``` +Params in a collapsed `
` still have values and still reach the command. Params in a non-selected `` of a `` do not ([Dynamic Options][std-dynamic]). + ### Validation -Use `` for constraining values. Use `min`/`max` attributes on integer/float params. Never use `optional="true"` when a default is appropriate — just set the default. +**Standard:** [security item 2][sec-2]. + +Order of preference: a typed parameter (`integer`, `float`, `boolean`, `select`, `data_column`) over free text; a `select` whenever the valid values are a finite set; free text only when nothing else describes the input, and then with a validator. + +Tripwire: Galaxy regex validators match from the start only. **Anchor the end** (`^...$`) or the validator accepts `good;anything`. An unanchored pattern with `*` accepts every string. + +Validator vs sanitizer: a validator rejects a bad value before the job starts and tells the user why; a sanitizer silently rewrites characters as the value is rendered. Prefer the validator. The default sanitizer is not shell escaping, and a custom `` that lets `'` through breaks the `'$value'` quoting boundary. Validation complements quoting; it does not replace it. + +Use `min`/`max` attributes on integer/float params. Never use `optional="true"` when a default is appropriate -- just set the default. ```xml @@ -495,10 +683,10 @@ Use `` for constraining values. Use `min`/`max` attributes on integer - + - [A-Za-z0-9:, ]* + ^[A-Za-z0-9:, ]*$ @@ -506,6 +694,8 @@ Use `` for constraining values. Use `min`/`max` attributes on integer label="Maximum items" help="Start small to verify results"/> ``` +Every free-text param gets a hostile-input test (`references/testing.md`). + ### Boolean Parameters Put the CLI flag in `truevalue`/`falsevalue`. Don't use booleans as conditionals — use `select` instead when other params depend on the choice. @@ -521,14 +711,25 @@ Put the CLI flag in `truevalue`/`falsevalue`. Don't use booleans as conditionals ### Compressed Datatype Support -When the underlying tool accepts compressed input natively (or you add decompression in the command section), accept both compressed and uncompressed formats: +**Standard:** [Data parameters][std-data-params]. + +If Galaxy has a compressed version of a datatype (`fasta.gz`, `fastqsanger.bz2`), the data param accepts both the compressed and the uncompressed format. If the underlying tool cannot read the compressed form, decompress in ``. Galaxy does not decompress for you. Reviewers ask "are fasta.gz supported?" as a matter of course. ```xml ``` -If you accept compressed formats, include tests with both compressed and uncompressed inputs to verify both paths work. +``` +#if $input.is_of_type('fasta.gz') + gunzip -c '$input' > input.fa && +#else + ln -s '$input' input.fa && +#end if +mytool input.fa +``` + +Include tests with both compressed and uncompressed inputs to verify both paths work. ### Multiple Inputs vs Repeat Parameters @@ -558,27 +759,69 @@ For paired-end data and multi-sample workflows: - + ``` Access paired ends: `$paired_input.forward` / `$paired_input.reverse` -Preserve element identifiers in loops: +### Identifiers as Filenames + +**Standard:** [security item 3][sec-3]. + +`element_identifier`, dataset names, and labels are user-influenced strings. Tripwire, in order of preference: + +1. Use a **fixed filename** (`ln -s '$input' input.bam`) unless the tool needs the identifier in the name (for example because it becomes a sample name in the output). +2. If it does, import `re`, convert with `str()`, map to a small alphabet, and **handle collisions**: two identifiers can sanitize to the same name. Add a stable unique suffix or fail. +3. Single-quote the result wherever it reaches the shell. ``` -#for $input in $inputs ---name '${re.sub('[^\w\-_]', '_', $input.element_identifier)}' +#import re +#for $i, $input in enumerate($inputs) + #set $safe_id = re.sub(r'[^\w.-]', '_', str($input.element_identifier)) + ln -s '$input' '${safe_id}_${i}.bam' && #end for ``` +The loop index is the unique suffix here. If the suffix would leak into output the user sees, detect duplicates and `exit 1` with a clear message instead. + +### Reference Data and Data Tables + +**Standard:** [integration_checklist][std-checklist] ("built-in data", "tool-data" bullets); [pulsar items 3 and 4][pulsar-3]. + +Reference data (indexes, databases) comes from a data table, never a hardcoded path and never `$__app__.tool_data_tables`: + +```xml + + + + + +``` + +In the command, use `'$reference.fields.path'` directly. Do not copy it through `str()` into a `#set` first: that bakes a server-side path into the command that remote execution cannot rewrite. + +Tripwire: a tool that uses a data table ships **four** files, and missing ones are a routine review finding. + +| File | Content | +|---|---| +| `tool-data/mytool_indexes.loc.sample` | Comment-only: column layout and an example row, all `#`-prefixed | +| `tool_data_table_conf.xml.sample` | The `
` definition pointing at `tool-data/mytool_indexes.loc` | +| `tool_data_table_conf.xml.test` | Same table, with `` | +| `test-data/mytool_indexes.loc` | Real tab-separated rows for the tests; paths use `${__HERE__}/...` | + +> **Verify:** the checklist calls `tool_data_table_conf.xml.sample` "comment-only" as well, but existing tools-iuc tools (for example `tools/bowtie2`) ship it with a live `
` definition. This skill follows the tools-iuc form. Copy a sibling tool's files rather than writing from scratch. + +Test setup is in `references/testing.md`. A tool that *populates* a data table is a data manager: see [data_managers][std-dm]. + ### Subcommand Strategy -Tools with subcommands (e.g., `samtools view`, `samtools sort`) should be separate tool wrappers when the subcommands need different resource allocations. Use a conditional only when they're closely related and share the same resource profile. +**Standard:** [Wrapping tools with subcommands][std-subcommands]. Separate wrappers when subcommands need different resources; a conditional only when they are closely related and share a resource profile. ### IUC-Specific Rules -See [IUC Best Practices](https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices.html) for the canonical standards. +Reviewer habits and style points. The first, third, and fifth are skill-side observations from review, not standards text. - **No `display="checkboxes"`** on multi-select params. Reviewers will flag it. Let Galaxy decide the widget. - **Mark defaults with `selected="true"`** on the `
+ value, dbkey, name, path + +
+ +``` + +`test-data/mytool_indexes.loc`, tab-separated, one row per test index, with the index itself committed under `test-data/`: + +``` +test_index hg38_test Test index ${__HERE__}/mytool_index +``` + +The test selects the row by its `value` column: + +```xml + + + + + +``` + +`planemo test` picks up `tool_data_table_conf.xml.test` from the tool directory by itself. The same tool also ships `tool_data_table_conf.xml.sample` and `tool-data/mytool_indexes.loc.sample` for real installs (see `../SKILL.md` Section 5). Copy a sibling tool's four files rather than writing them from memory; `tools/bowtie2` in tools-iuc is a working example. + +--- + +## Hostile-Input Test + +**Standard:** [security item 2](https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices/security.html#security-free-text): test punctuation, whitespace, quotes, newlines, empty values, and boundary lengths, not only the default. + +Every free-text parameter gets two kinds of test. + +**Rejected.** A value outside the validator's language never reaches the shell. The test expects failure and has no output checks: + +```xml + + + + +``` + +Useful hostile values: `a;b`, `$(id)`, `` `id` ``, `a'b`, `a"b`, `a b`, `../x`, `-x` (looks like a flag), and a trailing-junk value such as `good;rm` that only an end-anchored validator rejects. If this test *passes without* `expect_failure`, the validator is not anchored. + +**Accepted and intact.** The awkward values the validator does allow (spaces, dots, the longest reasonable value, the empty string for an optional param) arrive as one argument, unchanged: + +```xml + + + + + + + + + + + + +``` + +> **Verify:** a validator rejection happens before a job exists. Current Galaxy counts that as the expected failure for `expect_failure="true"` (checked in the test interactor of a 26.1 development checkout); this has not been checked on older Galaxy or planemo versions. If the rejected-value test errors instead of passing, say so in the hand-back rather than deleting the test. + +--- + +## Gotchas + +Field findings from real wrapper work. None of these is in the IUC standards, and most are not in the Galaxy or planemo docs either. Each entry says when to delete it. + +### A test `` only binds to a `data_collection` param + +A `type="data" multiple="true"` param accepts a list collection in the UI, but a test-side `` block cannot feed it. Give it a comma-separated list of files instead: + +```xml + + + + + + + + + + +``` + +*Delete when the planemo or Galaxy test docs state which param types a test `` can bind to.* + +### Unsniffable inputs need an explicit `ftype` + +Galaxy guesses the datatype of each uploaded test file. Files it cannot sniff (Salmon `quant.sf`, custom tabular or text formats) can end up with a type the param does not accept. Set it: + +```xml + +``` + +*Delete when the planemo docs cover `ftype` on test inputs.* + +### `has_line_matching` is anchored at both ends; `has_text` is literal + +`has_line_matching` wraps the expression as `^expression$` per line, so `expression="chr1"` does not match the line `chr1\t100`. Write `chr1\t.*`, or use `has_text_matching` for an unanchored search. `has_text` is a literal substring match: regex metacharacters in `text=` are not interpreted. + +*Delete when the schema docs for these assertions say so explicitly.* + +### `planemo test` fails building the Galaxy virtualenv + +Symptom: the run dies before any test, building packages such as `h5py` or `pyo3`, on a machine whose default `python3` is newer than the Galaxy release supports (seen with Python 3.14 and Galaxy 25.1). Pin the interpreter planemo uses for Galaxy: + +```bash +planemo test --galaxy_python_version 3.12 --biocontainers tools/mytool/ +``` + +This is troubleshooting for a local environment problem, not part of the standard command. Do not add it to documented or CI commands. + +*Delete when planemo picks a supported interpreter by itself or documents this failure.* + +### No mulled image on quay for a multi-requirement tool + +Symptom: `planemo test --biocontainers` cannot find a container for a tool with two or more requirements. The combination has not been registered and built yet (see `../SKILL.md` Section 9). Build it locally, then re-run the test: + +```bash +planemo mull tools/mytool/mytool.xml +planemo test --biocontainers tools/mytool/mytool.xml +``` + +Say in the hand-back that the image was built locally, so the reviewer knows the public image is still pending. + +> **Verify:** `planemo mull` is taken from planemo's own help text ("build containers for specified tools ... Galaxy can automatically discover this container"); it was not run end to end for this skill. The field report used `mulled-build build` directly; its exact arguments were not recorded, so they are not shown here. + +*Delete when the planemo docs describe testing a tool whose mulled container is not published yet.* + +### Python `csv.writer` writes CRLF + +`csv.writer` defaults to `\r\n` line endings. In a wrapper script that produces tabular output, this breaks `$`-anchored assertions and line-based assertions, and confuses downstream tabular tools. Pass `lineterminator="\n"`: + +```python +writer = csv.writer(handle, delimiter="\t", lineterminator="\n") +``` + +*Delete when this is covered in the Galaxy tool-development docs or the IUC standards.* + +### Generator expressions do not compile in Cheetah + +`#if any(str($b.code) for $b in $barcodes)` fails to compile (seen on Galaxy 25.1). The failure shows up as a tool that will not load or a command that cannot be built, not as a test assertion. Use an explicit loop: + +``` +#set $has_code = False +#for $b in $barcodes + #if str($b.code).strip() + #set $has_code = True + #end if +#end for +``` + +*Delete when the Galaxy Cheetah documentation lists unsupported Python constructs.* + +### `find ... -exec ... \;` is mangled in a command block + +Reported on Galaxy 25.1; the exact mechanism was not investigated. Replace it with a shell glob or loop, which also keeps file enumeration at runtime where remote execution needs it: + +``` +mkdir -p out && mv work/*.fastq out/ +for f in work/*.fastq; do gzip "\$f"; done +``` + +*Delete when the Galaxy command-block docs cover escaping of `\;`.* + +### `version_command` needs CDATA when it redirects + +`2>&1` contains `&`, which is not valid in XML text. `` is always in CDATA, so the problem only shows up in `version_command`: + +```xml +&1 | head -n 1]]> +``` + +*Delete when the IUC standards or schema docs show CDATA on `version_command`.* + +--- + ## Analyzing Test Failures ### Quick Summary from tool_test_output.json @@ -315,9 +509,11 @@ When `tool_test_output.json` reports `test-5`, that's the 6th test. - Give each test a **unique purpose** (e.g., "defaults", "compression on", "filtering active") - Always include `expect_num_outputs` to verify file counts -- **Test data under 1 MB** — use assertions instead of golden files for larger outputs +- **Test data: aim for under 1 MB in total.** Shrink inputs first. For large outputs, use assertions or `compare="contains"` instead of a golden file. For a large input that cannot be made smaller, deposit it in the [Galaxy Tool Developers Zenodo community](https://zenodo.org/communities/galaxy_tool_dev_sig) and reference it with `` - Include tests for output filters to verify filtering actually occurs - Include tests for error conditions with expected failure +- Include a hostile-input test for every free-text parameter +- Include a test with compressed input and one with uncompressed input when the tool accepts both --- diff --git a/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-help-diagrams.md b/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-help-diagrams.md index aafdce4..33fa953 100644 --- a/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-help-diagrams.md +++ b/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-help-diagrams.md @@ -2,6 +2,10 @@ Hand-coded SVG schematics that show input → operation → output for Galaxy tools. Embedded in tool XML `` sections via reStructuredText `.. image::` directives. +**Embedding a diagram means RST help.** Galaxy rewrites `.. image::` paths to the tool's `static/images/` directory only for reStructuredText help (still true in 26.2). In `` the image does not resolve. So a tool that ships a help image keeps RST help for the whole section, even though Markdown is otherwise preferred for new tools (`../SKILL.md` Section 2). Do not convert an RST help section that contains images to Markdown. + +**Standard:** [tool_xml: Including Images](https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices/tool_xml.html#including-images). + ## Where Diagrams Live **Built-in tools** (tools bundled with the Galaxy codebase): @@ -26,16 +30,20 @@ tool_collections/bamtools/bamtools_filter/static/images/complex-filters.png ## How to Embed in Tool XML -Inside the `` section (reStructuredText): +Inside an RST `` section (no `format` attribute, or `format="restructuredtext"`; not `format="markdown"`): -**External tools** — use `$PATH_TO_IMAGES` (resolves to the tool's `static/images/` dir): +**External tools** -- reference the bare filename, as the IUC standards do. Galaxy resolves it to the tool's `static/images/` directory: ```rst -.. image:: $PATH_TO_IMAGES/my_diagram.svg +.. image:: my_diagram.svg :alt: Description of the diagram :width: 500 ``` +The older form `.. image:: $PATH_TO_IMAGES/my_diagram.svg` still works: Galaxy strips the `$PATH_TO_IMAGES` prefix before resolving the path, and its own comment calls the prefix no longer required. Most existing tools-iuc help sections still use it. When editing an existing tool, keep whichever form it has; for a new tool use the bare filename. + +> **Verify:** the path rewriting was read from a Galaxy 26.1 development checkout (`set_image_paths`), where it runs for tools installed from a Tool Shed. A local `planemo serve` may therefore show a broken image even when the directive is right. Check a sibling tool with images the same way before concluding the path is wrong. + **Built-in Galaxy tools** — use `${static_path}` (resolves to Galaxy's central static dir): ```rst @@ -180,7 +188,7 @@ When a tool has multiple usage patterns (e.g., build_list): 8. **Save as SVG** in `/static/images/` (external tools) or `static/images/tools//` (built-in tools). -9. **Add `.. image::` directive** to the tool's `` section. +9. **Add `.. image::` directive** to the tool's `` section, which must be RST (see the top of this file). ## Complete Example: Filter Empty @@ -270,5 +278,6 @@ A simple 1-input → 1-output diagram highlighting the removed element: - [ ] Uses 2–3 example elements (not more) - [ ] Bottom annotation explains the transformation - [ ] Saved to `static/images/` (external tools) or `static/images/tools//` (built-in) -- [ ] `.. image::` directive added to tool `` using `$PATH_TO_IMAGES` (external) or `${static_path}` (built-in), with `:alt:` and `:width:` +- [ ] Tool `` is RST, not `format="markdown"` +- [ ] `.. image::` directive added to tool `` using the bare filename (external) or `${static_path}` (built-in), with `:alt:` and `:width:` - [ ] Renders correctly in a browser before committing diff --git a/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-placement.md b/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-placement.md index 7c08c2d..f693420 100644 --- a/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-placement.md +++ b/plugins/galaxy-dev-skills/skills/tool-dev/references/tool-placement.md @@ -27,6 +27,7 @@ Where should I create this tool? **Repository**: https://github.com/galaxyproject/tools-iuc **Create here when**: +- The software is under an OSI-approved license and nothing it needs (dependencies, models, reference data) carries a use restriction. This is a gate, not a preference: check it before writing anything. See [Licensing and Use Restrictions](https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices/licensing.html) - Tool is widely used in the community - Tool is well-maintained upstream - Tool would benefit multiple users @@ -43,19 +44,20 @@ Where should I create this tool? 1. Check existing PRs — someone may already be working on it 2. Open an issue to discuss the tool before implementing 3. Fork the repository -4. Create tool following IUC guidelines (see `../SKILL.md`) -5. Submit PR -6. Address review feedback +4. Create tool following the [IUC standards](https://galaxy-iuc-standards.readthedocs.io/en/latest/) (how-to in `../SKILL.md`) +5. Run the four pre-PR passes (`../SKILL.md` Section 10) and put the result in the PR description +6. Submit PR +7. Address review feedback -**IUC requirements**: +**IUC requirements** (summary only; the standards are authoritative, and the target repo's own convention wins where it differs): - Follow directory structure: `tools/toolname/` -- Include `macros.xml` for version tokens +- Version from `@TOOL_VERSION@` / `@VERSION_SUFFIX@` tokens. A shared `macros.xml` only when two or more tools share something; a single tool can keep tokens inline - Include comprehensive tests with `expect_num_outputs` and `assert_contents` -- Test data under 1 MB -- Write clear help section (`format="markdown"`) +- Test data: aim for under 1 MB. Unavoidable large inputs go to the Galaxy Tool Developers Zenodo community and are referenced with `location=` +- Write a clear help section. Markdown (`format="markdown"`, profile >= 24.2) is preferred for new tools; use RST when the tool ships help images or the repo's sibling tools use RST. Existing RST help need not be converted - Add citations (`type="doi"`) - Use `argument=` on CLI-mapped params -- Bioconda/conda-forge dependencies +- Bioconda/conda-forge dependencies, pinned by version --- @@ -123,9 +125,11 @@ Regardless of where you create the tool: **Required**: valid XML structure, version information, at least one test case, help section, proper input/output definitions. -**Recommended**: multiple test cases, comprehensive help, citations, macros for version tokens, proper error handling. +**Recommended**: multiple test cases, comprehensive help, citations, version tokens, proper error handling. -**Best practice**: test data < 1 MB, flexible test assertions, clear parameter descriptions, sensible defaults. +**Best practice**: test data under 1 MB where possible, flexible test assertions, clear parameter descriptions, sensible defaults. + +The security and remote-execution passes in `../SKILL.md` Section 10 apply wherever the tool will run on a shared server, not only to tools-iuc submissions. --- diff --git a/plugins/galaxy-skills/.claude-plugin/plugin.json b/plugins/galaxy-skills/.claude-plugin/plugin.json index 51d914a..4eafe2d 100644 --- a/plugins/galaxy-skills/.claude-plugin/plugin.json +++ b/plugins/galaxy-skills/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "galaxy-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Curated Galaxy skills for running an analysis: connecting to a Galaxy server, the MCP tool surface, dataset collections, user-defined tools, workflow reports and turning an ad-hoc history into a reusable workflow. Mirror of galaxyproject/galaxy-skills (skills/).", "author": { "name": "The Galaxy Project", diff --git a/plugins/galaxy-skills/.codex-plugin/plugin.json b/plugins/galaxy-skills/.codex-plugin/plugin.json index d35336e..2d79e83 100644 --- a/plugins/galaxy-skills/.codex-plugin/plugin.json +++ b/plugins/galaxy-skills/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "galaxy-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Curated Galaxy skills for running an analysis: connecting to a Galaxy server, the MCP tool surface, dataset collections, user-defined tools, workflow reports and turning an ad-hoc history into a reusable workflow. Mirror of galaxyproject/galaxy-skills (skills/).", "author": { "name": "The Galaxy Project", diff --git a/plugins/galaxy-skills/.cursor-plugin/plugin.json b/plugins/galaxy-skills/.cursor-plugin/plugin.json index 2648b8f..0031497 100644 --- a/plugins/galaxy-skills/.cursor-plugin/plugin.json +++ b/plugins/galaxy-skills/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "galaxy-skills", - "version": "2026.9.17", + "version": "2026.11.16", "description": "Curated Galaxy skills for running an analysis: connecting to a Galaxy server, the MCP tool surface, dataset collections, user-defined tools, workflow reports and turning an ad-hoc history into a reusable workflow. Mirror of galaxyproject/galaxy-skills (skills/).", "author": { "name": "The Galaxy Project" diff --git a/plugins/galaxy-skills/UPSTREAM.json b/plugins/galaxy-skills/UPSTREAM.json index 67bdf22..979824e 100644 --- a/plugins/galaxy-skills/UPSTREAM.json +++ b/plugins/galaxy-skills/UPSTREAM.json @@ -1,7 +1,7 @@ { "repository": "https://github.com/galaxyproject/galaxy-skills", "ref": "main", - "commit": "7eb5a1dd2e3afed412f0cfebaa39236f36695845", + "commit": "5ec0fea302cae826708e1e198e89f9206886d8f9", "path": "skills", - "synced_at": "2026-09-17T20:49:30Z" + "synced_at": "2026-11-16T06:42:22Z" } diff --git a/plugins/galaxy-skills/skills/udt-authoring/SKILL.md b/plugins/galaxy-skills/skills/udt-authoring/SKILL.md index d86766e..504a15e 100644 --- a/plugins/galaxy-skills/skills/udt-authoring/SKILL.md +++ b/plugins/galaxy-skills/skills/udt-authoring/SKILL.md @@ -51,7 +51,7 @@ If you find yourself writing `command:`, `$param`, `#for#`, `truevalue:`, or `${ ```yaml class: GalaxyUserTool # required, exactly this id: my-tool # lowercase, ^[a-z][a-z0-9_-]*$, 3-255 chars (recommended) -version: "0.1.0" # required -- schema validation rejects a versionless tool +version: "0.1.0" # required, and must be PEP 440 or create is refused name: My Tool # required, min 5 chars description: One line shown in the tool menu container: quay.io/biocontainers/seqkit:2.8.2--h9ee0642_0 # required STRING, a real image @@ -188,7 +188,11 @@ check offline before submitting. Three tiers, weakest to strongest: There is no in-place update API -- every `create_user_tool` call makes a **new** tool. When you iterate, bump `version` (e.g. `0.1.0` -> `0.2.0`); Galaxy keeps every prior version under the user's -account. Heavy iteration accumulates stale versions, so once a design stabilizes, offer to deactivate +account. **Keep the bumped version PEP 440**: create is refused with +`400 Tool failed lint checks: ToolVersionPEP404` for anything else, so a descriptive scratch version +like `0.1.0-probe` fails while `0.1.0.dev1` or `0.1.0+probe1` works. This is the moment the mistake +happens, because a name is the natural thing to type for a throwaway -- see +`references/schema-reference.md`. Heavy iteration accumulates stale versions, so once a design stabilizes, offer to deactivate the superseded ones with `delete_user_tool` -- it marks the tool inactive (hidden from the toolbox), not a hard delete; the records remain. diff --git a/plugins/galaxy-skills/skills/udt-authoring/references/common-mistakes.md b/plugins/galaxy-skills/skills/udt-authoring/references/common-mistakes.md index dc97ba1..c4e388c 100644 --- a/plugins/galaxy-skills/skills/udt-authoring/references/common-mistakes.md +++ b/plugins/galaxy-skills/skills/udt-authoring/references/common-mistakes.md @@ -14,6 +14,7 @@ plus a few schema-specific traps. Run through this before submitting. - [ ] If `shell_command` runs a script by name (`python script.py`), a `configfiles` entry creates that exact filename -- otherwise inline it with `python -c` / `Rscript -e`. - [ ] No `min`/`max` on a `data` input (it's already required; use `optional: true` to relax, `multiple: true` for a list). - [ ] `id` matches `^[a-z][a-z0-9_-]*$`; `name` is at least 5 characters. +- [ ] `version` is **PEP 440** (`0.2.0`, `0.1.0.dev1`, `0.1.0+probe1`) -- a bare `-suffix` like `0.1.0-probe` is not, and create is refused. - [ ] No rejected fields: `truevalue`, `falsevalue`, `argument`, `parameter_type`, `${on_string}`, `${tool.name}`. - [ ] Booleans become flags via a ternary, not `truevalue`/`falsevalue`. - [ ] The parallelism flag's **value** is `$GALAXY_SLOTS` (+ a `resource` requirement), not a hardcoded number -- and not merely echoed to a log while the flag stays a literal. @@ -39,6 +40,7 @@ plus a few schema-specific traps. Run through this before submitting. | `truevalue: --x` / `falsevalue: ""` on a boolean | XML-only fields, rejected. | `value: false` + `$(inputs.x ? '--x' : '')` in the command. | | `${on_string}`, `${tool.name}` in labels | Cheetah macros; not supported. | Use a plain string label, or `format_source` to inherit. | | `id: My_Tool` / `id: 2pass` | Must be lowercase and start with a letter (`string_pattern_mismatch`). | `id: my-tool`, `id: two-pass`. | +| `version: "0.1.0-probe"` (or any non-PEP-440 string) | Create is **refused**: `400 Tool failed lint checks: ToolVersionPEP404`. Galaxy's linter records this as a *warning*, so nothing about it reads as fatal -- but a lint finding rejects the create, and the error names a linter class rather than the field. Most likely when inventing a version for a throwaway registration. | A release (`0.2.0`), a dev release (`0.1.0.dev1`) or a local version (`0.1.0+probe1`). PEP 440's only legal `-x` is `-`, an implicit post-release, so `0.1.0-1` works and `-probe` is not a version at all. `scripts/validate.py` catches it (exit 2). | | Hardcoded `--threads 8` -- even while echoing `$GALAXY_SLOTS` elsewhere | Ignores the job's real allocation; recording the value isn't using it. | Make the flag's value `$GALAXY_SLOTS` (`--threads $GALAXY_SLOTS`) + `resource` `cores_min`. | | `container: ubuntu:latest` for a bioinformatics tool | Generic image won't have the binary; not reproducible. | Use the tool's biocontainer. | | Invented image/tag/flags | A guessed container or CLI flag fails at runtime. | Use only images/flags you can verify; otherwise ask. | diff --git a/plugins/galaxy-skills/skills/udt-authoring/references/schema-reference.md b/plugins/galaxy-skills/skills/udt-authoring/references/schema-reference.md index 903a33c..39818af 100644 --- a/plugins/galaxy-skills/skills/udt-authoring/references/schema-reference.md +++ b/plugins/galaxy-skills/skills/udt-authoring/references/schema-reference.md @@ -153,3 +153,28 @@ Beyond type/shape checks, `UserToolSource` runs these. Error `code`s are stable `id` violations surface as Pydantic's `string_pattern_mismatch`; a too-short `name` as `string_too_short`. + +### The schema is not the whole gate: create also runs tool lint + +Passing `UserToolSource` is necessary and not sufficient. `create` lints the tool as well and +**refuses on a lint finding**, answering `400 Tool failed lint checks: `. The one that +catches people is the version: + +``` +400 {"err_msg":"Tool failed lint checks: ToolVersionPEP404: Tool version [0.1.0-probe] + is not compliant with PEP 440.","err_code":400008} +``` + +Two things make this worth knowing in advance. The rule is not in the table above, so a +schema-clean definition can still be rejected; and Galaxy's own linter records it with +`lint_ctx.warn` (`galaxy/tool_util/linters/general.py::ToolVersionPEP404`), so every habit around +linters says it is advisory -- while for a user-defined tool it is fatal. The error names a linter +class rather than the field, which does not point at the version string you just typed. + +Use a release (`0.2.0`), a dev release (`0.1.0.dev1`) or a local version (`0.1.0+probe1`) -- +PEP 440's only legal `-x` is `-` (an implicit post-release), so `0.1.0-1` is valid and +`-probe` is not a version at all. This matters most for a **throwaway** registration, since there +is no update: every create makes a new tool, so a scratch version gets invented precisely when a +name is the natural thing to type. + +`scripts/validate.py` runs the same lint and exits 2 on findings, so it catches this offline.