From 866ba558b0a8ddb93d433c26e9b6895b562ccad4 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 30 Oct 2026 12:00:26 +0000 Subject: [PATCH 1/3] test: make E2E reruns idempotent, read previous and Coder logs, add MCP failure rows (#390) Change-Id: I238039e70250ec6c8023a1928d747b92c86510ba Signed-off-by: Thomas Kosiewski --- .github/workflows/ci.yaml | 6 ++ hack/e2e-workspace-lifecycle.sh | 60 +++++++++++++++----- hack/e2e-workspace-lifecycle_test.sh | 82 ++++++++++++++++++++++------ 3 files changed, 115 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5a1860ea..fba48316 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -720,6 +720,12 @@ jobs: echo "=== Coder deployment logs ===" kubectl -n coder logs deploy/coder --tail=200 || true echo "" + echo "=== CoderMCPServers ===" + kubectl -n coder get codermcpservers -o yaml || true + echo "" + echo "=== MCP stub logs ===" + kubectl -n coder logs deploy/mcp-stub --tail=200 || true + echo "" echo "=== CNPG controller logs ===" kubectl -n cnpg-system logs deploy/cnpg-controller-manager --tail=200 || true echo "" diff --git a/hack/e2e-workspace-lifecycle.sh b/hack/e2e-workspace-lifecycle.sh index d5e0037d..d7312e02 100755 --- a/hack/e2e-workspace-lifecycle.sh +++ b/hack/e2e-workspace-lifecycle.sh @@ -109,6 +109,27 @@ coder_status() { # : a GET without -f that prints the HTTP status; the bod curl -sS --max-time 30 -o "$WORK/coder-raw.json" -w '%{http_code}' -H @"$WORK/coder.hdr" "$CODER_URL$1" } CODER_NOT_FOUND="Resource not found or you do not have access to this resource" # Coder's own 404 message (#325) +# coder_ensure : POSTs the body unless the collection holds an object +# with that key, which a failed run leaves when it keeps the namespace (#390); prints the object. +coder_ensure() { + local found + found=$(coder_api GET "$1" | jq -c --arg k "$2" --arg v "$3" '[.[] | select(.[$k] == $v)] | if length < 2 then .[0] else error("duplicate") end') || + return 1 + [[ $found == null ]] || { log "$4 $3 exists in Coder from an earlier run: reusing it" >&2 && printf '%s\n' "$found" && return 0; } + coder_api POST "$1" "$5" +} +# pod_logs : the log of every container of the non-terminating pods. After a restart it adds the log +# of the previous container, so the checks also see the lines from before the restart (#156). +pod_logs() { + local rows p c n + rows=$(k -n "$1" get pods -l "$2" -o json | jq -er '[.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name as $p | + .status.containerStatuses[]? | "\($p) \(.name) \(.restartCount)"] | if length > 0 then .[] else error("no containers") end') || return 1 + : >"$3" + while read -r p c n; do + if ((n > 0)); then k -n "$1" logs "$p" -c "$c" --previous >>"$3" || return 1; fi + k -n "$1" logs "$p" -c "$c" >>"$3" || return 1 + done <<<"$rows" +} # build_done : true once the build reached status; fails on terminal failure. build_done() { @@ -596,7 +617,11 @@ new_key() { # : 48 random hex characters key_secret() { # : an Opaque Secret for $AI_ORIGIN that holds the key file jq -n --arg n "$2" --arg ns "$NS" --rawfile v "$WORK/$3" --argjson l "$4" --arg o "$AI_ORIGIN" '{apiVersion: "v1", kind: "Secret", metadata: {name: $n, namespace: $ns, labels: $l, annotations: {"coder.com/credential-destinations": $o}}, type: "Opaque", - data: {"api-key": ($v | @base64)}}' >"$WORK/secret-$2.json" && k "$1" -f "$WORK/secret-$2.json" >/dev/null + data: {"api-key": ($v | @base64)}}' >"$WORK/secret-$2.json" || return 1 + k "$1" -f "$WORK/secret-$2.json" >/dev/null 2>"$WORK/secret.err" && return 0 + if [[ $1 != create ]] || ! grep -q '(AlreadyExists)' "$WORK/secret.err"; then cat "$WORK/secret.err" >&2 && return 1; fi + log "Secret $2 exists from an earlier run: replacing it" # a failed run keeps its Secrets in a kept namespace (#390) + k replace -f "$WORK/secret-$2.json" >/dev/null } ai_provider() { # [baseURL]: kubectl apply of an openai-compat CoderAIProvider jq -n --arg n "$1" --arg ns "$NS" --arg s "$2" --arg k "$3" --arg u "${4:-$AI_ORIGIN/v1}" '{apiVersion: "coder.com/v1alpha1", @@ -656,15 +681,16 @@ rv_recorded() { ai_state "$AI" && jq -e --arg rv "$SECRET_RV" '.status.credentia wait_until "AI provider $AI to record Secret resourceVersion $SECRET_RV" rv_recorded ai_matches ai-key-2 "E2E $AI" || fail "Coder provider $AI_ID lost the second key: $(ai_coder_seen)" [[ $AI_KEY_ID == "$ROTATED_KEY_ID" ]] || fail "a metadata-only Secret change sent a new key ($ROTATED_KEY_ID -> $AI_KEY_ID)" -k -n "$OP_NS" logs deploy/coder-k8s >"$WORK/operator-ai.txt" || fail "cannot read the operator log" +pod_logs "$OP_NS" "$OP_SELECTOR" "$WORK/operator-ai.txt" || fail "cannot read the operator log" LEAKS=$({ cat "$WORK/ai-key-1" && echo && cat "$WORK/ai-key-2"; } | grep -cFf - "$WORK/operator-ai.txt" || true) [[ $LEAKS == 0 ]] || fail "the operator log holds an AI provider key ($LEAKS lines)" phase ai-rotation step "CoderAIProvider refusals: an unlabeled Secret, the operator token, a foreign origin and a taken name write nothing to Coder (plan 8.2 P3)" T0=$SECONDS -coder_api POST /api/v2/ai/providers "$(jq -nc --arg u "$AI_ORIGIN/v1" '{type: "openai-compat", name: "e2e-taken", base_url: $u, enabled: false}')" \ - >/dev/null || fail "cannot create provider e2e-taken through the Coder API" +coder_ensure /api/v2/ai/providers name e2e-taken "AI provider" \ + "$(jq -nc --arg u "$AI_ORIGIN/v1" '{type: "openai-compat", name: "e2e-taken", base_url: $u, enabled: false}')" >/dev/null || + fail "cannot create provider e2e-taken through the Coder API" new_key ai-key-3 || fail "cannot generate the third AI provider key" key_secret create e2e-ai-key-unlabeled ai-key-3 '{}' || fail "cannot create Secret e2e-ai-key-unlabeled" AI_BEFORE=$(ai_list) || fail "cannot list the Coder AI providers" @@ -793,9 +819,10 @@ phase gw-secret-not-owned step "CoderAIGatewayKey leak check: no key-shaped value or Secret value in the operator log or the objects (plan 9.5)" T0=$SECONDS -k -n "$OP_NS" logs deploy/coder-k8s >"$WORK/operator-gw.txt" || fail "cannot read the operator log" +pod_logs "$OP_NS" "$OP_SELECTOR" "$WORK/operator-gw.txt" || fail "cannot read the operator log" k -n "$NS" get coderaigatewaykey e2e-gw-off e2e-gw-hijack -o yaml >"$WORK/gw-objects.yaml" || fail "cannot read the AI Gateway key objects" -# A key is a run of exactly 43 letters and digits. Only the log lines of the key controller are searched for one. +# A key is a run of exactly 43 letters and digits. Only the log lines of the key controller are searched for one. Kind runs +# Coder without a license, so no key exists and the shape check is a regression guard only. The exact-value check is the real test. LEAKS=$({ grep -i coderaigatewaykey "$WORK/operator-gw.txt" || true; cat "$WORK/gw-objects.yaml"; } | grep -cE '(^|[^A-Za-z0-9])[A-Za-z0-9]{43}([^A-Za-z0-9]|$)' || true) LEAKS=$((LEAKS + $(cat "$WORK/operator-gw.txt" "$WORK/gw-objects.yaml" | grep -cFf "$WORK/ai-key-gw" || true))) @@ -816,6 +843,7 @@ phase gw-delete # holds only SHA-256 hashes of the headers. Keys are synthetic and stay in 0600 files: never argv or output. MCP=e2e-mcp MCP_SECRET=e2e-mcp-key MCP_STUB_IMAGE=${E2E_MCP_STUB_IMAGE:-e2e.local/mcpstub:e2e} MCP_HOST=mcp-stub.$NS.svc:8080 MCP_HOST2=mcp-stub-2.$NS.svc:8080 MCPS=/api/v2/organizations/$ORG/mcp-servers +CODER_SELECTOR=app.kubernetes.io/name=coder-control-plane,app.kubernetes.io/instance=coder # the Coder pods of codercontrolplane coder MCP_SEEN="" MCP_S="" MCP_OFF=0 LLM_OFF=0 mcp_server() { # [url] [spec JSON]: kubectl apply of an api_key CoderMCPServer (slug = name) that Everyone can read local x=${5:-} @@ -895,9 +923,10 @@ jq -n --arg ns "$NS" --arg i "$MCP_STUB_IMAGE" '{apiVersion: "v1", kind: "List", apply_manifest "$WORK/stub-mcp.json" stub_ready() { k -n "$NS" get deploy mcp-stub -o json | jq -e '.status.readyReplicas == 1' >/dev/null; } wait_until "deploy/mcp-stub Ready" stub_ready -PROVIDER_ID=$(coder_api POST /api/v2/ai/providers "$(jq -nc --arg u "http://$MCP_HOST/v1" '{type: "openai-compat", name: "e2e-mcp-llm", - display_name: "E2E MCP stub", enabled: true, base_url: $u, api_keys: ["e2e-stub-llm-key"]}')" | jq -er '.id') || - fail "cannot create the stub LLM provider through the Coder API" +PROVIDER_ID=$(coder_ensure /api/v2/ai/providers name e2e-mcp-llm "AI provider" "$(jq -nc --arg u "http://$MCP_HOST/v1" '{type: "openai-compat", + name: "e2e-mcp-llm", display_name: "E2E MCP stub", enabled: true, base_url: $u, api_keys: ["e2e-stub-llm-key"]}')" | + jq -er --arg u "http://$MCP_HOST/v1" 'select(.enabled and .base_url == $u) | .id') || + fail "cannot create the stub LLM provider e2e-mcp-llm (enabled, base URL http://$MCP_HOST/v1) through the Coder API" MODEL_ID=$(coder_api POST "/api/v2/organizations/$ORG/chats/models" "$(jq -nc --arg p "$PROVIDER_ID" '{ai_provider_id: $p, model: "stub-model", context_limit: 100000, display_name: "E2E stub model", enabled: true, is_default: true}')" | jq -er '.id') || fail "cannot create the stub chat model" # The chats run as the tester, an organization member with a password login: the operator user does not run chats. @@ -930,9 +959,10 @@ jq -e --arg s "$MCP" --arg u "http://$MCP_HOST/mcp" '.slug == $s and .url == $u "$WORK/mcp-coder.json" >/dev/null || fail "Coder config $MCP_ID does not match the spec of $MCP: $(mcp_seen)" ACL=$(mcp_acl) || fail "cannot read the ACL of MCP server config $MCP_ID" [[ $ACL == '{"users":[],"groups":[{"name":"Everyone","role":"read"}]}' ]] || fail "the ACL of $MCP is not Everyone read only: $ACL" -k -n "$OP_NS" logs deploy/coder-k8s >"$WORK/operator-mcp.txt" || fail "cannot read the operator log" -LEAKS=$(cat "$WORK/mcp-$MCP.state.json" "$WORK/operator-mcp.txt" | grep -cFf "$WORK/ai-key-mcp-1" || true) -[[ $LEAKS == 0 ]] || fail "CoderMCPServer $MCP or the operator log holds the MCP server key ($LEAKS lines)" +pod_logs "$OP_NS" "$OP_SELECTOR" "$WORK/operator-mcp.txt" || fail "cannot read the operator log" +pod_logs "$NS" "$CODER_SELECTOR" "$WORK/coder-mcp.txt" || fail "cannot read the Coder log" +LEAKS=$(cat "$WORK/mcp-$MCP.state.json" "$WORK/operator-mcp.txt" "$WORK/coder-mcp.txt" | grep -cFf "$WORK/ai-key-mcp-1" || true) +[[ $LEAKS == 0 ]] || fail "CoderMCPServer $MCP, the operator log or the Coder log holds the MCP server key ($LEAKS lines)" phase mcp-apply step "CoderMCPServer chat: the tester's chat reaches the stub with the key; the LLM gets the stub tools (plan 7.3 P2)" @@ -959,8 +989,8 @@ phase mcp-rotation step "CoderMCPServer refusals: an unlabeled Secret, the operator token, a foreign origin, a taken slug and force_on write nothing (plan 7.3 P4)" T0=$SECONDS -coder_api POST "$MCPS" "$(jq -nc --arg u "http://$MCP_HOST/mcp" '{display_name: "Taken out of band", slug: "e2e-mcp-taken", url: $u, - transport: "streamable_http", availability: "default_off", enabled: false, auth_type: "none"}')" >/dev/null || +coder_ensure "$MCPS" slug e2e-mcp-taken "MCP server config" "$(jq -nc --arg u "http://$MCP_HOST/mcp" '{display_name: "Taken out of band", + slug: "e2e-mcp-taken", url: $u, transport: "streamable_http", availability: "default_off", enabled: false, auth_type: "none"}')" >/dev/null || fail "cannot create MCP server config e2e-mcp-taken through the Coder API" new_key ai-key-mcp-3 || fail "cannot generate the third MCP server key" AI_ORIGIN=http://$MCP_HOST key_secret create e2e-mcp-unlabeled ai-key-mcp-3 '{}' || fail "cannot create Secret e2e-mcp-unlabeled" @@ -1072,7 +1102,7 @@ ns_gone() { # the namespace disappears (#209), and with it the test, the provide } TIMEOUT=$NS_DELETE_TIMEOUT wait_until "test, provider, control plane, and namespace $NS deleted (ControlPlaneGone release)" ns_gone NS_DELETE_SECONDS=$((SECONDS - T0)) -k -n "$OP_NS" logs deploy/coder-k8s >"$WORK/operator-ns.txt" || fail "cannot read the operator log after the namespace deletion" +pod_logs "$OP_NS" "$OP_SELECTOR" "$WORK/operator-ns.txt" || fail "cannot read the operator log after the namespace deletion" grep -F 'Releasing the finalizer without a delete in Coder' "$WORK/operator-ns.txt" | grep -F '"ControlPlaneGone"' | grep -qF '"e2e-ns-keep"' || fail "the operator log has no ControlPlaneGone release of AI provider e2e-ns-keep (plan 8.2 P7)" grep -F 'Releasing the finalizer without a delete in Coder: the MCP server config' "$WORK/operator-ns.txt" | grep -F '"ControlPlaneGone"' | diff --git a/hack/e2e-workspace-lifecycle_test.sh b/hack/e2e-workspace-lifecycle_test.sh index 7204784c..c7b0ac82 100755 --- a/hack/e2e-workspace-lifecycle_test.sh +++ b/hack/e2e-workspace-lifecycle_test.sh @@ -21,13 +21,14 @@ cat >"$STUBS/kubectl" <<'STUB' set -euo pipefail S=$STUB_STATE echo "kubectl $*" >>"$S/calls.log" -raw="" file="" patch="" all=$* pos=() +raw="" file="" patch="" sel="" all=$* pos=() while (($#)); do case $1 in --raw) raw=$2; shift 2 ;; -f) file=$2; shift 2 ;; -p) patch=$2; shift 2 ;; - -n | -o | -l | -c) shift 2 ;; + -l) sel=$2; shift 2 ;; + -n | -o | -c) shift 2 ;; -v=* | --request-timeout=* | --type=* | --wait=*) shift ;; *) pos+=("$1"); shift ;; esac @@ -105,6 +106,7 @@ mcp_get() { # : one reconcile of a fake CoderMCPServer controller against c=$S/mcpc/$(jq -r .spec.slug "$cr").json u=$(jq -r .spec.url "$cr") sf=$S/secrets/$(jq -r .spec.auth.apiKey.secretRef.name "$cr").json if [[ -f $S/ns-deleted ]]; then ai_json "$cr" ControlPlaneGone; [[ $SCENARIO == mcp-ns-stuck ]] || rm "$cr"; return; fi if [[ ,$(cat "$S/mcp-orgs" 2>/dev/null), != *,$(jq -r .spec.organization "$cr"),* ]]; then r=MCPServersNotAllowed + [[ $SCENARIO != mcp-unlisted-writes ]] || jq -n --arg s "$(jq -r .spec.slug "$cr")" '{id: ("mcp-" + $s), slug: $s, updated_at: 0}' >"$c" elif [[ $(jq -r '.spec.availability // ""' "$cr") == force_on ]]; then r=ForceOnNotAllowed elif [[ ! -f $sf ]]; then r=CredentialRefused # only the driver's Secrets are in the store; the operator token is refused elif [[ $(jq -r '.metadata.labels["coder.com/credential"] // ""' "$sf") != true ]]; then r=CredentialNotOptedIn @@ -112,6 +114,7 @@ mcp_get() { # : one reconcile of a fake CoderMCPServer controller against r=CredentialDestinationNotAllowed && [[ $SCENARIO != mcp-wrong-reason ]] || r=CredentialMalformed [[ $SCENARIO != mcp-url-early || ! -f $c ]] || ai_set "$c" --arg u "$u" '.url = $u' elif [[ -f $c && $(jq -r '.status.configID // ""' "$cr") == "" ]]; then r=SlugConflict + [[ $SCENARIO != mcp-conflict-writes ]] || ai_set "$c" '.updated_at += 1' elif [[ $SCENARIO == mcp-never-ready ]]; then r=CoderUnavailable elif [[ -f $c && ($(jq .enabled "$c") == false || -f $S/mcp-killed) && $(jq .metadata.generation "$cr") == "$(jq .status.appliedGeneration "$cr")" ]]; then r=DisabledExternally && [[ $SCENARIO != mcp-kill-undone ]] || { ai_set "$c" '.enabled = true' && touch "$S/mcp-killed"; } @@ -130,9 +133,12 @@ mcp_get() { # : one reconcile of a fake CoderMCPServer controller against fi ai_json "$cr" "$r" } -ai_watch() { local f; for f in "$S"/cr/*.json; do [[ $(jq -r .spec.apiKeySecretRef.name "$f") != "$1" ]] || ai_get "$(basename "$f" .json)" >/dev/null; done; } +ai_watch() { local f; for f in "$S"/cr/*.json; do [[ ! -f $f || $(jq -r .spec.apiKeySecretRef.name "$f") != "$1" ]] || ai_get "$(basename "$f" .json)" >/dev/null; done; } case "${pos[0]}:${pos[1]:-}" in - get:pods) cat "$S/pods.json" ;; + get:pods) # the operator pods, or the Coder pod of codercontrolplane coder + if [[ $sel == app.kubernetes.io/name=coder-control-plane,app.kubernetes.io/instance=coder ]]; then + echo '{"items":[{"metadata":{"name":"coder-5f7d8-x2x4q"},"status":{"containerStatuses":[{"name":"coder","restartCount":0}]}}]}' + else cat "$S/pods.json"; fi ;; get:coderaiprovider) ai_get "${pos[2]}" ;; get:codermcpserver) mcp_get "${pos[2]}" ;; patch:codermcpserver) ai_set "$S/mcp/${pos[2]}.json" --argjson p "$patch" '. * $p | .metadata.generation += 1'; echo patched ;; @@ -217,13 +223,22 @@ case "${pos[0]}:${pos[1]:-}" in if [[ -f $f ]]; then cat "$f"; elif [[ ${pos[2]} == op-token ]]; then printf '{"data":{"token":"%s"}}\n' "$(printf secret-token-value | base64)" else err NotFound "secrets \"${pos[2]}\" not found"; fi ;; port-forward:*) echo $$ >"$S/pf.pid"; exec sleep 300 ;; - logs:*) echo "I1002 server started"; [[ $SCENARIO != ai-log-leak || ! -f $S/aisent-e2e-openai ]] || { cat "$S/aisent-e2e-openai"; echo; } + logs:coder-k8s-* | logs:deploy/coder-k8s) # the operator container restarted once: its earlier lines are only in the previous log (#156) + if [[ ${pos[2]:-} == --previous ]]; then + [[ ${pos[1]} == coder-k8s-new || ${pos[1]} == deploy/coder-k8s ]] || err BadRequest "previous terminated container \"manager\" in pod \"${pos[1]}\" not found" + [[ $SCENARIO != gw-log-leak || ! -f $S/secrets/e2e-gw-hijack.json ]] || + echo "INFO stub {\"controller\": \"coderaigatewaykey\", \"value\": \"$(jq -r '.data["api-key"] | @base64d' "$S/secrets/e2e-gw-hijack.json")\"}" + [[ ! -f $S/ns-deleted || $SCENARIO == ai-ns-no-release ]] || echo 'INFO Releasing the finalizer without a delete in Coder: the provider can stay in the Coder database {"controller": "coderaiprovider", "reason": "ControlPlaneGone", "provider": "e2e-ns-keep"}' + [[ ! -f $S/ns-deleted || $SCENARIO == mcp-ns-no-release ]] || echo 'INFO Releasing the finalizer without a delete in Coder: the MCP server config can stay in the Coder database {"controller": "codermcpserver", "reason": "ControlPlaneGone", "slug": "e2e-mcp-keep"}' + exit 0 + fi + echo "I1002 server started"; [[ $SCENARIO != ai-log-leak || ! -f $S/aisent-e2e-openai ]] || { cat "$S/aisent-e2e-openai"; echo; } [[ $SCENARIO != log-leak ]] || echo "2026-10-02T10:00:00Z [info] [provisioner|Planning infrastructure] Terraform 1.14.0 ok" - echo 'INFO Starting Controller {"controller": "coderaigatewaykey", "controllerKind": "CoderAIGatewayKey"}' - [[ $SCENARIO != gw-log-leak || ! -f $S/secrets/e2e-gw-hijack.json ]] || - echo "INFO stub {\"controller\": \"coderaigatewaykey\", \"value\": \"$(jq -r '.data["api-key"] | @base64d' "$S/secrets/e2e-gw-hijack.json")\"}" - [[ ! -f $S/ns-deleted || $SCENARIO == ai-ns-no-release ]] || echo 'INFO Releasing the finalizer without a delete in Coder: the provider can stay in the Coder database {"controller": "coderaiprovider", "reason": "ControlPlaneGone", "provider": "e2e-ns-keep"}' - [[ ! -f $S/ns-deleted ]] || echo 'INFO Releasing the finalizer without a delete in Coder: the MCP server config can stay in the Coder database {"controller": "codermcpserver", "reason": "ControlPlaneGone", "slug": "e2e-mcp-keep"}' ;; + echo 'INFO Starting Controller {"controller": "coderaigatewaykey", "controllerKind": "CoderAIGatewayKey"}' ;; + logs:coder-5f7d8-x2x4q) # the Coder pod, never restarted (mcp-coder-log-leak: it logs the MCP server key) + [[ ${pos[2]:-} != --previous ]] || err BadRequest 'previous terminated container "coder" in pod "coder-5f7d8-x2x4q" not found' + echo "Started HTTP listener at http://0.0.0.0:8080" + [[ $SCENARIO != mcp-coder-log-leak || ! -f $S/mcpc/e2e-mcp.json.key ]] || { cat "$S/mcpc/e2e-mcp.json.key"; echo; } ;; get:) [[ $raw != /api/v1/namespaces/coder/services/mcp-stub:8080/proxy/report ]] || exec cat "$S/report.json" if [[ $raw == */codertemplatetests?watch=1* ]]; then # the e2e-ttl watch: passed (ttl-failed: failed), then deleted @@ -269,8 +284,9 @@ case "${pos[0]}:${pos[1]:-}" in create:) if [[ ${file##*/} == tt-*.json ]]; then echo 0 >"$S/tt-$(jq -r .metadata.name "$file")"; echo created; exit 0; fi if [[ ${file##*/} == secret-*.json ]]; then # an AI provider key Secret; every key value is kept for the leak checks + n=$(jq -r .metadata.name "$file") && f=$S/secrets/$n.json && [[ ! -f $f ]] || err AlreadyExists "secrets \"$n\" already exists" jq -r '.data["api-key"] | @base64d' "$file" >>"$S/ai-keys.txt" - jq '.metadata.resourceVersion = "1"' "$file" >"$S/secrets/$(jq -r .metadata.name "$file").json"; echo created; exit 0 + jq '.metadata.resourceVersion = "1"' "$file" >"$f"; echo created; exit 0 fi if [[ $raw == */codertemplates/*/promote* ]]; then # promote-* scenarios break the activation read -r id av n st <"$S/template"; v=$(jq -r .spec.versionID "$file") @@ -411,8 +427,9 @@ case "$method $path" in "POST /api/v2/organizations/coder/chats/models") echo '{"id":"model-1"}' ;; "POST /api/v2/users/login") echo '{"session_token":"member-token-value"}' ;; "GET /api/v2/organizations/coder/mcp-servers") f=("$S"/mcpc/*.json) && if [[ -e ${f[0]} ]]; then jq -s . "${f[@]}"; else echo '[]'; fi ;; - "POST /api/v2/organizations/coder/mcp-servers") # a config created out of band - jq '{id: ("mcp-" + .slug), slug, url, enabled, display_name, updated_at: 0, groups: ["Everyone"]}' <<<"$data" >"$S/mcpc/$(jq -r .slug <<<"$data").json" ;; + "POST /api/v2/organizations/coder/mcp-servers") # a config created out of band; the slug is unique in the organization + f=$S/mcpc/$(jq -r .slug <<<"$data").json && [[ ! -f $f ]] || { echo "curl: (22) The requested URL returned error: 409" >&2; exit 22; } + jq '{id: ("mcp-" + .slug), slug, url, enabled, display_name, updated_at: 0, groups: ["Everyone"]}' <<<"$data" >"$f" && cat "$f" ;; "GET /api/v2/organizations/coder/mcp-servers/mcp-"*/acl) f=${path%/acl} && jq '{users: [], groups: [.groups[] | {name: ., role: "read"}]}' "$S/mcpc/${f##*/mcp-}.json" ;; "GET /api/v2/organizations/coder/mcp-servers/mcp-"* | "PATCH /api/v2/organizations/coder/mcp-servers/mcp-"*) # a config by ID; PATCH bumps updated_at f=$S/mcpc/${path##*/mcp-}.json @@ -476,7 +493,21 @@ exec "$REAL_JQ" "$@" STUB chmod +x "$STUBS"/* -# run_scenario [VAR=value...]: runs the driver against the stubs; sets T, S, RC, SECS. +# rerun_seed: what a failed run leaves in a kept namespace, the out-of-band Coder objects and the driver's Secrets (#390) +rerun_seed() { + local n + jq -n '{id: "aip-e2e-taken", name: "e2e-taken", type: "openai-compat", base_url: "http://ai-upstream.invalid:8080/v1", enabled: false, + display_name: "e2e-taken", api_keys: [], updated_at: "1"}' >"$S/aip/e2e-taken.json" + jq -n '{id: "aip-e2e-mcp-llm", name: "e2e-mcp-llm", type: "openai-compat", base_url: "http://mcp-stub.coder.svc:8080/v1", enabled: true, + display_name: "E2E MCP stub", api_keys: [{id: "key-old", masked: "e2e-...-key"}], updated_at: "1"}' >"$S/aip/e2e-mcp-llm.json" + jq -n '{id: "mcp-e2e-mcp-taken", slug: "e2e-mcp-taken", url: "http://mcp-stub.coder.svc:8080/mcp", enabled: false, + display_name: "Taken out of band", updated_at: 0, groups: ["Everyone"]}' >"$S/mcpc/e2e-mcp-taken.json" + for n in e2e-ai-key e2e-ai-key-unlabeled e2e-gw-hijack e2e-mcp-key e2e-mcp-unlabeled; do + jq -n --arg n "$n" '{metadata: {name: $n, resourceVersion: "7"}, data: {"api-key": ("stale" | @base64)}}' >"$S/secrets/$n.json" + done +} +# run_scenario [VAR=value...]: runs the driver against the stubs; sets T, S, RC, SECS. SEED names a function that +# adds state before the driver runs. # SCENARIO_TIMEOUT caps one driver run. The full lifecycle with the MCP phases takes about 70 s on a loaded host. SCENARIO_TIMEOUT=${E2E_TEST_SCENARIO_TIMEOUT_SECONDS:-180} run_scenario() { @@ -484,12 +515,12 @@ run_scenario() { shift T=$BASE/$name S=$BASE/$name/state mkdir -p "$S/ws" "$S/secrets" "$S/cr" "$S/aip" "$S/gw" "$S/mcp" "$S/mcpc" && echo '{"mcp":[],"llm":[]}' >"$S/report.json" - : >"$S/calls.log" && : >"$S/events" && echo 0 >"$S/counter" + : >"$S/calls.log" && : >"$S/events" && echo 0 >"$S/counter" && "${SEED:-:}" jq -n '{items: [ {metadata: {name: "coder-k8s-old", deletionTimestamp: "2026-09-23T00:00:00Z"}, spec: {containers: [{image: "ghcr.io/coder/coder-k8s:e2e"}]}, - status: {podIP: "10.244.0.4", conditions: [{type: "Ready", status: "True"}], containerStatuses: [{imageID: "docker.io/library/import-2026-09-22@\($old)"}]}}, + status: {podIP: "10.244.0.4", conditions: [{type: "Ready", status: "True"}], containerStatuses: [{name: "manager", restartCount: 0, imageID: "docker.io/library/import-2026-09-22@\($old)"}]}}, {metadata: {name: "coder-k8s-new"}, spec: {containers: [{image: "ghcr.io/coder/coder-k8s:e2e"}]}, - status: {podIP: "10.244.0.5", conditions: [{type: "Ready", status: "True"}], containerStatuses: [{imageID: "docker.io/library/import-2026-09-23@\($new)"}]}}]}' \ + status: {podIP: "10.244.0.5", conditions: [{type: "Ready", status: "True"}], containerStatuses: [{name: "manager", restartCount: 1, imageID: "docker.io/library/import-2026-09-23@\($new)"}]}}]}' \ --arg old "$OLD_DIGEST" --arg new "$NEW_DIGEST" >"$S/pods.json" jq -n --arg ip "$endpoint_ip" '{items: [{endpoints: [{addresses: [$ip], conditions: {ready: true}}]}]}' >"$S/endpoints.json" RC=0 @@ -611,6 +642,10 @@ check "AI Gateway key phases: NotAllowed, then LicenseRequired after the opt-in, [[ ! -f $S/gw/e2e-gw-off.json && ! -f $S/gw/e2e-gw-hijack.json && $(sed -n 4p "$S/ai-keys.txt" | tr -d "\n" | wc -c) == 43 ]] && jq -e ".spec == {controlPlaneRef: {name: \"coder\"}, secretName: \"e2e-gw-hijack\"}" "$T/work/gw-e2e-gw-hijack.json" >/dev/null && grep -q "curl -sS --max-time 30 -o .*/coder-raw.json .*/api/v2/ai-gateway/keys$" "$S/calls.log"' +check "leak checks read the Coder pod log; operator log checks read the previous container log after a restart (#156)" eval ' + grep -q "^kubectl --request-timeout=30s -n coder logs coder-5f7d8-x2x4q -c coder$" "$S/calls.log" && + [[ $(grep -c "^kubectl --request-timeout=30s -n coder-system logs coder-k8s-new -c manager --previous$" "$S/calls.log") == 4 ]] && + ! grep -q "logs coder-k8s-old" "$S/calls.log"' check "MCP server phases: each refusal reason seen; chats at both hosts and as the tester; e2e-mcp-taken kept; e2e-mcp deleted; e2e-mcp-keep released" \ eval 'out_has "MCP server e2e-mcp-unlisted: False MCPServersNotAllowed" && out_has "MCP server e2e-mcp-unlabeled: False CredentialNotOptedIn" && out_has "MCP server e2e-mcp-refused: False CredentialRefused" && out_has "MCP server e2e-mcp-origin: False CredentialDestinationNotAllowed" && @@ -621,6 +656,13 @@ check "MCP server phases: each refusal reason seen; chats at both hosts and as t jq -e ".spec.auth.apiKey.secretRef == {name: \"op-token\", key: \"token\"}" "$T/work/mcp-e2e-mcp-refused.json" >/dev/null && grep -q "get --raw /api/v1/namespaces/coder/services/mcp-stub:8080/proxy/report" "$S/calls.log"' +echo "TEST rerun-kept-namespace: a rerun reuses the Coder objects and replaces the Secrets that a failed run left (#390)" +SEED=rerun_seed run_scenario rerun-kept-namespace; summary +check "driver exits 0 and prints PASS" eval '[[ $RC -eq 0 ]] && out_has "PASS: workspace lifecycle"' +check "reuses e2e-taken, e2e-mcp-llm and e2e-mcp-taken; replaces the five Secrets" eval 'out_has "AI provider e2e-taken exists in Coder from an earlier run: reusing it" && + out_has "AI provider e2e-mcp-llm exists in Coder from an earlier run: reusing it" && out_has "MCP server config e2e-mcp-taken exists in Coder from an earlier run: reusing it" && + [[ $(grep -c "from an earlier run: replacing it" "$T/out") == 5 ]] && ! grep -qE "^curl .* -X POST .*/api/v2/(ai/providers|organizations/coder/mcp-servers)$" "$S/calls.log"' + echo "TEST image-mismatch: serving image differs from built image" run_scenario image-mismatch SERVING_ID="$OTHER"; summary check "fails with identity mismatch" failed_with "image identity mismatch" @@ -886,17 +928,21 @@ while IFS='|' read -r name msg muts; do done < 3|$M3 +mcp-conflict-writes|a refused CoderMCPServer wrote to Coder|$M4A mcp-wrong-reason|MCP server e2e-mcp-origin stalled with False CredentialMalformed stalled=True, want Ready=False CredentialDestinationNotAllowed|$M4A mcp-url-early|Coder config mcp-e2e-mcp changed before the Secret allowed the new origin|$M5 mcp-acl-open|a tester chat that selects e2e-mcp with access.groups [] answered HTTP 201, want 400 invalid or disabled|$M6,curl POST mcp-kill-undone|the controller undid the disable of MCP server e2e-mcp|$M7 mcp-delete-kept|MCP server config e2e-mcp (mcp-e2e-mcp) still exists in Coder after its object was deleted|$MCPM mcp-ns-stuck|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$NSM,kubectl patch,kubectl delete +mcp-ns-no-release|the operator log has no ControlPlaneGone release of MCP server e2e-mcp-keep|$NSM,kubectl patch,kubectl delete CASES echo "TEST missing-built-id: BUILT_IMAGE_ID is not a sha256 ID" From 10e5d3957edfcbbbed5ed53b5d896214b676b71d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 30 Oct 2026 17:03:30 +0000 Subject: [PATCH 2/3] test: give the namespace-release rows time to finish the namespace delete (#390) The CoderAIProvider and CoderMCPServer row groups forced a 2 s namespace delete timeout on every row. Rows that need the delete to finish (ai-ns-no-release, mcp-ns-no-release) could time out on a loaded host before they reached the ControlPlaneGone release check. Only the *-ns-stuck rows keep 2 s. agent-start-error's wall-clock bound becomes 120 s: still far below the 300 s agent timeout it guards against. Change-Id: I9c8b6668bb2b8cb5ee89158083c58fcb67e7e4b5 Signed-off-by: Thomas Kosiewski --- hack/e2e-workspace-lifecycle_test.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hack/e2e-workspace-lifecycle_test.sh b/hack/e2e-workspace-lifecycle_test.sh index c7b0ac82..2356fc15 100755 --- a/hack/e2e-workspace-lifecycle_test.sh +++ b/hack/e2e-workspace-lifecycle_test.sh @@ -847,7 +847,7 @@ check "driver exits 0 without the mirror check" eval '[[ $RC -eq 0 ]] && out_has echo "TEST agent-start-error: the agent lifecycle ends in start_error" run_scenario agent-start-error; summary check "fails at once on the lifecycle; no delete" eval 'failed_with "failed to start: main=connected/start_error" && - no_mutations_after "$LIFECYCLE,curl POST,curl PUT,kubectl apply,kubectl create" && ((SECS < 30))' + no_mutations_after "$LIFECYCLE,curl POST,curl PUT,kubectl apply,kubectl create" && ((SECS < 120))' echo "TEST agent-never-ready: the agent never connects" run_scenario agent-never-ready E2E_AGENT_TIMEOUT_SECONDS=2; summary @@ -887,7 +887,9 @@ CASES # name|expected message|mutations: each CoderAIProvider phase failure (plan 8.2) stops before the next mutation. while IFS='|' read -r name msg muts; do echo "TEST $name (CoderAIProvider phases)" - run_scenario "$name" E2E_NAMESPACE_DELETE_TIMEOUT_SECONDS=2 Date: Fri, 30 Oct 2026 17:29:02 +0000 Subject: [PATCH 3/3] test: reuse the stub chat model on kept-namespace reruns (#390) A rerun that reused e2e-mcp-llm still POSTed a new stub-model chat model, so each rerun added another default model. The driver now lists the organization's chat models and reuses stub-model of the provider when it is enabled and the default. Change-Id: I0b59c66deefbd5ded62340d57129375473104f36 Signed-off-by: Thomas Kosiewski --- hack/e2e-workspace-lifecycle.sh | 11 +++++++++-- hack/e2e-workspace-lifecycle_test.sh | 11 ++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/hack/e2e-workspace-lifecycle.sh b/hack/e2e-workspace-lifecycle.sh index d7312e02..a2dc97cc 100755 --- a/hack/e2e-workspace-lifecycle.sh +++ b/hack/e2e-workspace-lifecycle.sh @@ -927,8 +927,15 @@ PROVIDER_ID=$(coder_ensure /api/v2/ai/providers name e2e-mcp-llm "AI provider" " name: "e2e-mcp-llm", display_name: "E2E MCP stub", enabled: true, base_url: $u, api_keys: ["e2e-stub-llm-key"]}')" | jq -er --arg u "http://$MCP_HOST/v1" 'select(.enabled and .base_url == $u) | .id') || fail "cannot create the stub LLM provider e2e-mcp-llm (enabled, base URL http://$MCP_HOST/v1) through the Coder API" -MODEL_ID=$(coder_api POST "/api/v2/organizations/$ORG/chats/models" "$(jq -nc --arg p "$PROVIDER_ID" '{ai_provider_id: $p, model: "stub-model", - context_limit: 100000, display_name: "E2E stub model", enabled: true, is_default: true}')" | jq -er '.id') || fail "cannot create the stub chat model" +MODEL_ID=$(coder_api GET "/api/v2/organizations/$ORG/chats/models" | jq -er --arg p "$PROVIDER_ID" '[.models[] | + select(.ai_provider_id == $p and .model == "stub-model")] | if length > 1 then error("duplicate") else .[0] // {} | + if . == {} or (.enabled and .is_default) then .id // "" else error("disabled or not the default") end end') || + fail "cannot reuse the stub chat model stub-model of e2e-mcp-llm: it is duplicated, disabled or not the default" +if [[ -n $MODEL_ID ]]; then log "chat model stub-model exists in Coder from an earlier run: reusing it" # (#390) +else + MODEL_ID=$(coder_api POST "/api/v2/organizations/$ORG/chats/models" "$(jq -nc --arg p "$PROVIDER_ID" '{ai_provider_id: $p, model: "stub-model", + context_limit: 100000, display_name: "E2E stub model", enabled: true, is_default: true}')" | jq -er '.id') || fail "cannot create the stub chat model" +fi # The chats run as the tester, an organization member with a password login: the operator user does not run chats. MEMBER_TOKEN=$(coder_api POST /api/v2/users/login "$(jq -nc --arg e "$TESTER@e2e.example.com" --arg p "$TESTER_PASSWORD" '{email: $e, password: $p}')" | jq -er '.session_token') || fail "cannot log in as the tester $TESTER" diff --git a/hack/e2e-workspace-lifecycle_test.sh b/hack/e2e-workspace-lifecycle_test.sh index 2356fc15..60a2a8b0 100755 --- a/hack/e2e-workspace-lifecycle_test.sh +++ b/hack/e2e-workspace-lifecycle_test.sh @@ -424,7 +424,9 @@ case "$method $path" in [[ -f $f ]] || { echo "curl: (22) The requested URL returned error: 404" >&2; exit 22; } [[ $method == GET ]] || { jq --argjson d "$data" '. + $d | .updated_at = "2"' "$f" >"$f.tmp" && mv "$f.tmp" "$f"; } cat "$f" ;; - "POST /api/v2/organizations/coder/chats/models") echo '{"id":"model-1"}' ;; + "GET /api/v2/organizations/coder/chats/models") jq -n --slurpfile m <(cat "$S/chat-models.json" 2>/dev/null) '{models: $m, providers: []}' ;; + "POST /api/v2/organizations/coder/chats/models") + jq -c '{id: "model-1", ai_provider_id, model, enabled, is_default}' <<<"$data" | tee -a "$S/chat-models.json" ;; "POST /api/v2/users/login") echo '{"session_token":"member-token-value"}' ;; "GET /api/v2/organizations/coder/mcp-servers") f=("$S"/mcpc/*.json) && if [[ -e ${f[0]} ]]; then jq -s . "${f[@]}"; else echo '[]'; fi ;; "POST /api/v2/organizations/coder/mcp-servers") # a config created out of band; the slug is unique in the organization @@ -502,6 +504,7 @@ rerun_seed() { display_name: "E2E MCP stub", api_keys: [{id: "key-old", masked: "e2e-...-key"}], updated_at: "1"}' >"$S/aip/e2e-mcp-llm.json" jq -n '{id: "mcp-e2e-mcp-taken", slug: "e2e-mcp-taken", url: "http://mcp-stub.coder.svc:8080/mcp", enabled: false, display_name: "Taken out of band", updated_at: 0, groups: ["Everyone"]}' >"$S/mcpc/e2e-mcp-taken.json" + jq -nc '{id: "model-old", ai_provider_id: "aip-e2e-mcp-llm", model: "stub-model", enabled: true, is_default: true}' >"$S/chat-models.json" for n in e2e-ai-key e2e-ai-key-unlabeled e2e-gw-hijack e2e-mcp-key e2e-mcp-unlabeled; do jq -n --arg n "$n" '{metadata: {name: $n, resourceVersion: "7"}, data: {"api-key": ("stale" | @base64)}}' >"$S/secrets/$n.json" done @@ -659,9 +662,11 @@ check "MCP server phases: each refusal reason seen; chats at both hosts and as t echo "TEST rerun-kept-namespace: a rerun reuses the Coder objects and replaces the Secrets that a failed run left (#390)" SEED=rerun_seed run_scenario rerun-kept-namespace; summary check "driver exits 0 and prints PASS" eval '[[ $RC -eq 0 ]] && out_has "PASS: workspace lifecycle"' -check "reuses e2e-taken, e2e-mcp-llm and e2e-mcp-taken; replaces the five Secrets" eval 'out_has "AI provider e2e-taken exists in Coder from an earlier run: reusing it" && +check "reuses e2e-taken, e2e-mcp-llm, its chat model and e2e-mcp-taken; replaces the five Secrets" eval 'out_has "AI provider e2e-taken exists in Coder from an earlier run: reusing it" && out_has "AI provider e2e-mcp-llm exists in Coder from an earlier run: reusing it" && out_has "MCP server config e2e-mcp-taken exists in Coder from an earlier run: reusing it" && - [[ $(grep -c "from an earlier run: replacing it" "$T/out") == 5 ]] && ! grep -qE "^curl .* -X POST .*/api/v2/(ai/providers|organizations/coder/mcp-servers)$" "$S/calls.log"' + out_has "chat model stub-model exists in Coder from an earlier run: reusing it" && [[ $(wc -l <"$S/chat-models.json") == 1 ]] && + [[ $(grep -c "from an earlier run: replacing it" "$T/out") == 5 ]] && + ! grep -qE "^curl .* -X POST .*/api/v2/(ai/providers|organizations/coder/(mcp-servers|chats/models))$" "$S/calls.log"' echo "TEST image-mismatch: serving image differs from built image" run_scenario image-mismatch SERVING_ID="$OTHER"; summary