From 7bd9d9860c310b537539e46ea9733b0e7798a8c3 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 15 Oct 2026 18:55:04 +0200 Subject: [PATCH 01/12] Add the E2E workflow: deploy the template into a throwaway account and read telemetry back The workflow creates an AWS source, deploys the template with that source's ClusterId, token and id, pastes the outputs back into the source, waits for the sync's subscription filters, produces logs, a custom metric, an X-Ray trace and a CloudTrail management event, and reads all four back through the team's SQL API connection with exact counts. Pull requests and pushes to main deploy the checkout's template; the daily schedule deploys the published one and fails when it differs from main. Everything is deleted afterwards, plus leftovers older than two hours. AWS access is OIDC through the role in e2e/bootstrap.yaml. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 324 ++++++++++++++++++++ .github/workflows/notify_linear_failure.yml | 121 ++++++++ README.md | 1 + e2e/bootstrap.yaml | 42 +++ 4 files changed, 488 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 .github/workflows/notify_linear_failure.yml create mode 100644 e2e/bootstrap.yaml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..5ed6093 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,324 @@ +name: "E2E" + +# Deploys the CloudFormation template into a throwaway AWS account the way a customer does: creates an AWS source, +# deploys the stack with that source's ClusterId/SourceToken/SourceId, pastes the stack outputs back into the source, +# then produces logs, a custom metric, an X-Ray trace and a CloudTrail management event in AWS and reads all four back +# through the team's SQL API connection with exact row counts. Pull requests and pushes to main deploy the template +# from the checkout; the daily schedule deploys the published one and fails when it differs from main. Everything the +# run creates in AWS and Better Stack is named after RUN_ID and deleted in the cleanup step, together with leftovers of +# earlier runs older than two hours. The docs check fails when the docs article stops naming a template toggle or +# output. Pull requests from forks are skipped: they get no OIDC token. +# See the docs guide in https://betterstack.com/docs/logs/aws-cloudformation/ +# +# AWS account: better-stack-e2e-aws-stack (251917137055), a member of the Better Stack organization that holds nothing +# but this test. Root sign-in: 1Password, Engineering vault, item "AWS better-stack-e2e-aws-stack". The one standing +# resource is the CloudFormation stack `e2e-bootstrap` from e2e/bootstrap.yaml: a GitHub OIDC provider and IAM role +# `github-actions-e2e` with AdministratorAccess trusting this repository. The template deploys IAM roles, Lambdas, +# Firehose streams, CloudTrail and S3, and its role and bucket names are fixed per account, so administrator access in +# an otherwise empty account is the honest scope and only one run may deploy at a time (the concurrency group below). +# Secrets: TEST_TEAM_TELEMETRY_API, TEST_TEAM_CONNECTION_USERNAME, TEST_TEAM_CONNECTION_PASSWORD (the E2E testing team), +# LINEAR_* (see notify_linear_failure.yml). + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: '15 5 * * *' + workflow_dispatch: + inputs: + template_url: + description: 'Template to deploy instead of the checkout (any URL)' + type: string + +permissions: + id-token: write + contents: read + actions: read + +# The template names its IAM roles and buckets globally, so one stack per account at a time. +concurrency: + group: e2e-aws-stack + cancel-in-progress: false + +env: + AWS_REGION: us-east-1 + AWS_ROLE_ARN: arn:aws:iam::251917137055:role/github-actions-e2e + API: https://telemetry.betterstack.com + DOCS_URL: https://betterstack.com/docs/logs/aws-cloudformation/ + PUBLISHED_TEMPLATE_URL: https://better-stack-cloudformation.s3.amazonaws.com/better-stack-full.yaml + TEMPLATE_URL_OVERRIDE: ${{ inputs.template_url }} + RUN_ID: gh${{ github.run_id }}a${{ github.run_attempt }} + SOURCE_PREFIX: e2e-aws-stack + STACK_NAME: e2e-gh${{ github.run_id }}a${{ github.run_attempt }} + LOG_GROUP: /e2e/gh${{ github.run_id }}a${{ github.run_attempt }} + MARKER: e2e run_id=gh${{ github.run_id }}a${{ github.run_attempt }} + TEST_TEAM_TELEMETRY_API: ${{ secrets.TEST_TEAM_TELEMETRY_API }} + TEST_TEAM_CONNECTION_USERNAME: ${{ secrets.TEST_TEAM_CONNECTION_USERNAME }} + TEST_TEAM_CONNECTION_PASSWORD: ${{ secrets.TEST_TEAM_CONNECTION_PASSWORD }} + +jobs: + test: + name: Stack round trip + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: "[Setup] Checkout code" + uses: actions/checkout@v7 + + - name: "[Check] Pick the template, docs article names every toggle and output, published template vs main" + run: | + if [ -n "$TEMPLATE_URL_OVERRIDE" ]; then + curl -sSfL "$TEMPLATE_URL_OVERRIDE" -o template.yaml; TEMPLATE_FROM="$TEMPLATE_URL_OVERRIDE" + elif [ "${{ github.event_name }}" = schedule ]; then + curl -sSfL "$PUBLISHED_TEMPLATE_URL" -o template.yaml; TEMPLATE_FROM="$PUBLISHED_TEMPLATE_URL" + else + cp cloudformation/full/better-stack-full.yaml template.yaml; TEMPLATE_FROM="this checkout" + fi + echo "TEMPLATE_FROM=$TEMPLATE_FROM ($(git rev-parse --short HEAD))" >> "$GITHUB_ENV" + curl -sSfL "$PUBLISHED_TEMPLATE_URL" -o template-published.yaml + if ! diff -q template-published.yaml cloudformation/full/better-stack-full.yaml >/dev/null; then + if [ "${{ github.event_name }}" = schedule ]; then + echo "::error::$PUBLISHED_TEMPLATE_URL differs from cloudformation/full/better-stack-full.yaml on main: the published template was not updated after a merge" + diff template-published.yaml cloudformation/full/better-stack-full.yaml | head -40 + exit 1 + fi + echo "::warning::$PUBLISHED_TEMPLATE_URL differs from the template in this checkout (expected while a change is unreleased)" + fi + TOGGLES=$(awk '/^Parameters:/{p=1;next} /^[A-Z][A-Za-z]*:/{p=0} p && /^ [A-Za-z]+:$/{name=substr($1,1,length($1)-1)} p && /^ - .false.$/{print name}' template.yaml | sort -u) + OUTPUTS=$(awk '/^Outputs:/{p=1;next} p && /^ [A-Za-z]+:$/{print substr($1,1,length($1)-1)}' template.yaml) + [ -n "$TOGGLES" ] && [ -n "$OUTPUTS" ] || { echo "::error::found no true/false parameters or no outputs in the template, this check needs updating"; exit 1; } + curl -sSfL "$DOCS_URL" -o docs.html + MISSING="" + for name in $TOGGLES $OUTPUTS https://github.com/BetterStackHQ/aws; do grep -q "$name" docs.html || MISSING="$MISSING $name"; done + [ -z "$MISSING" ] || { echo "::error::$DOCS_URL does not mention:$MISSING"; exit 1; } + echo "template from $TEMPLATE_FROM; docs article mentions every toggle ($(echo $TOGGLES)) and output ($(echo $OUTPUTS))" + + - name: "[Setup] AWS credentials (OIDC)" + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + # Stack creation and deletion can take 20 min each; the role allows 2 h sessions. + role-duration-seconds: 7200 + + - name: "[Check] Template is valid CloudFormation" + run: aws cloudformation validate-template --template-body file://template.yaml >/dev/null + + - name: "[Setup] Create AWS source" + run: | + RES=$(curl -sS -X POST "$API/api/v1/sources" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg name "$SOURCE_PREFIX $RUN_ID" '{name: $name, platform: "aws"}')") + jq -e '.data.id' <<<"$RES" >/dev/null || { echo "::error::creating the source failed: $RES"; exit 1; } + SOURCE_TOKEN=$(jq -r '.data.attributes.token' <<<"$RES") + echo "::add-mask::$SOURCE_TOKEN" + { + echo "SOURCE_ID=$(jq -r '.data.id' <<<"$RES")" + echo "SOURCE_TOKEN=$SOURCE_TOKEN" + echo "DATA_REGION=$(jq -r '.data.attributes.data_region' <<<"$RES")" + echo "INGESTING_HOST=$(jq -r '.data.attributes.ingesting_host' <<<"$RES")" + echo "COLLECTION=t$(jq -r '.data.attributes.team_id' <<<"$RES")_$(jq -r '.data.attributes.table_name' <<<"$RES")" + echo "CONNECT_HOST=$(jq -r '.data.attributes.data_region' <<<"$RES")-connect.betterstackdata.com" + echo "ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)" + } >> "$GITHUB_ENV" + echo "source $(jq -r '.data.id' <<<"$RES") on $(jq -r '.data.attributes.data_region' <<<"$RES")" + + # A fresh token reaches the ingesters with a delay; until then they answer 401 and Firehose would retry its + # batches. Send warm-up lines until 20 in a row are accepted, then let it settle. + - name: "[Setup] Warm up the source until ingest accepts its token" + run: | + STREAK=0 + DEADLINE=$((SECONDS + 300)) + while [ "$STREAK" -lt 20 ]; do + CODE=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "https://$INGESTING_HOST/" -H "Authorization: Bearer $SOURCE_TOKEN" -H 'Content-Type: application/json' -d "{\"message\":\"e2e warm-up run_id=$RUN_ID\"}") + if [ "$CODE" = 202 ]; then STREAK=$((STREAK + 1)); else echo "ingest answered HTTP $CODE after $STREAK accepted"; STREAK=0; fi + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::ingest did not accept the source token 20 times in a row within 5 min"; exit 1; } + sleep 1 + done + sleep 15 + echo "ingest accepts the token" + + - name: "[Setup] Create the log group before the stack, so the first sync finds it" + run: | + aws logs create-log-group --log-group-name "$LOG_GROUP" + aws logs create-log-stream --log-group-name "$LOG_GROUP" --log-stream-name run + + - name: "[Test] Deploy the CloudFormation stack" + run: | + aws cloudformation create-stack --stack-name "$STACK_NAME" --template-body file://template.yaml --capabilities CAPABILITY_NAMED_IAM \ + --parameters "ParameterKey=ClusterId,ParameterValue=$DATA_REGION" "ParameterKey=SourceToken,ParameterValue=$SOURCE_TOKEN" "ParameterKey=SourceId,ParameterValue=$SOURCE_ID" + if ! aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME"; then + echo "::error::stack creation failed: $(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --query 'Stacks[0].StackStatus' --output text)" + aws cloudformation describe-stack-events --stack-name "$STACK_NAME" --query "StackEvents[?ends_with(ResourceStatus, 'FAILED')].[LogicalResourceId,ResourceStatus,ResourceStatusReason]" --output table + exit 1 + fi + OUTPUTS=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --query 'Stacks[0].Outputs' --output json) + ROLE_ARN=$(jq -r '.[] | select(.OutputKey == "IntegrationRoleArn") | .OutputValue' <<<"$OUTPUTS") + EXTERNAL_ID=$(jq -r '.[] | select(.OutputKey == "ExternalId") | .OutputValue' <<<"$OUTPUTS") + [ -n "$ROLE_ARN" ] && [ -n "$EXTERNAL_ID" ] || { echo "::error::stack outputs lack IntegrationRoleArn or ExternalId: $OUTPUTS"; exit 1; } + # The external id is half of what Better Stack needs to assume the integration role; this repo is public. + echo "::add-mask::$EXTERNAL_ID" + echo "ROLE_ARN=$ROLE_ARN" >> "$GITHUB_ENV" + echo "EXTERNAL_ID=$EXTERNAL_ID" >> "$GITHUB_ENV" + echo "stack outputs: IntegrationRoleArn $ROLE_ARN, ExternalId (masked)" + + # X-Ray creates the aws/spans log group when it processes the first span, and the sync subscribes only groups + # that exist, so a throwaway trace goes in before the source is connected. + - name: "[Test] Warm-up trace so that aws/spans exists before the first sync" + run: | + NOW=$(date +%s) + TRACE_ID="1-$(printf '%x' "$NOW")-$(openssl rand -hex 12)" + SEGMENT=$(jq -nc --arg name "e2e-warm-up" --arg id "$(openssl rand -hex 8)" --arg trace "$TRACE_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \ + '{name: $name, id: $id, trace_id: $trace, start_time: $start, end_time: $end}') + aws xray put-trace-segments --trace-segment-documents "$SEGMENT" --query 'UnprocessedTraceSegments' --output json | jq -e 'length == 0' >/dev/null + + # aws_account_id would select an already connected account; a new connection is the role ARN + external id only. + - name: "[Test] Paste the stack outputs into the source (connects the AWS account, enqueues the sync)" + run: | + BODY=$(jq -nc --arg r "$ROLE_ARN" --arg e "$EXTERNAL_ID" '{aws_role_arn: $r, aws_external_id: $e}') + for attempt in 1 2 3; do + CODE=$(curl -sS -o connect.json -w '%{http_code}' -X PATCH "$API/api/v1/sources/$SOURCE_ID" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' -d "$BODY") + [ "$CODE" = 200 ] && { echo "source connected to $ROLE_ARN"; exit 0; } + # IAM propagation occasionally refuses the first AssumeRole on a fresh role; one bounded retry on that answer only. + if [ "$CODE" = 422 ] && grep -qi assume connect.json && [ "$attempt" -lt 3 ]; then echo "attempt $attempt refused: $(cat connect.json), retrying in 20 s"; sleep 20; continue; fi + echo "::error::connecting the source failed: HTTP $CODE $(cat connect.json)"; exit 1 + done + + - name: "[Test] Wait until the sync subscribed the log group, aws/spans and the CloudTrail group" + run: | + DEADLINE=$((SECONDS + 480)) + while :; do + PENDING="" + for group in "$LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + aws logs describe-subscription-filters --log-group-name "$group" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text 2>/dev/null | grep -q firehose || PENDING="$PENDING $group" + done + [ -z "$PENDING" ] && { echo "all three log groups carry the BetterStackLogs subscription filter"; exit 0; } + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::no BetterStackLogs subscription filter after 8 min on:$PENDING"; exit 1; } + echo "waiting for subscriptions on:$PENDING"; sleep 15 + done + + - name: "[Test] Produce logs, a custom metric, a two-segment trace and a CloudTrail management event" + run: | + echo "SINCE=dt > toDateTime('$(date -u -d '-10 minutes' '+%F %T')')" >> "$GITHUB_ENV" + jq -nc --arg m "$MARKER" --arg run "$RUN_ID" '[range(1; 21) as $n | {timestamp: ((now * 1000 | floor) + $n), message: ({message: $m, e2e: {run_id: $run, n: $n}} | tojson)}]' > events.json + aws logs put-log-events --log-group-name "$LOG_GROUP" --log-stream-name run --log-events file://events.json >/dev/null + jq -nc --arg run "$RUN_ID" --arg t1 "$(date -u -d '-2 minutes' +%FT%TZ)" --arg t2 "$(date -u -d '-1 minute' +%FT%TZ)" --arg t3 "$(date -u +%FT%TZ)" \ + '[$t1, $t2, $t3] | to_entries | map({MetricName: "RunMarker", Value: (.key + 1), Unit: "Count", Timestamp: .value, Dimensions: [{Name: "RunId", Value: $run}]})' > metrics.json + aws cloudwatch put-metric-data --namespace E2eTest/AwsStack --metric-data file://metrics.json + NOW=$(date +%s) + TRACE_ID="1-$(printf '%x' "$NOW")-$(openssl rand -hex 12)" + PARENT_ID=$(openssl rand -hex 8) + PARENT=$(jq -nc --arg name "e2e-parent-$RUN_ID" --arg id "$PARENT_ID" --arg trace "$TRACE_ID" --arg run "$RUN_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \ + '{name: $name, id: $id, trace_id: $trace, start_time: $start, end_time: $end, annotations: {run_id: $run}}') + CHILD=$(jq -nc --arg name "e2e-child-$RUN_ID" --arg id "$(openssl rand -hex 8)" --arg parent "$PARENT_ID" --arg trace "$TRACE_ID" --arg run "$RUN_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \ + '{name: $name, id: $id, parent_id: $parent, trace_id: $trace, start_time: ($start + 0.2), end_time: ($end - 0.2), annotations: {run_id: $run}}') + aws xray put-trace-segments --trace-segment-documents "$PARENT" "$CHILD" --query 'UnprocessedTraceSegments' --output json | jq -e 'length == 0' >/dev/null + # The X-Ray trace id 1-<8 hex epoch>-<24 hex> is stored as the 32-hex OpenTelemetry id. + echo "TRACE_HEX=$(cut -d- -f2,3 <<<"$TRACE_ID" | tr -d -)" >> "$GITHUB_ENV" + # A management event that happens after the trail exists and names this run. + aws logs put-retention-policy --log-group-name "$LOG_GROUP" --retention-in-days 1 + echo "20 log events, 3 datapoints of E2eTest/AwsStack RunMarker, trace $TRACE_ID, PutRetentionPolicy on $LOG_GROUP" + + - name: "[Test] Read everything back through the SQL API connection" + run: | + set -o pipefail + sql() { + CODE=$(curl -sS -o response.json -w '%{http_code}' -u "$TEST_TEAM_CONNECTION_USERNAME:$TEST_TEAM_CONNECTION_PASSWORD" -H 'Content-Type: plain/text' --data-binary "$1 FORMAT JSONEachRow" "https://$CONNECT_HOST/") + # A source's logs, spans and metrics collections each exist only from their first stored row on; + # until then the query is "no rows yet". + [ "$CODE" = 500 ] && grep -q CLUSTER_DOESNT_EXIST response.json && return 0 + [ "$CODE" = 200 ] || { echo "::error::SQL API on $CONNECT_HOST answered HTTP $CODE: $(head -c 300 response.json)" >&2; return 1; } + cat response.json + } + count() { ROWS=$(sql "SELECT count() AS n FROM $1") || return 1; [ -n "$ROWS" ] && jq -r '.n' <<<"$ROWS" || echo 0; } + counts() { + LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0") + CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 AND position(raw, '$LOG_GROUP') > 0") + SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'trace_id') = '$TRACE_HEX'") + METRICS=$(count "remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' AND toString(tags) LIKE '%$RUN_ID%'") + echo "logs $LOGS/20, cloudtrail $CLOUDTRAIL/1, spans $SPANS/2, metric rows $METRICS" + } + DEADLINE=$((SECONDS + 900)) + while :; do + counts + [ "$LOGS" -ge 20 ] && [ "$CLOUDTRAIL" -ge 1 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && break + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 15 min"; exit 1; } + sleep 20 + done + # Everything arrived; a last look 60 s later catches duplicate delivery. + sleep 60 + counts + [ "$LOGS" = 20 ] && [ "$CLOUDTRAIL" = 1 ] && [ "$SPANS" = 2 ] || { echo "::error::unexpected row counts: logs $LOGS (20), cloudtrail $CLOUDTRAIL (1), spans $SPANS (2)"; exit 1; } + mkdir -p out + sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json + sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 LIMIT 1" > out/sample_cloudtrail.json + sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'trace_id') = '$TRACE_HEX' ORDER BY JSONExtractUInt(raw, 'start_time_unix_nano')" > out/sample_spans.json + sql "SELECT name, tags FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' GROUP BY name, tags LIMIT 3" > out/sample_metric.json + { + echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" + echo + echo "- template from $TEMPLATE_FROM, source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`" + echo "- read back through \`$CONNECT_HOST\`: logs $LOGS, cloudtrail $CLOUDTRAIL, spans $SPANS, metric rows $METRICS" + echo + echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" + } > out/summary.md + + - name: "[Cleanup] Delete this run's resources in Better Stack and AWS (and stale ones)" + if: ${{ always() }} + run: | + set -o pipefail + mkdir -p out + say() { echo "$1"; echo "$1" >> out/cleanup.md; } + could_not_list() { say "- ❌ could not list $1, so none were deleted"; touch cleanup.failed; } + say "### Cleanup" + say "" + STALE_BEFORE=$(date -u -d '-2 hours' +%FT%TZ) + # Sources first: while the integration role still exists Better Stack can unsubscribe cleanly. + SOURCES=$(curl -sS --fail-with-body "$API/api/v1/sources?name=$SOURCE_PREFIX&per_page=250" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" \ + | jq -r --arg run " $RUN_ID" --arg before "$STALE_BEFORE" '.data[] | select((.attributes.name | endswith($run)) or .attributes.created_at < $before) | "\(.id) \(.attributes.name)"') || { could_not_list sources; SOURCES=""; } + while read -r id name; do + [ -n "$id" ] || continue + CODE=$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE "$API/api/v1/sources/$id" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API") + [ "$CODE" = 204 ] && say "- deleted source $id \`$name\`" || { say "- ❌ could not delete source $id \`$name\` (HTTP $CODE)"; touch cleanup.failed; } + done <<<"$SOURCES" + STACKS=$(aws cloudformation describe-stacks --query "Stacks[?starts_with(StackName, 'e2e-')].[StackName,CreationTime]" --output json \ + | jq -r --arg name "$STACK_NAME" --arg before "$STALE_BEFORE" '.[] | select(.[0] == $name or .[1] < $before) | .[0]') || { could_not_list stacks; STACKS=""; } + for stack in $STACKS; do + aws cloudformation delete-stack --stack-name "$stack" + if aws cloudformation wait stack-delete-complete --stack-name "$stack"; then say "- deleted stack \`$stack\`"; else + say "- ❌ could not delete stack \`$stack\`: $(aws cloudformation describe-stack-events --stack-name "$stack" --query "StackEvents[?ends_with(ResourceStatus, 'FAILED')].[LogicalResourceId,ResourceStatusReason]" --output text | head -5)"; touch cleanup.failed + fi + done + # The stack retains its buckets by design, and X-Ray creates aws/spans outside the stack. + ACCOUNT_ID=${ACCOUNT_ID:-$(aws sts get-caller-identity --query Account --output text || true)} + [ -n "$ACCOUNT_ID" ] || could_not_list buckets + for bucket in "better-stack-firehose-$ACCOUNT_ID-$AWS_REGION" "better-stack-cloudtrail-$ACCOUNT_ID-$AWS_REGION"; do + [ -n "$ACCOUNT_ID" ] && aws s3api head-bucket --bucket "$bucket" 2>/dev/null || continue + if aws s3 rb "s3://$bucket" --force >/dev/null; then say "- deleted bucket \`$bucket\`"; else say "- ❌ could not delete bucket \`$bucket\`"; touch cleanup.failed; fi + done + LOG_GROUPS=$(aws logs describe-log-groups --query 'logGroups[].[logGroupName,creationTime]' --output json \ + | jq -r --arg mine "$LOG_GROUP" --argjson before "$(( $(date -u -d '-2 hours' +%s) * 1000 ))" '.[] | select(.[0] == "aws/spans" or (.[0] | startswith("/e2e/")) and (.[0] == $mine or .[1] < $before)) | .[0]') || { could_not_list "log groups"; LOG_GROUPS=""; } + for group in $LOG_GROUPS; do + if aws logs delete-log-group --log-group-name "$group"; then say "- deleted log group \`$group\`"; else say "- ❌ could not delete log group \`$group\`"; touch cleanup.failed; fi + done + [ "$(wc -l < out/cleanup.md)" -gt 2 ] || say "- nothing to delete" + [ ! -f cleanup.failed ] || { say ""; say "❌ cleanup left resources behind"; exit 1; } + + - name: "[Report] Job summary" + if: ${{ always() }} + run: | + if [ -f out/summary.md ]; then cat out/summary.md >> "$GITHUB_STEP_SUMMARY"; else echo "No summary, the run failed before verification." >> "$GITHUB_STEP_SUMMARY"; fi + # Rows come back as JSONEachRow with the stored event as a JSON string in raw; render them as a pretty array. + for f in out/sample_*.json; do [ -f "$f" ] && { echo; echo "
$(basename "$f")"; echo; echo '```json'; jq -s 'map(if has("raw") then .raw |= (fromjson? // .) else . end)' "$f"; echo '```'; echo '
'; echo; } >> "$GITHUB_STEP_SUMMARY"; done + [ -f out/cleanup.md ] && { echo; cat out/cleanup.md; } >> "$GITHUB_STEP_SUMMARY" + true + + notify-linear: + if: ${{ failure() && (github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }} + needs: [test] + uses: ./.github/workflows/notify_linear_failure.yml + with: + assignee: petr + secrets: inherit diff --git a/.github/workflows/notify_linear_failure.yml b/.github/workflows/notify_linear_failure.yml new file mode 100644 index 0000000..30f9428 --- /dev/null +++ b/.github/workflows/notify_linear_failure.yml @@ -0,0 +1,121 @@ +name: "Notify Linear on failure" + +on: + workflow_call: + inputs: + assignee: + description: "Assignee: pawel or petr" + required: true + type: string + +jobs: + create-linear-issue: + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: "Gather failure context" + id: ctx + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id || github.run_id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt || github.run_attempt }} + run: | + FAILED=$(gh api \ + "repos/${{ github.repository }}/actions/runs/$RUN_ID/attempts/$RUN_ATTEMPT/jobs" \ + --paginate \ + --jq '[.jobs[] + | select(.conclusion == "failure") + | "- **\(.name)** — failed at _\([.steps[] | select(.conclusion == "failure") | .name] | join(", "))_"] + | join("\n")') + + if [ -z "$FAILED" ]; then + FAILED="_(no failed jobs reported by the API yet)_" + fi + + { + echo "failed_jobs<> "$GITHUB_OUTPUT" + - name: "Create Linear issue" + env: + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} + LINEAR_TEAM_ID: ${{ secrets.LINEAR_TEAM_TELEMETRY_ID }} + LINEAR_STATUS_PLAN_ID: ${{ secrets.LINEAR_STATUS_PLAN_ID }} + LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID: ${{ secrets.LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID }} + LINEAR_PROJECT_COLLECTORS_ID: ${{ secrets.LINEAR_PROJECT_COLLECTORS_ID }} + LINEAR_ASSIGNEE_PAWEL_ID: ${{ secrets.LINEAR_ASSIGNEE_PAWEL_ID }} + LINEAR_ASSIGNEE_PETR_ID: ${{ secrets.LINEAR_ASSIGNEE_PETR_ID }} + ASSIGNEE: ${{ inputs.assignee }} + WORKFLOW: ${{ github.event.workflow_run.name || github.workflow }} + REF_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }} + RUN_URL: ${{ github.event.workflow_run.html_url || format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt || github.run_attempt }} + EVENT_NAME: ${{ github.event.workflow_run.event || github.event_name }} + SHA: ${{ github.event.workflow_run.head_sha || github.sha }} + COMMIT_MESSAGE: ${{ github.event.workflow_run.head_commit.message || github.event.head_commit.message }} + FAILED_JOBS: ${{ steps.ctx.outputs.failed_jobs }} + run: | + if [ "$ASSIGNEE" = "pawel" ]; then + ASSIGNEE_ID="$LINEAR_ASSIGNEE_PAWEL_ID" + PROJECT_ID="$LINEAR_PROJECT_COLLECTORS_ID" + else + ASSIGNEE_ID="$LINEAR_ASSIGNEE_PETR_ID" + PROJECT_ID="$LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID" + fi + + SHORT_SHA="${SHA:0:7}" + + # head_commit is only populated on push events; skip the line otherwise + COMMIT_LINE="**Commit:** \`$SHORT_SHA\`" + if [ -n "$COMMIT_MESSAGE" ]; then + COMMIT_SUBJECT="$(printf '%s' "$COMMIT_MESSAGE" | head -n1)" + COMMIT_LINE="**Commit:** \`$SHORT_SHA\` — $COMMIT_SUBJECT" + fi + + ATTEMPT_NOTE="" + if [ "$RUN_ATTEMPT" != "1" ]; then + ATTEMPT_NOTE=" (attempt #$RUN_ATTEMPT)" + fi + + TITLE="E2E failure: $WORKFLOW" + DESCRIPTION=$(cat < Date: Thu, 15 Oct 2026 19:08:15 +0200 Subject: [PATCH 02/12] Sweep leftovers before each run, keep the cleanup in e2e/cleanup.sh A cancelled job gets only a short grace period, so its cleanup can be killed while the stack is still deleting; the next run then collided with the leftover for up to two hours. The cleanup now lives in e2e/cleanup.sh, runs before and after every run, and deletes everything of ours regardless of age, since the concurrency group allows one run per account. It matches run stacks by their e2e-gh prefix: the earlier e2e- prefix also matched the standing e2e-bootstrap stack and only its age had protected it. configure-aws-credentials goes to v5. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 48 +++++++-------------------------------- e2e/cleanup.sh | 48 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 40 deletions(-) create mode 100755 e2e/cleanup.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5ed6093..e3b13f3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -97,7 +97,7 @@ jobs: echo "template from $TEMPLATE_FROM; docs article mentions every toggle ($(echo $TOGGLES)) and output ($(echo $OUTPUTS))" - name: "[Setup] AWS credentials (OIDC)" - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@v5 with: role-to-assume: ${{ env.AWS_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} @@ -107,6 +107,11 @@ jobs: - name: "[Check] Template is valid CloudFormation" run: aws cloudformation validate-template --template-body file://template.yaml >/dev/null + # A cancelled job gets only a short grace period, so its cleanup can be killed while the stack is still being + # deleted; sweeping first keeps such leftovers from failing this run. + - name: "[Setup] Sweep leftovers of earlier runs" + run: bash e2e/cleanup.sh + - name: "[Setup] Create AWS source" run: | RES=$(curl -sS -X POST "$API/api/v1/sources" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ @@ -265,46 +270,9 @@ jobs: echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" } > out/summary.md - - name: "[Cleanup] Delete this run's resources in Better Stack and AWS (and stale ones)" + - name: "[Cleanup] Delete this run's resources in Better Stack and AWS" if: ${{ always() }} - run: | - set -o pipefail - mkdir -p out - say() { echo "$1"; echo "$1" >> out/cleanup.md; } - could_not_list() { say "- ❌ could not list $1, so none were deleted"; touch cleanup.failed; } - say "### Cleanup" - say "" - STALE_BEFORE=$(date -u -d '-2 hours' +%FT%TZ) - # Sources first: while the integration role still exists Better Stack can unsubscribe cleanly. - SOURCES=$(curl -sS --fail-with-body "$API/api/v1/sources?name=$SOURCE_PREFIX&per_page=250" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" \ - | jq -r --arg run " $RUN_ID" --arg before "$STALE_BEFORE" '.data[] | select((.attributes.name | endswith($run)) or .attributes.created_at < $before) | "\(.id) \(.attributes.name)"') || { could_not_list sources; SOURCES=""; } - while read -r id name; do - [ -n "$id" ] || continue - CODE=$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE "$API/api/v1/sources/$id" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API") - [ "$CODE" = 204 ] && say "- deleted source $id \`$name\`" || { say "- ❌ could not delete source $id \`$name\` (HTTP $CODE)"; touch cleanup.failed; } - done <<<"$SOURCES" - STACKS=$(aws cloudformation describe-stacks --query "Stacks[?starts_with(StackName, 'e2e-')].[StackName,CreationTime]" --output json \ - | jq -r --arg name "$STACK_NAME" --arg before "$STALE_BEFORE" '.[] | select(.[0] == $name or .[1] < $before) | .[0]') || { could_not_list stacks; STACKS=""; } - for stack in $STACKS; do - aws cloudformation delete-stack --stack-name "$stack" - if aws cloudformation wait stack-delete-complete --stack-name "$stack"; then say "- deleted stack \`$stack\`"; else - say "- ❌ could not delete stack \`$stack\`: $(aws cloudformation describe-stack-events --stack-name "$stack" --query "StackEvents[?ends_with(ResourceStatus, 'FAILED')].[LogicalResourceId,ResourceStatusReason]" --output text | head -5)"; touch cleanup.failed - fi - done - # The stack retains its buckets by design, and X-Ray creates aws/spans outside the stack. - ACCOUNT_ID=${ACCOUNT_ID:-$(aws sts get-caller-identity --query Account --output text || true)} - [ -n "$ACCOUNT_ID" ] || could_not_list buckets - for bucket in "better-stack-firehose-$ACCOUNT_ID-$AWS_REGION" "better-stack-cloudtrail-$ACCOUNT_ID-$AWS_REGION"; do - [ -n "$ACCOUNT_ID" ] && aws s3api head-bucket --bucket "$bucket" 2>/dev/null || continue - if aws s3 rb "s3://$bucket" --force >/dev/null; then say "- deleted bucket \`$bucket\`"; else say "- ❌ could not delete bucket \`$bucket\`"; touch cleanup.failed; fi - done - LOG_GROUPS=$(aws logs describe-log-groups --query 'logGroups[].[logGroupName,creationTime]' --output json \ - | jq -r --arg mine "$LOG_GROUP" --argjson before "$(( $(date -u -d '-2 hours' +%s) * 1000 ))" '.[] | select(.[0] == "aws/spans" or (.[0] | startswith("/e2e/")) and (.[0] == $mine or .[1] < $before)) | .[0]') || { could_not_list "log groups"; LOG_GROUPS=""; } - for group in $LOG_GROUPS; do - if aws logs delete-log-group --log-group-name "$group"; then say "- deleted log group \`$group\`"; else say "- ❌ could not delete log group \`$group\`"; touch cleanup.failed; fi - done - [ "$(wc -l < out/cleanup.md)" -gt 2 ] || say "- nothing to delete" - [ ! -f cleanup.failed ] || { say ""; say "❌ cleanup left resources behind"; exit 1; } + run: bash e2e/cleanup.sh - name: "[Report] Job summary" if: ${{ always() }} diff --git a/e2e/cleanup.sh b/e2e/cleanup.sh new file mode 100755 index 0000000..60ad359 --- /dev/null +++ b/e2e/cleanup.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Deletes everything the E2E workflow creates: its sources in Better Stack, every e2e-gh* stack in the AWS account +# (runs are named e2e-gha; e2e-bootstrap is the standing OIDC stack and must stay), the buckets the +# stack retains by design, aws/spans (X-Ray creates it outside the stack) and the /e2e/* log groups. The workflow runs +# this twice: before a run, so leftovers of a killed cleanup never block the next run, and after it. One run at a time +# per account (the workflow's concurrency group), so nothing with these names belongs to a live run. +# Writes a markdown report to out/cleanup.md and exits 1 when something could not be listed or deleted. +# Env: API, TEST_TEAM_TELEMETRY_API, SOURCE_PREFIX, AWS_REGION (+ AWS credentials for the account). +set -euo pipefail + +mkdir -p out +: > out/cleanup.md +say() { echo "$1"; echo "$1" >> out/cleanup.md; } +could_not_list() { say "- ❌ could not list $1, so none were deleted"; touch cleanup.failed; } +rm -f cleanup.failed +say "### Cleanup" +say "" + +# Sources first: while the integration role still exists Better Stack can unsubscribe cleanly. +SOURCES=$(curl -sS --fail-with-body "$API/api/v1/sources?name=$SOURCE_PREFIX&per_page=250" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" \ + | jq -r --arg prefix "$SOURCE_PREFIX " '.data[] | select(.attributes.name | startswith($prefix)) | "\(.id) \(.attributes.name)"') || { could_not_list sources; SOURCES=""; } +while read -r id name; do + [ -n "$id" ] || continue + CODE=$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE "$API/api/v1/sources/$id" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API") + [ "$CODE" = 204 ] && say "- deleted source $id \`$name\`" || { say "- ❌ could not delete source $id \`$name\` (HTTP $CODE)"; touch cleanup.failed; } +done <<<"$SOURCES" + +STACKS=$(aws cloudformation describe-stacks --query "Stacks[?starts_with(StackName, 'e2e-gh')].StackName" --output text) || { could_not_list stacks; STACKS=""; } +for stack in $STACKS; do + aws cloudformation delete-stack --stack-name "$stack" + if aws cloudformation wait stack-delete-complete --stack-name "$stack"; then say "- deleted stack \`$stack\`"; else + say "- ❌ could not delete stack \`$stack\`: $(aws cloudformation describe-stack-events --stack-name "$stack" --query "StackEvents[?ends_with(ResourceStatus, 'FAILED')].[LogicalResourceId,ResourceStatusReason]" --output text | head -5)"; touch cleanup.failed + fi +done + +ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) || { could_not_list buckets; ACCOUNT_ID=""; } +for bucket in "better-stack-firehose-$ACCOUNT_ID-$AWS_REGION" "better-stack-cloudtrail-$ACCOUNT_ID-$AWS_REGION"; do + [ -n "$ACCOUNT_ID" ] && aws s3api head-bucket --bucket "$bucket" 2>/dev/null || continue + if aws s3 rb "s3://$bucket" --force >/dev/null; then say "- deleted bucket \`$bucket\`"; else say "- ❌ could not delete bucket \`$bucket\`"; touch cleanup.failed; fi +done + +LOG_GROUPS=$(aws logs describe-log-groups --query "logGroups[?logGroupName == 'aws/spans' || starts_with(logGroupName, '/e2e/')].logGroupName" --output text) || { could_not_list "log groups"; LOG_GROUPS=""; } +for group in $LOG_GROUPS; do + if aws logs delete-log-group --log-group-name "$group"; then say "- deleted log group \`$group\`"; else say "- ❌ could not delete log group \`$group\`"; touch cleanup.failed; fi +done + +[ "$(wc -l < out/cleanup.md)" -gt 2 ] || say "- nothing to delete" +[ ! -f cleanup.failed ] || { say ""; say "❌ cleanup left resources behind"; exit 1; } From 18b0a40f77f766309f743af8273024e7721bad06 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 15 Oct 2026 19:35:14 +0200 Subject: [PATCH 03/12] Subscribe aws/spans and the CloudTrail group through the API A fresh customer account gets both subscribed by default. The sandbox account is permanent in Better Stack (one AwsAccount per organization and AWS account id, kept across source deletions), and each earlier run's stack deletion looked like an external unsubscribe to the sync's reconcile, which then never re-subscribed them. The run now creates explicit subscriptions for the two through the log-group-subscriptions API once X-Ray has created aws/spans; the run's own log group still relies on auto-subscribe. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e3b13f3..4e673c4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,7 +2,9 @@ name: "E2E" # Deploys the CloudFormation template into a throwaway AWS account the way a customer does: creates an AWS source, # deploys the stack with that source's ClusterId/SourceToken/SourceId, pastes the stack outputs back into the source, -# then produces logs, a custom metric, an X-Ray trace and a CloudTrail management event in AWS and reads all four back +# subscribes aws/spans and the CloudTrail group through the API (see that step for why the defaults cannot be relied on +# in a permanent sandbox account), then produces logs, a custom metric, an X-Ray trace and a CloudTrail management +# event in AWS and reads all four back # through the team's SQL API connection with exact row counts. Pull requests and pushes to main deploy the template # from the checkout; the daily schedule deploys the published one and fails when it differs from main. Everything the # run creates in AWS and Better Stack is named after RUN_ID and deleted in the cleanup step, together with leftovers of @@ -191,6 +193,25 @@ jobs: echo "::error::connecting the source failed: HTTP $CODE $(cat connect.json)"; exit 1 done + # A fresh customer account gets aws/spans and the CloudTrail group subscribed by default. This sandbox account is + # permanent in Better Stack (one record per organization and AWS account id, kept across source deletions), and + # every earlier run's stack deletion looked like an external unsubscribe to the sync's reconcile, which then never + # re-subscribes those two. They are subscribed here through the log-group-subscriptions API, which covers that + # surface too; the run's own log group stays on auto-subscribe. + - name: "[Test] Subscribe aws/spans and the CloudTrail group through the API" + run: | + DEADLINE=$((SECONDS + 180)) + until aws logs describe-log-groups --log-group-name-prefix aws/spans --query "logGroups[?logGroupName=='aws/spans'].logGroupName" --output text | grep -q spans; do + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::X-Ray did not create aws/spans within 3 min of the warm-up trace"; exit 1; } + sleep 10 + done + for group in aws/spans /aws/cloudtrail/better-stack; do + CODE=$(curl -sS -o subscription.json -w '%{http_code}' -X POST "$API/api/v2/sources/$SOURCE_ID/aws-log-group-subscriptions" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg r "$AWS_REGION" --arg n "$group" '{region: $r, name: $n, subscribed: true}')") + [ "$CODE" = 201 ] || { echo "::error::subscribing $group failed: HTTP $CODE $(cat subscription.json)"; exit 1; } + echo "subscription requested for $group" + done + - name: "[Test] Wait until the sync subscribed the log group, aws/spans and the CloudTrail group" run: | DEADLINE=$((SECONDS + 480)) From 782ede2ae411d62a48e7812d43e35f0df5ba1603 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 15 Oct 2026 19:53:42 +0200 Subject: [PATCH 04/12] Static and dynamic log groups, frozen subscription behaviour, logs in both Two log groups are created before the stack: one named after the run and one always named /e2e/static, which every run deletes and recreates. The sync must auto-subscribe the new one on its own. The static group, aws/spans and the CloudTrail group must stay unsubscribed, because the sandbox account record is permanent in Better Stack (T-21948) and a deleted and recreated group counts as explicitly unsubscribed (T-21949); that step names both issues and fails when either is fixed. The three are then subscribed through the log-group-subscriptions API, and 20 lines are written into each of the two groups and counted per group. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 75 ++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4e673c4..f65bedd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,9 +2,9 @@ name: "E2E" # Deploys the CloudFormation template into a throwaway AWS account the way a customer does: creates an AWS source, # deploys the stack with that source's ClusterId/SourceToken/SourceId, pastes the stack outputs back into the source, -# subscribes aws/spans and the CloudTrail group through the API (see that step for why the defaults cannot be relied on -# in a permanent sandbox account), then produces logs, a custom metric, an X-Ray trace and a CloudTrail management -# event in AWS and reads all four back +# checks that the sync auto-subscribes a new log group and leaves recreated ones alone (frozen behaviour, see the step +# naming T-21948 and T-21949), subscribes those through the API, then produces logs, a custom metric, an X-Ray trace and +# a CloudTrail management event in AWS and reads all four back # through the team's SQL API connection with exact row counts. Pull requests and pushes to main deploy the template # from the checkout; the daily schedule deploys the published one and fails when it differs from main. Everything the # run creates in AWS and Better Stack is named after RUN_ID and deleted in the cleanup step, together with leftovers of @@ -54,6 +54,8 @@ env: SOURCE_PREFIX: e2e-aws-stack STACK_NAME: e2e-gh${{ github.run_id }}a${{ github.run_attempt }} LOG_GROUP: /e2e/gh${{ github.run_id }}a${{ github.run_attempt }} + # Recreated under the same name by every run, so Better Stack sees it with the history of the previous runs. + STATIC_LOG_GROUP: /e2e/static MARKER: e2e run_id=gh${{ github.run_id }}a${{ github.run_attempt }} TEST_TEAM_TELEMETRY_API: ${{ secrets.TEST_TEAM_TELEMETRY_API }} TEST_TEAM_CONNECTION_USERNAME: ${{ secrets.TEST_TEAM_CONNECTION_USERNAME }} @@ -147,10 +149,12 @@ jobs: sleep 15 echo "ingest accepts the token" - - name: "[Setup] Create the log group before the stack, so the first sync finds it" + - name: "[Setup] Create the two log groups before the stack, so the first sync finds them" run: | - aws logs create-log-group --log-group-name "$LOG_GROUP" - aws logs create-log-stream --log-group-name "$LOG_GROUP" --log-stream-name run + for group in "$LOG_GROUP" "$STATIC_LOG_GROUP"; do + aws logs create-log-group --log-group-name "$group" + aws logs create-log-stream --log-group-name "$group" --log-stream-name run + done - name: "[Test] Deploy the CloudFormation stack" run: | @@ -193,34 +197,53 @@ jobs: echo "::error::connecting the source failed: HTTP $CODE $(cat connect.json)"; exit 1 done - # A fresh customer account gets aws/spans and the CloudTrail group subscribed by default. This sandbox account is - # permanent in Better Stack (one record per organization and AWS account id, kept across source deletions), and - # every earlier run's stack deletion looked like an external unsubscribe to the sync's reconcile, which then never - # re-subscribes those two. They are subscribed here through the log-group-subscriptions API, which covers that - # surface too; the run's own log group stays on auto-subscribe. - - name: "[Test] Subscribe aws/spans and the CloudTrail group through the API" + - name: "[Test] The sync subscribes the new log group by itself" + run: | + DEADLINE=$((SECONDS + 480)) + until aws logs describe-subscription-filters --log-group-name "$LOG_GROUP" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text | grep -q firehose; do + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::the sync did not subscribe $LOG_GROUP within 8 min"; exit 1; } + echo "waiting for the sync to subscribe $LOG_GROUP"; sleep 15 + done + echo "$LOG_GROUP carries the BetterStackLogs subscription filter" + + # Frozen behaviour. A fresh account would get aws/spans and the CloudTrail group subscribed by default and the + # static group by auto-subscribe. This sandbox account is permanent in Better Stack (T-21948), and all three + # groups are deleted and recreated by every run, which the sync's reconcile treats as an explicit unsubscribe + # (T-21949). So the sync that just subscribed the new group must have left these three alone. When either + # issue is fixed this step fails and the test moves on to the new behaviour. + - name: "[Test] The recreated static group, aws/spans and the CloudTrail group stay unsubscribed (see T-21948, T-21949)" run: | DEADLINE=$((SECONDS + 180)) until aws logs describe-log-groups --log-group-name-prefix aws/spans --query "logGroups[?logGroupName=='aws/spans'].logGroupName" --output text | grep -q spans; do [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::X-Ray did not create aws/spans within 3 min of the warm-up trace"; exit 1; } sleep 10 done - for group in aws/spans /aws/cloudtrail/better-stack; do + for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + if aws logs describe-subscription-filters --log-group-name "$group" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text | grep -q firehose; then + echo "::error::$group was subscribed by the sync although it was deleted and recreated: the behaviour this test freezes has changed (see T-21948, T-21949), update the test" + exit 1 + fi + echo "$group stays unsubscribed, as expected" + done + + - name: "[Test] Subscribe them through the log-group-subscriptions API" + run: | + for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do CODE=$(curl -sS -o subscription.json -w '%{http_code}' -X POST "$API/api/v2/sources/$SOURCE_ID/aws-log-group-subscriptions" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ -d "$(jq -nc --arg r "$AWS_REGION" --arg n "$group" '{region: $r, name: $n, subscribed: true}')") [ "$CODE" = 201 ] || { echo "::error::subscribing $group failed: HTTP $CODE $(cat subscription.json)"; exit 1; } echo "subscription requested for $group" done - - name: "[Test] Wait until the sync subscribed the log group, aws/spans and the CloudTrail group" + - name: "[Test] Wait until the sync applied the three subscriptions" run: | DEADLINE=$((SECONDS + 480)) while :; do PENDING="" - for group in "$LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do aws logs describe-subscription-filters --log-group-name "$group" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text 2>/dev/null | grep -q firehose || PENDING="$PENDING $group" done - [ -z "$PENDING" ] && { echo "all three log groups carry the BetterStackLogs subscription filter"; exit 0; } + [ -z "$PENDING" ] && { echo "all four log groups carry the BetterStackLogs subscription filter"; exit 0; } [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::no BetterStackLogs subscription filter after 8 min on:$PENDING"; exit 1; } echo "waiting for subscriptions on:$PENDING"; sleep 15 done @@ -228,8 +251,10 @@ jobs: - name: "[Test] Produce logs, a custom metric, a two-segment trace and a CloudTrail management event" run: | echo "SINCE=dt > toDateTime('$(date -u -d '-10 minutes' '+%F %T')')" >> "$GITHUB_ENV" - jq -nc --arg m "$MARKER" --arg run "$RUN_ID" '[range(1; 21) as $n | {timestamp: ((now * 1000 | floor) + $n), message: ({message: $m, e2e: {run_id: $run, n: $n}} | tojson)}]' > events.json - aws logs put-log-events --log-group-name "$LOG_GROUP" --log-stream-name run --log-events file://events.json >/dev/null + for group in "$LOG_GROUP" "$STATIC_LOG_GROUP"; do + jq -nc --arg m "$MARKER" --arg run "$RUN_ID" --arg g "$group" '[range(1; 21) as $n | {timestamp: ((now * 1000 | floor) + $n), message: ({message: $m, e2e: {run_id: $run, log_group: $g, n: $n}} | tojson)}]' > events.json + aws logs put-log-events --log-group-name "$group" --log-stream-name run --log-events file://events.json >/dev/null + done jq -nc --arg run "$RUN_ID" --arg t1 "$(date -u -d '-2 minutes' +%FT%TZ)" --arg t2 "$(date -u -d '-1 minute' +%FT%TZ)" --arg t3 "$(date -u +%FT%TZ)" \ '[$t1, $t2, $t3] | to_entries | map({MetricName: "RunMarker", Value: (.key + 1), Unit: "Count", Timestamp: .value, Dimensions: [{Name: "RunId", Value: $run}]})' > metrics.json aws cloudwatch put-metric-data --namespace E2eTest/AwsStack --metric-data file://metrics.json @@ -245,7 +270,7 @@ jobs: echo "TRACE_HEX=$(cut -d- -f2,3 <<<"$TRACE_ID" | tr -d -)" >> "$GITHUB_ENV" # A management event that happens after the trail exists and names this run. aws logs put-retention-policy --log-group-name "$LOG_GROUP" --retention-in-days 1 - echo "20 log events, 3 datapoints of E2eTest/AwsStack RunMarker, trace $TRACE_ID, PutRetentionPolicy on $LOG_GROUP" + echo "20 log events into each of $LOG_GROUP and $STATIC_LOG_GROUP, 3 datapoints of E2eTest/AwsStack RunMarker, trace $TRACE_ID, PutRetentionPolicy on $LOG_GROUP" - name: "[Test] Read everything back through the SQL API connection" run: | @@ -260,23 +285,24 @@ jobs: } count() { ROWS=$(sql "SELECT count() AS n FROM $1") || return 1; [ -n "$ROWS" ] && jq -r '.n' <<<"$ROWS" || echo 0; } counts() { - LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0") + LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$LOG_GROUP\"') > 0") + STATIC_LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$STATIC_LOG_GROUP\"') > 0") CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 AND position(raw, '$LOG_GROUP') > 0") SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'trace_id') = '$TRACE_HEX'") METRICS=$(count "remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' AND toString(tags) LIKE '%$RUN_ID%'") - echo "logs $LOGS/20, cloudtrail $CLOUDTRAIL/1, spans $SPANS/2, metric rows $METRICS" + echo "logs $LOGS/20 + static $STATIC_LOGS/20, cloudtrail $CLOUDTRAIL/1, spans $SPANS/2, metric rows $METRICS" } DEADLINE=$((SECONDS + 900)) while :; do counts - [ "$LOGS" -ge 20 ] && [ "$CLOUDTRAIL" -ge 1 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && break + [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$CLOUDTRAIL" -ge 1 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && break [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 15 min"; exit 1; } sleep 20 done # Everything arrived; a last look 60 s later catches duplicate delivery. sleep 60 counts - [ "$LOGS" = 20 ] && [ "$CLOUDTRAIL" = 1 ] && [ "$SPANS" = 2 ] || { echo "::error::unexpected row counts: logs $LOGS (20), cloudtrail $CLOUDTRAIL (1), spans $SPANS (2)"; exit 1; } + [ "$LOGS" = 20 ] && [ "$STATIC_LOGS" = 20 ] && [ "$CLOUDTRAIL" = 1 ] && [ "$SPANS" = 2 ] || { echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), cloudtrail $CLOUDTRAIL (1), spans $SPANS (2)"; exit 1; } mkdir -p out sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 LIMIT 1" > out/sample_cloudtrail.json @@ -286,7 +312,8 @@ jobs: echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" echo echo "- template from $TEMPLATE_FROM, source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`" - echo "- read back through \`$CONNECT_HOST\`: logs $LOGS, cloudtrail $CLOUDTRAIL, spans $SPANS, metric rows $METRICS" + echo "- \`$LOG_GROUP\` auto-subscribed; \`$STATIC_LOG_GROUP\`, \`aws/spans\` and \`/aws/cloudtrail/better-stack\` stayed unsubscribed after their recreation (T-21948, T-21949) and were subscribed through the API" + echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, cloudtrail $CLOUDTRAIL, spans $SPANS, metric rows $METRICS" echo echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" } > out/summary.md From df2b7f912df382213b52e01f91a6595a3d377e7f Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 15 Oct 2026 20:01:58 +0200 Subject: [PATCH 05/12] Count spans under the span key, give CloudTrail 25 minutes, check the stored trace id The mapper stores an X-Ray span as {dt, resource, span: {...}}, so the trace id and name live under span. Spans are counted by their names and the sample's stored trace ids must equal the X-Ray trace id in its 32-hex form. CloudTrail delivers management events within 15 minutes and the trail is minutes old at that point, so the read-back waits up to 25 minutes and, on a timeout, saves the newest CloudTrail and span rows for the summary. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f65bedd..f96b0e4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -288,15 +288,18 @@ jobs: LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$LOG_GROUP\"') > 0") STATIC_LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$STATIC_LOG_GROUP\"') > 0") CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 AND position(raw, '$LOG_GROUP') > 0") - SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'trace_id') = '$TRACE_HEX'") + # The mapper stores each X-Ray span as {dt, resource, span: {trace_id, span_id, name, ...}}. + SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID')") METRICS=$(count "remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' AND toString(tags) LIKE '%$RUN_ID%'") echo "logs $LOGS/20 + static $STATIC_LOGS/20, cloudtrail $CLOUDTRAIL/1, spans $SPANS/2, metric rows $METRICS" } - DEADLINE=$((SECONDS + 900)) + # CloudTrail promises management events within 15 minutes and the trail is minutes old; everything else + # arrives within a few minutes. + DEADLINE=$((SECONDS + 1500)) while :; do counts [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$CLOUDTRAIL" -ge 1 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && break - [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 15 min"; exit 1; } + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 25 min"; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'eventSource') > 0 ORDER BY dt DESC LIMIT 3" > out/sample_cloudtrail.json; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } sleep 20 done # Everything arrived; a last look 60 s later catches duplicate delivery. @@ -306,7 +309,10 @@ jobs: mkdir -p out sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 LIMIT 1" > out/sample_cloudtrail.json - sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'trace_id') = '$TRACE_HEX' ORDER BY JSONExtractUInt(raw, 'start_time_unix_nano')" > out/sample_spans.json + sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json + # The X-Ray trace id 1-- should be stored as the 32-hex id the mapper takes from Transaction Search. + STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') + [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || { echo "::error::stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"; exit 1; } sql "SELECT name, tags FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' GROUP BY name, tags LIMIT 3" > out/sample_metric.json { echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" From 48443594bbf0e5d9652feb02b0eaabe22126b0c3 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 15 Oct 2026 20:57:17 +0200 Subject: [PATCH 06/12] Freeze CloudTrail at no rows (T-21951), merge the frozen check with the API subscriptions CloudTrail writes its events into /aws/cloudtrail/better-stack and the subscription filter is in place, but CloudWatch Logs forwards none of CloudTrail's batches to the Firehose, while events written into the same group by hand are forwarded and a CloudTrail record pushed into the Firehose by hand reaches the source (T-21951). The read-back no longer waits for CloudTrail; it asserts that no CloudTrail row arrived and fails, naming the issue, once rows start flowing. The frozen-subscription check and the API subscriptions are one step now. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f96b0e4..480b59f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,7 +4,8 @@ name: "E2E" # deploys the stack with that source's ClusterId/SourceToken/SourceId, pastes the stack outputs back into the source, # checks that the sync auto-subscribes a new log group and leaves recreated ones alone (frozen behaviour, see the step # naming T-21948 and T-21949), subscribes those through the API, then produces logs, a custom metric, an X-Ray trace and -# a CloudTrail management event in AWS and reads all four back +# a CloudTrail management event in AWS and reads logs, spans and metrics back (CloudTrail rows are frozen at none, see +# T-21951 in the read-back step) # through the team's SQL API connection with exact row counts. Pull requests and pushes to main deploy the template # from the checkout; the daily schedule deploys the published one and fails when it differs from main. Everything the # run creates in AWS and Better Stack is named after RUN_ID and deleted in the cleanup step, together with leftovers of @@ -211,7 +212,7 @@ jobs: # groups are deleted and recreated by every run, which the sync's reconcile treats as an explicit unsubscribe # (T-21949). So the sync that just subscribed the new group must have left these three alone. When either # issue is fixed this step fails and the test moves on to the new behaviour. - - name: "[Test] The recreated static group, aws/spans and the CloudTrail group stay unsubscribed (see T-21948, T-21949)" + - name: "[Test] The recreated static group, aws/spans and the CloudTrail group stay unsubscribed (see T-21948, T-21949), then get subscribed through the API" run: | DEADLINE=$((SECONDS + 180)) until aws logs describe-log-groups --log-group-name-prefix aws/spans --query "logGroups[?logGroupName=='aws/spans'].logGroupName" --output text | grep -q spans; do @@ -225,9 +226,6 @@ jobs: fi echo "$group stays unsubscribed, as expected" done - - - name: "[Test] Subscribe them through the log-group-subscriptions API" - run: | for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do CODE=$(curl -sS -o subscription.json -w '%{http_code}' -X POST "$API/api/v2/sources/$SOURCE_ID/aws-log-group-subscriptions" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ -d "$(jq -nc --arg r "$AWS_REGION" --arg n "$group" '{region: $r, name: $n, subscribed: true}')") @@ -287,28 +285,30 @@ jobs: counts() { LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$LOG_GROUP\"') > 0") STATIC_LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$STATIC_LOG_GROUP\"') > 0") - CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 AND position(raw, '$LOG_GROUP') > 0") + # Any CloudTrail event at all; the trail writes dozens within minutes of the stack creation. + CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '\"eventSource\"') > 0") # The mapper stores each X-Ray span as {dt, resource, span: {trace_id, span_id, name, ...}}. SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID')") METRICS=$(count "remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' AND toString(tags) LIKE '%$RUN_ID%'") - echo "logs $LOGS/20 + static $STATIC_LOGS/20, cloudtrail $CLOUDTRAIL/1, spans $SPANS/2, metric rows $METRICS" + echo "logs $LOGS/20 + static $STATIC_LOGS/20, spans $SPANS/2, metric rows $METRICS, cloudtrail rows $CLOUDTRAIL (frozen at 0, see T-21951)" } - # CloudTrail promises management events within 15 minutes and the trail is minutes old; everything else - # arrives within a few minutes. - DEADLINE=$((SECONDS + 1500)) + DEADLINE=$((SECONDS + 900)) while :; do counts - [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$CLOUDTRAIL" -ge 1 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && break - [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 25 min"; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'eventSource') > 0 ORDER BY dt DESC LIMIT 3" > out/sample_cloudtrail.json; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } + [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && break + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 15 min"; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } sleep 20 done # Everything arrived; a last look 60 s later catches duplicate delivery. sleep 60 counts - [ "$LOGS" = 20 ] && [ "$STATIC_LOGS" = 20 ] && [ "$CLOUDTRAIL" = 1 ] && [ "$SPANS" = 2 ] || { echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), cloudtrail $CLOUDTRAIL (1), spans $SPANS (2)"; exit 1; } + [ "$LOGS" = 20 ] && [ "$STATIC_LOGS" = 20 ] && [ "$SPANS" = 2 ] || { echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2)"; exit 1; } + # Frozen behaviour: CloudWatch Logs does not forward what CloudTrail writes into /aws/cloudtrail/better-stack + # to the Firehose, although the trail delivers and the subscription filter is in place (T-21951). The trail + # has written dozens of events by now, so a working path shows rows here. + [ "$CLOUDTRAIL" = 0 ] || { echo "::error::$CLOUDTRAIL CloudTrail rows arrived: the behaviour this test freezes has changed (see T-21951), update the test"; exit 1; } mkdir -p out sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json - sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, 'PutRetentionPolicy') > 0 LIMIT 1" > out/sample_cloudtrail.json sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json # The X-Ray trace id 1-- should be stored as the 32-hex id the mapper takes from Transaction Search. STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') @@ -319,7 +319,7 @@ jobs: echo echo "- template from $TEMPLATE_FROM, source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`" echo "- \`$LOG_GROUP\` auto-subscribed; \`$STATIC_LOG_GROUP\`, \`aws/spans\` and \`/aws/cloudtrail/better-stack\` stayed unsubscribed after their recreation (T-21948, T-21949) and were subscribed through the API" - echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, cloudtrail $CLOUDTRAIL, spans $SPANS, metric rows $METRICS" + echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric rows $METRICS; CloudTrail rows $CLOUDTRAIL, frozen at 0 (T-21951)" echo echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" } > out/summary.md From 4771dcccaa0ac7cbae8df22880eeb5b8a0a25373 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 15 Oct 2026 21:11:33 +0200 Subject: [PATCH 07/12] Word the frozen CloudTrail check as within the run's lifetime Production accounts with older stacks do receive CloudTrail rows (T-21951 updated); the sandbox only ever sees a stack's first half hour. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 480b59f..21a5652 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -303,10 +303,11 @@ jobs: sleep 60 counts [ "$LOGS" = 20 ] && [ "$STATIC_LOGS" = 20 ] && [ "$SPANS" = 2 ] || { echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2)"; exit 1; } - # Frozen behaviour: CloudWatch Logs does not forward what CloudTrail writes into /aws/cloudtrail/better-stack - # to the Firehose, although the trail delivers and the subscription filter is in place (T-21951). The trail - # has written dozens of events by now, so a working path shows rows here. - [ "$CLOUDTRAIL" = 0 ] || { echo "::error::$CLOUDTRAIL CloudTrail rows arrived: the behaviour this test freezes has changed (see T-21951), update the test"; exit 1; } + # Frozen behaviour: within the run's lifetime CloudWatch Logs forwards none of what CloudTrail writes into + # /aws/cloudtrail/better-stack to the Firehose, although the trail delivers and the subscription filter is in + # place (T-21951; production accounts with older stacks do receive CloudTrail rows). The trail has written + # dozens of events by now, so a working path shows rows here. + [ "$CLOUDTRAIL" = 0 ] || { echo "::error::$CLOUDTRAIL CloudTrail rows arrived within the run: the behaviour this test freezes has changed (see T-21951), update the test"; exit 1; } mkdir -p out sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json From 0a3fe4fa2f6aa8d5ab9499e54c6f590a053c3303 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Thu, 15 Oct 2026 22:50:09 +0200 Subject: [PATCH 08/12] Wait out stale role sessions, require the run's CloudTrail event Every run deletes the template's IAM roles and the next one recreates them under the same names, while AWS keeps using sessions of the deleted roles for up to an hour. CloudWatch Logs holds the subscription role's session per log group name, so delivery from aws/spans, /aws/cloudtrail/better-stack and /e2e/static failed with a non-retryable error, the filter paused for 10 minutes and skipped what arrived meanwhile, even for a log group recreated with a new filter. CloudTrail holds its delivery role's session the same way and wrote nothing into its log group. Measured in the sandbox account: stale filters recovered exactly one hour after CloudWatch Logs had assumed the role, and a stack deployed an hour after the previous one forwarded all 123 CloudTrail events with no delivery error. A run now waits until 65 minutes after the latest e2e-gh* stack deletion, then refreshes its AWS credentials. The CloudTrail check requires this run's PutRetentionPolicy event as exactly one row instead of freezing CloudTrail at none, the read-back allows 20 minutes and prints each group's incoming, forwarded and delivery-error counts when something is missing. The sweep also deletes /aws/application-signals/data, which X-Ray Transaction Search creates outside the stack. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/e2e.yml | 77 ++++++++++++++++++++++++++++----------- e2e/cleanup.sh | 11 +++--- 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 21a5652..b855318 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,13 +4,12 @@ name: "E2E" # deploys the stack with that source's ClusterId/SourceToken/SourceId, pastes the stack outputs back into the source, # checks that the sync auto-subscribes a new log group and leaves recreated ones alone (frozen behaviour, see the step # naming T-21948 and T-21949), subscribes those through the API, then produces logs, a custom metric, an X-Ray trace and -# a CloudTrail management event in AWS and reads logs, spans and metrics back (CloudTrail rows are frozen at none, see -# T-21951 in the read-back step) -# through the team's SQL API connection with exact row counts. Pull requests and pushes to main deploy the template -# from the checkout; the daily schedule deploys the published one and fails when it differs from main. Everything the -# run creates in AWS and Better Stack is named after RUN_ID and deleted in the cleanup step, together with leftovers of -# earlier runs older than two hours. The docs check fails when the docs article stops naming a template toggle or -# output. Pull requests from forks are skipped: they get no OIDC token. +# a CloudTrail management event in AWS and reads all of them back through the team's SQL API connection with exact row +# counts. Pull requests and pushes to main deploy the template from the checkout; the daily schedule deploys the +# published one and fails when it differs from main. Everything the run creates in AWS and Better Stack is named after +# RUN_ID and deleted in the cleanup step, together with leftovers of earlier runs. A run that starts within an hour of +# the previous run's stack deletion first waits for the hour to pass (see that step). The docs check fails when the +# docs article stops naming a template toggle or output. Pull requests from forks are skipped: they get no OIDC token. # See the docs guide in https://betterstack.com/docs/logs/aws-cloudformation/ # # AWS account: better-stack-e2e-aws-stack (251917137055), a member of the Better Stack organization that holds nothing @@ -117,6 +116,32 @@ jobs: - name: "[Setup] Sweep leftovers of earlier runs" run: bash e2e/cleanup.sh + # Each run deletes the template's IAM roles and the next run recreates them under the same names, while AWS keeps + # using sessions of the deleted roles for up to an hour. CloudWatch Logs holds the subscription role's session per + # log group name: delivery with a deleted role's session fails with a non-retryable error, the filter pauses for + # 10 minutes, skips whatever arrives meanwhile, and fails again on every retry until the session expires, even + # when the log group is recreated with a new filter. aws/spans, /aws/cloudtrail/better-stack and the static group + # have the same names in every run. CloudTrail holds its role's session for writing into + # /aws/cloudtrail/better-stack the same way and writes nothing until it expires. Customers keep their roles. + - name: "[Setup] Wait until an hour has passed since the previous run deleted its stack" + id: cooldown + run: | + LAST=$(aws cloudformation list-stacks --stack-status-filter DELETE_COMPLETE --query "max(StackSummaries[?starts_with(StackName, 'e2e-gh')].DeletionTime)" --output text) + [ "$LAST" != None ] || { echo "no earlier stack in this account"; exit 0; } + WAIT=$(($(date -d "$LAST" +%s) + 3900 - $(date +%s))) + [ "$WAIT" -gt 0 ] || { echo "the previous stack was deleted at $LAST, over 65 min ago"; exit 0; } + echo "the previous stack was deleted at $LAST, waiting $((WAIT / 60 + 1)) min" + sleep "$WAIT" + echo "waited=true" >> "$GITHUB_OUTPUT" + + - name: "[Setup] Fresh AWS credentials after the wait" + if: ${{ steps.cooldown.outputs.waited == 'true' }} + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + role-duration-seconds: 7200 + - name: "[Setup] Create AWS source" run: | RES=$(curl -sS -X POST "$API/api/v1/sources" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ @@ -241,7 +266,7 @@ jobs: for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do aws logs describe-subscription-filters --log-group-name "$group" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text 2>/dev/null | grep -q firehose || PENDING="$PENDING $group" done - [ -z "$PENDING" ] && { echo "all four log groups carry the BetterStackLogs subscription filter"; exit 0; } + [ -z "$PENDING" ] && { echo "all three log groups carry the BetterStackLogs subscription filter"; exit 0; } [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::no BetterStackLogs subscription filter after 8 min on:$PENDING"; exit 1; } echo "waiting for subscriptions on:$PENDING"; sleep 15 done @@ -285,29 +310,38 @@ jobs: counts() { LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$LOG_GROUP\"') > 0") STATIC_LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$STATIC_LOG_GROUP\"') > 0") - # Any CloudTrail event at all; the trail writes dozens within minutes of the stack creation. - CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '\"eventSource\"') > 0") + # The mapper stores a CloudTrail event as {id, message: , context: {logGroup, ...}, dt}; this run's + # PutRetentionPolicy call names the run's log group. + CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'context', 'logGroup') = '/aws/cloudtrail/better-stack' AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'") # The mapper stores each X-Ray span as {dt, resource, span: {trace_id, span_id, name, ...}}. SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID')") METRICS=$(count "remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' AND toString(tags) LIKE '%$RUN_ID%'") - echo "logs $LOGS/20 + static $STATIC_LOGS/20, spans $SPANS/2, metric rows $METRICS, cloudtrail rows $CLOUDTRAIL (frozen at 0, see T-21951)" + echo "logs $LOGS/20 + static $STATIC_LOGS/20, spans $SPANS/2, metric rows $METRICS, CloudTrail event $CLOUDTRAIL/1" + } + # What CloudWatch Logs did with each group in the last 40 min; DeliveryErrors mean the filter paused for 10 min + # and skipped what arrived meanwhile. + forwarding() { + for group in "$LOG_GROUP" "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + for metric in IncomingLogEvents ForwardedLogEvents DeliveryErrors; do + DIMS="Name=LogGroupName,Value=$group" + [ "$metric" = IncomingLogEvents ] || DIMS="$DIMS Name=DestinationType,Value=Firehose Name=FilterName,Value=BetterStackLogs" + echo "$group $metric $(aws cloudwatch get-metric-statistics --namespace AWS/Logs --metric-name "$metric" --dimensions $DIMS \ + --start-time "$(date -u -d '-40 minutes' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 2400 --statistics Sum --query 'sum(Datapoints[].Sum)' --output text)" + done + done } - DEADLINE=$((SECONDS + 900)) + # CloudTrail hands events to CloudWatch Logs within about 5 min of the call, so allow 20 min. + DEADLINE=$((SECONDS + 1200)) while :; do counts - [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && break - [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 15 min"; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } + [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && [ "$CLOUDTRAIL" -ge 1 ] && break + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 20 min"; forwarding; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } sleep 20 done # Everything arrived; a last look 60 s later catches duplicate delivery. sleep 60 counts - [ "$LOGS" = 20 ] && [ "$STATIC_LOGS" = 20 ] && [ "$SPANS" = 2 ] || { echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2)"; exit 1; } - # Frozen behaviour: within the run's lifetime CloudWatch Logs forwards none of what CloudTrail writes into - # /aws/cloudtrail/better-stack to the Firehose, although the trail delivers and the subscription filter is in - # place (T-21951; production accounts with older stacks do receive CloudTrail rows). The trail has written - # dozens of events by now, so a working path shows rows here. - [ "$CLOUDTRAIL" = 0 ] || { echo "::error::$CLOUDTRAIL CloudTrail rows arrived within the run: the behaviour this test freezes has changed (see T-21951), update the test"; exit 1; } + [ "$LOGS" = 20 ] && [ "$STATIC_LOGS" = 20 ] && [ "$SPANS" = 2 ] && [ "$CLOUDTRAIL" = 1 ] || { echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)"; exit 1; } mkdir -p out sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json @@ -315,12 +349,13 @@ jobs: STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || { echo "::error::stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"; exit 1; } sql "SELECT name, tags FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' GROUP BY name, tags LIMIT 3" > out/sample_metric.json + sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'" > out/sample_cloudtrail.json { echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" echo echo "- template from $TEMPLATE_FROM, source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`" echo "- \`$LOG_GROUP\` auto-subscribed; \`$STATIC_LOG_GROUP\`, \`aws/spans\` and \`/aws/cloudtrail/better-stack\` stayed unsubscribed after their recreation (T-21948, T-21949) and were subscribed through the API" - echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric rows $METRICS; CloudTrail rows $CLOUDTRAIL, frozen at 0 (T-21951)" + echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric rows $METRICS, CloudTrail event $CLOUDTRAIL" echo echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" } > out/summary.md diff --git a/e2e/cleanup.sh b/e2e/cleanup.sh index 60ad359..a639a48 100755 --- a/e2e/cleanup.sh +++ b/e2e/cleanup.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # Deletes everything the E2E workflow creates: its sources in Better Stack, every e2e-gh* stack in the AWS account # (runs are named e2e-gha; e2e-bootstrap is the standing OIDC stack and must stay), the buckets the -# stack retains by design, aws/spans (X-Ray creates it outside the stack) and the /e2e/* log groups. The workflow runs -# this twice: before a run, so leftovers of a killed cleanup never block the next run, and after it. One run at a time -# per account (the workflow's concurrency group), so nothing with these names belongs to a live run. +# stack retains by design, aws/spans and /aws/application-signals/data (X-Ray Transaction Search creates them outside +# the stack) and the /e2e/* log groups. The workflow runs this twice: before a run, so leftovers of a killed cleanup +# never block the next run, and after it. One run at a time per account (the workflow's concurrency group), so nothing +# with these names belongs to a live run. # Writes a markdown report to out/cleanup.md and exits 1 when something could not be listed or deleted. # Env: API, TEST_TEAM_TELEMETRY_API, SOURCE_PREFIX, AWS_REGION (+ AWS credentials for the account). set -euo pipefail @@ -35,11 +36,11 @@ done ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) || { could_not_list buckets; ACCOUNT_ID=""; } for bucket in "better-stack-firehose-$ACCOUNT_ID-$AWS_REGION" "better-stack-cloudtrail-$ACCOUNT_ID-$AWS_REGION"; do - [ -n "$ACCOUNT_ID" ] && aws s3api head-bucket --bucket "$bucket" 2>/dev/null || continue + [ -n "$ACCOUNT_ID" ] && aws s3api head-bucket --bucket "$bucket" >/dev/null 2>&1 || continue if aws s3 rb "s3://$bucket" --force >/dev/null; then say "- deleted bucket \`$bucket\`"; else say "- ❌ could not delete bucket \`$bucket\`"; touch cleanup.failed; fi done -LOG_GROUPS=$(aws logs describe-log-groups --query "logGroups[?logGroupName == 'aws/spans' || starts_with(logGroupName, '/e2e/')].logGroupName" --output text) || { could_not_list "log groups"; LOG_GROUPS=""; } +LOG_GROUPS=$(aws logs describe-log-groups --query "logGroups[?logGroupName == 'aws/spans' || logGroupName == '/aws/application-signals/data' || starts_with(logGroupName, '/e2e/')].logGroupName" --output text) || { could_not_list "log groups"; LOG_GROUPS=""; } for group in $LOG_GROUPS; do if aws logs delete-log-group --log-group-name "$group"; then say "- deleted log group \`$group\`"; else say "- ❌ could not delete log group \`$group\`"; touch cleanup.failed; fi done From 540c07efc904f9c122343edd7d6256940e7c5b00 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Fri, 16 Oct 2026 11:16:27 +0200 Subject: [PATCH 09/12] Warn instead of waiting an hour after the previous run The hour-long wait goes. A step measures how long ago the latest e2e-gh* stack was deleted. More than 65 minutes ago, the read-back is strict as before and waits for the static group, aws/spans and the CloudTrail event. Within that hour it waits only for the run's own log group and the metric; the static group, aws/spans or CloudTrail delivering nothing becomes a warning in the log and the job summary, with the forwarding metrics printed. Any other wrong count still fails, and the span trace id is checked whenever the spans arrived. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/e2e.yml | 81 +++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b855318..a671e79 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -7,9 +7,10 @@ name: "E2E" # a CloudTrail management event in AWS and reads all of them back through the team's SQL API connection with exact row # counts. Pull requests and pushes to main deploy the template from the checkout; the daily schedule deploys the # published one and fails when it differs from main. Everything the run creates in AWS and Better Stack is named after -# RUN_ID and deleted in the cleanup step, together with leftovers of earlier runs. A run that starts within an hour of -# the previous run's stack deletion first waits for the hour to pass (see that step). The docs check fails when the -# docs article stops naming a template toggle or output. Pull requests from forks are skipped: they get no OIDC token. +# RUN_ID and deleted in the cleanup step, together with leftovers of earlier runs. Within an hour of the previous run's +# stack deletion, the static group, aws/spans and CloudTrail delivering nothing is a warning instead of a failure (see +# the step that measures it). The docs check fails when the docs article stops naming a template toggle or output. +# Pull requests from forks are skipped: they get no OIDC token. # See the docs guide in https://betterstack.com/docs/logs/aws-cloudformation/ # # AWS account: better-stack-e2e-aws-stack (251917137055), a member of the Better Stack organization that holds nothing @@ -123,24 +124,17 @@ jobs: # when the log group is recreated with a new filter. aws/spans, /aws/cloudtrail/better-stack and the static group # have the same names in every run. CloudTrail holds its role's session for writing into # /aws/cloudtrail/better-stack the same way and writes nothing until it expires. Customers keep their roles. - - name: "[Setup] Wait until an hour has passed since the previous run deleted its stack" - id: cooldown + # So within that hour the read-back waits only for the run's own log group and the metric, and the static group, + # aws/spans or CloudTrail delivering nothing is a warning. 65 min, because the roles go a minute or two after the + # stack deletion starts. + - name: "[Setup] Measure how long ago the previous run deleted its stack" run: | LAST=$(aws cloudformation list-stacks --stack-status-filter DELETE_COMPLETE --query "max(StackSummaries[?starts_with(StackName, 'e2e-gh')].DeletionTime)" --output text) - [ "$LAST" != None ] || { echo "no earlier stack in this account"; exit 0; } - WAIT=$(($(date -d "$LAST" +%s) + 3900 - $(date +%s))) - [ "$WAIT" -gt 0 ] || { echo "the previous stack was deleted at $LAST, over 65 min ago"; exit 0; } - echo "the previous stack was deleted at $LAST, waiting $((WAIT / 60 + 1)) min" - sleep "$WAIT" - echo "waited=true" >> "$GITHUB_OUTPUT" - - - name: "[Setup] Fresh AWS credentials after the wait" - if: ${{ steps.cooldown.outputs.waited == 'true' }} - uses: aws-actions/configure-aws-credentials@v5 - with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - role-duration-seconds: 7200 + [ "$LAST" != None ] || { echo "no earlier stack in this account, everything is checked strictly"; exit 0; } + AGE=$((($(date +%s) - $(date -d "$LAST" +%s)) / 60)) + [ "$AGE" -lt 65 ] || { echo "the previous stack was deleted $AGE min ago, everything is checked strictly"; exit 0; } + echo "RECENT_STACK_MIN=$AGE" >> "$GITHUB_ENV" + echo "the previous stack was deleted $AGE min ago: the static group, aws/spans and CloudTrail delivering nothing will be a warning" - name: "[Setup] Create AWS source" run: | @@ -330,26 +324,49 @@ jobs: done done } - # CloudTrail hands events to CloudWatch Logs within about 5 min of the call, so allow 20 min. + # Within an hour of the previous stack deletion the static group, aws/spans and CloudTrail may deliver nothing + # at all, so only the run's own log group and the metric are waited for (see the step measuring that hour). + # Otherwise everything is: CloudTrail hands events to CloudWatch Logs within about 5 min of the call. + if [ -n "${RECENT_STACK_MIN:-}" ]; then + arrived() { [ "$LOGS" -ge 20 ] && [ "$METRICS" -ge 1 ]; } + else + arrived() { [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && [ "$CLOUDTRAIL" -ge 1 ]; } + fi DEADLINE=$((SECONDS + 1200)) while :; do counts - [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && [ "$CLOUDTRAIL" -ge 1 ] && break + arrived && break [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 20 min"; forwarding; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } sleep 20 done - # Everything arrived; a last look 60 s later catches duplicate delivery. + # A last look 60 s later catches duplicate delivery. Every count must be exact; within that hour a fixed-name + # source that delivered nothing is a warning instead. sleep 60 counts - [ "$LOGS" = 20 ] && [ "$STATIC_LOGS" = 20 ] && [ "$SPANS" = 2 ] && [ "$CLOUDTRAIL" = 1 ] || { echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)"; exit 1; } + MISSING=() + for check in "logs:$LOGS:20" "static:$STATIC_LOGS:20" "spans:$SPANS:2" "cloudtrail:$CLOUDTRAIL:1"; do + IFS=: read -r name got want <<<"$check" + [ "$got" = "$want" ] && continue + if [ -n "${RECENT_STACK_MIN:-}" ] && [ "$name" != logs ] && [ "$got" = 0 ]; then MISSING+=("$name"); continue; fi + echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)" + exit 1 + done + if [ "${#MISSING[@]}" -gt 0 ]; then + NAMES=$(printf '%s\n' "${MISSING[@]}" | sed -e "s#^static\$#the logs of $STATIC_LOG_GROUP#" -e 's#^spans$#the spans#' -e 's#^cloudtrail$#the CloudTrail event#' | paste -sd, - | sed 's/,/, /g') + WARNING="Not verified: $NAMES. The previous run deleted its stack $RECENT_STACK_MIN min before this one, and for up to an hour AWS keeps using sessions of the IAM roles every run recreates under the same names, so CloudWatch Logs and CloudTrail can skip this data. A run more than an hour after the previous one checks it." + echo "::warning::$WARNING" + forwarding + fi mkdir -p out sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json - sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json - # The X-Ray trace id 1-- should be stored as the 32-hex id the mapper takes from Transaction Search. - STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') - [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || { echo "::error::stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"; exit 1; } + if [ "$SPANS" = 2 ]; then + sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json + # The X-Ray trace id 1-- should be stored as the 32-hex id the mapper takes from Transaction Search. + STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') + [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || { echo "::error::stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"; exit 1; } + fi sql "SELECT name, tags FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' GROUP BY name, tags LIMIT 3" > out/sample_metric.json - sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'" > out/sample_cloudtrail.json + [ "$CLOUDTRAIL" = 0 ] || sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'" > out/sample_cloudtrail.json { echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" echo @@ -357,7 +374,13 @@ jobs: echo "- \`$LOG_GROUP\` auto-subscribed; \`$STATIC_LOG_GROUP\`, \`aws/spans\` and \`/aws/cloudtrail/better-stack\` stayed unsubscribed after their recreation (T-21948, T-21949) and were subscribed through the API" echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric rows $METRICS, CloudTrail event $CLOUDTRAIL" echo - echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" + if [ "${#MISSING[@]}" -gt 0 ]; then + echo "⚠️ $WARNING" + echo + echo "✅ everything else round-tripped through the stack" + else + echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" + fi } > out/summary.md - name: "[Cleanup] Delete this run's resources in Better Stack and AWS" From d122e6bb6400b91a4ab9bc89e6ca21e5f1a28b0f Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Fri, 16 Oct 2026 12:28:44 +0200 Subject: [PATCH 10/12] Drop the Pawel branch from the Linear failure notification This workflow only ever assigns Petr, so the assignee input, the Pawel and Collectors secrets and the branch choosing between them go. The issue lands in the Telemetry team, state Plan, project Ingestion for everything, priority High, assigned to Petr. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/e2e.yml | 2 -- .github/workflows/notify_linear_failure.yml | 20 ++------------------ 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a671e79..acb74e1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -400,6 +400,4 @@ jobs: if: ${{ failure() && (github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }} needs: [test] uses: ./.github/workflows/notify_linear_failure.yml - with: - assignee: petr secrets: inherit diff --git a/.github/workflows/notify_linear_failure.yml b/.github/workflows/notify_linear_failure.yml index 30f9428..529be28 100644 --- a/.github/workflows/notify_linear_failure.yml +++ b/.github/workflows/notify_linear_failure.yml @@ -2,11 +2,6 @@ name: "Notify Linear on failure" on: workflow_call: - inputs: - assignee: - description: "Assignee: pawel or petr" - required: true - type: string jobs: create-linear-issue: @@ -45,10 +40,7 @@ jobs: LINEAR_TEAM_ID: ${{ secrets.LINEAR_TEAM_TELEMETRY_ID }} LINEAR_STATUS_PLAN_ID: ${{ secrets.LINEAR_STATUS_PLAN_ID }} LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID: ${{ secrets.LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID }} - LINEAR_PROJECT_COLLECTORS_ID: ${{ secrets.LINEAR_PROJECT_COLLECTORS_ID }} - LINEAR_ASSIGNEE_PAWEL_ID: ${{ secrets.LINEAR_ASSIGNEE_PAWEL_ID }} LINEAR_ASSIGNEE_PETR_ID: ${{ secrets.LINEAR_ASSIGNEE_PETR_ID }} - ASSIGNEE: ${{ inputs.assignee }} WORKFLOW: ${{ github.event.workflow_run.name || github.workflow }} REF_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }} RUN_URL: ${{ github.event.workflow_run.html_url || format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} @@ -58,14 +50,6 @@ jobs: COMMIT_MESSAGE: ${{ github.event.workflow_run.head_commit.message || github.event.head_commit.message }} FAILED_JOBS: ${{ steps.ctx.outputs.failed_jobs }} run: | - if [ "$ASSIGNEE" = "pawel" ]; then - ASSIGNEE_ID="$LINEAR_ASSIGNEE_PAWEL_ID" - PROJECT_ID="$LINEAR_PROJECT_COLLECTORS_ID" - else - ASSIGNEE_ID="$LINEAR_ASSIGNEE_PETR_ID" - PROJECT_ID="$LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID" - fi - SHORT_SHA="${SHA:0:7}" # head_commit is only populated on push events; skip the line otherwise @@ -97,9 +81,9 @@ jobs: --arg title "$TITLE" \ --arg description "$DESCRIPTION" \ --arg teamId "$LINEAR_TEAM_ID" \ - --arg assigneeId "$ASSIGNEE_ID" \ + --arg assigneeId "$LINEAR_ASSIGNEE_PETR_ID" \ --arg stateId "$LINEAR_STATUS_PLAN_ID" \ - --arg projectId "$PROJECT_ID" \ + --arg projectId "$LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID" \ '{ query: "mutation IssueCreate($input: IssueCreateInput!) { issueCreate(input: $input) { success issue { identifier url } } }", variables: { From 6701c004508c536f523bdc2a9b64b545b51b2dea Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Fri, 16 Oct 2026 12:29:30 +0200 Subject: [PATCH 11/12] Check every metric value, show the values in the sample The read-back only counted metric rows and the sample showed name and tags, so wrong values would have passed. The three datapoints now carry 1, 2 and 3 in three consecutive minutes. The mapper stores each minute's aggregate as the metric plus _max, _min, _sum and _count, and all 15 points must hold exactly what was sent (1 for _count), comparing the lowest and highest stored value per point and grouping by minute, since AWS does not document whether the metric stream stamps a minute's start. The sample shows the metric's three minutes with their values. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/e2e.yml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index acb74e1..9a663b2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -272,9 +272,12 @@ jobs: jq -nc --arg m "$MARKER" --arg run "$RUN_ID" --arg g "$group" '[range(1; 21) as $n | {timestamp: ((now * 1000 | floor) + $n), message: ({message: $m, e2e: {run_id: $run, log_group: $g, n: $n}} | tojson)}]' > events.json aws logs put-log-events --log-group-name "$group" --log-stream-name run --log-events file://events.json >/dev/null done - jq -nc --arg run "$RUN_ID" --arg t1 "$(date -u -d '-2 minutes' +%FT%TZ)" --arg t2 "$(date -u -d '-1 minute' +%FT%TZ)" --arg t3 "$(date -u +%FT%TZ)" \ - '[$t1, $t2, $t3] | to_entries | map({MetricName: "RunMarker", Value: (.key + 1), Unit: "Count", Timestamp: .value, Dimensions: [{Name: "RunId", Value: $run}]})' > metrics.json + # Values 1, 2 and 3 in three consecutive minutes; the read-back checks each minute's value. + T=$(date +%s) + jq -nc --arg run "$RUN_ID" --argjson t "$T" \ + '[range(3) | {MetricName: "RunMarker", Value: (. + 1), Unit: "Count", Timestamp: ($t - (2 - .) * 60 | todate), Dimensions: [{Name: "RunId", Value: $run}]}]' > metrics.json aws cloudwatch put-metric-data --namespace E2eTest/AwsStack --metric-data file://metrics.json + echo "METRIC_MINUTES=$(((T - 120) / 60 * 60)) $(((T - 60) / 60 * 60)) $((T / 60 * 60))" >> "$GITHUB_ENV" NOW=$(date +%s) TRACE_ID="1-$(printf '%x' "$NOW")-$(openssl rand -hex 12)" PARENT_ID=$(openssl rand -hex 8) @@ -301,6 +304,8 @@ jobs: cat response.json } count() { ROWS=$(sql "SELECT count() AS n FROM $1") || return 1; [ -n "$ROWS" ] && jq -r '.n' <<<"$ROWS" || echo 0; } + METRIC=e2e_test.aws_stack.run_marker + METRIC_FILTER="tags['run_id'] = '$RUN_ID' AND startsWith(name, '$METRIC')" counts() { LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$LOG_GROUP\"') > 0") STATIC_LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$STATIC_LOG_GROUP\"') > 0") @@ -309,8 +314,10 @@ jobs: CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'context', 'logGroup') = '/aws/cloudtrail/better-stack' AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'") # The mapper stores each X-Ray span as {dt, resource, span: {trace_id, span_id, name, ...}}. SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID')") - METRICS=$(count "remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' AND toString(tags) LIKE '%$RUN_ID%'") - echo "logs $LOGS/20 + static $STATIC_LOGS/20, spans $SPANS/2, metric rows $METRICS, CloudTrail event $CLOUDTRAIL/1" + # The metric stream sends one aggregate per minute; the mapper stores it as the metric plus _max, _min, _sum and + # _count, so three minutes make 15 points. + METRICS=$(count "(SELECT name, dt FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND $METRIC_FILTER GROUP BY name, dt)") + echo "logs $LOGS/20 + static $STATIC_LOGS/20, spans $SPANS/2, metric points $METRICS/15, CloudTrail event $CLOUDTRAIL/1" } # What CloudWatch Logs did with each group in the last 40 min; DeliveryErrors mean the filter paused for 10 min # and skipped what arrived meanwhile. @@ -328,9 +335,9 @@ jobs: # at all, so only the run's own log group and the metric are waited for (see the step measuring that hour). # Otherwise everything is: CloudTrail hands events to CloudWatch Logs within about 5 min of the call. if [ -n "${RECENT_STACK_MIN:-}" ]; then - arrived() { [ "$LOGS" -ge 20 ] && [ "$METRICS" -ge 1 ]; } + arrived() { [ "$LOGS" -ge 20 ] && [ "$METRICS" -ge 15 ]; } else - arrived() { [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 1 ] && [ "$CLOUDTRAIL" -ge 1 ]; } + arrived() { [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 15 ] && [ "$CLOUDTRAIL" -ge 1 ]; } fi DEADLINE=$((SECONDS + 1200)) while :; do @@ -351,6 +358,15 @@ jobs: echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)" exit 1 done + # Each minute must hold exactly the value sent in it: 1, 2 and 3 for the metric, _max, _min and _sum, 1 for _count. + # The lowest and highest stored value per point catch a re-sent minute with a different value. Points are + # compared by minute, since AWS does not document whether the stream stamps a minute's start. + EXPECTED=$(jq -nc --arg n "$METRIC" --arg m "$METRIC_MINUTES" '[$m | split(" ") | to_entries[] | (.value | tonumber) as $t | (.key + 1) as $v + | ({name: $n, v: $v}, {name: "\($n)_max", v: $v}, {name: "\($n)_min", v: $v}, {name: "\($n)_sum", v: $v}, {name: "\($n)_count", v: 1}) + | {name, minute: $t, min: .v, max: .v}] | sort_by(.name, .minute)') + ACTUAL=$(sql "SELECT name, toUnixTimestamp(dt) AS t, minMerge(value_min) AS min, maxMerge(value_max) AS max FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND $METRIC_FILTER GROUP BY name, dt" \ + | jq -sc 'map({name, minute: (((.t | tonumber) / 60 | floor) * 60), min: (.min | tonumber), max: (.max | tonumber)}) | sort_by(.name, .minute)') + [ "$ACTUAL" = "$EXPECTED" ] || { echo "::error::metric points differ from the datapoints sent: expected $EXPECTED, got $ACTUAL"; exit 1; } if [ "${#MISSING[@]}" -gt 0 ]; then NAMES=$(printf '%s\n' "${MISSING[@]}" | sed -e "s#^static\$#the logs of $STATIC_LOG_GROUP#" -e 's#^spans$#the spans#' -e 's#^cloudtrail$#the CloudTrail event#' | paste -sd, - | sed 's/,/, /g') WARNING="Not verified: $NAMES. The previous run deleted its stack $RECENT_STACK_MIN min before this one, and for up to an hour AWS keeps using sessions of the IAM roles every run recreates under the same names, so CloudWatch Logs and CloudTrail can skip this data. A run more than an hour after the previous one checks it." @@ -365,14 +381,14 @@ jobs: STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || { echo "::error::stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"; exit 1; } fi - sql "SELECT name, tags FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND name = 'e2e_test.aws_stack.run_marker' GROUP BY name, tags LIMIT 3" > out/sample_metric.json + sql "SELECT name, dt, tags, avgMerge(value_avg) AS value FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND tags['run_id'] = '$RUN_ID' AND name = '$METRIC' GROUP BY name, dt, tags ORDER BY dt" > out/sample_metric.json [ "$CLOUDTRAIL" = 0 ] || sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'" > out/sample_cloudtrail.json { echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" echo echo "- template from $TEMPLATE_FROM, source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`" echo "- \`$LOG_GROUP\` auto-subscribed; \`$STATIC_LOG_GROUP\`, \`aws/spans\` and \`/aws/cloudtrail/better-stack\` stayed unsubscribed after their recreation (T-21948, T-21949) and were subscribed through the API" - echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric rows $METRICS, CloudTrail event $CLOUDTRAIL" + echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric points $METRICS with the values sent, CloudTrail event $CLOUDTRAIL" echo if [ "${#MISSING[@]}" -gt 0 ]; then echo "⚠️ $WARNING" From 4cb828f6b04270bb4b68d354bca6a541b3384a4b Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Fri, 16 Oct 2026 12:30:06 +0200 Subject: [PATCH 12/12] Remove e2e/bootstrap.yaml The bootstrap stack is deployed once per account and already exists, so the template does not need to live in this repository; the header still names the standing e2e-bootstrap stack. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/e2e.yml | 8 ++++---- e2e/bootstrap.yaml | 42 --------------------------------------- 2 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 e2e/bootstrap.yaml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9a663b2..f9b7e57 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -15,10 +15,10 @@ name: "E2E" # # AWS account: better-stack-e2e-aws-stack (251917137055), a member of the Better Stack organization that holds nothing # but this test. Root sign-in: 1Password, Engineering vault, item "AWS better-stack-e2e-aws-stack". The one standing -# resource is the CloudFormation stack `e2e-bootstrap` from e2e/bootstrap.yaml: a GitHub OIDC provider and IAM role -# `github-actions-e2e` with AdministratorAccess trusting this repository. The template deploys IAM roles, Lambdas, -# Firehose streams, CloudTrail and S3, and its role and bucket names are fixed per account, so administrator access in -# an otherwise empty account is the honest scope and only one run may deploy at a time (the concurrency group below). +# resource is the CloudFormation stack `e2e-bootstrap`: a GitHub OIDC provider and IAM role `github-actions-e2e` with +# AdministratorAccess trusting this repository. The template deploys IAM roles, Lambdas, Firehose streams, CloudTrail +# and S3, and its role and bucket names are fixed per account, so administrator access in an otherwise empty account is +# the honest scope and only one run may deploy at a time (the concurrency group below). # Secrets: TEST_TEAM_TELEMETRY_API, TEST_TEAM_CONNECTION_USERNAME, TEST_TEAM_CONNECTION_PASSWORD (the E2E testing team), # LINEAR_* (see notify_linear_failure.yml). diff --git a/e2e/bootstrap.yaml b/e2e/bootstrap.yaml deleted file mode 100644 index 96c260d..0000000 --- a/e2e/bootstrap.yaml +++ /dev/null @@ -1,42 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: One-time bootstrap of an E2E CI account. GitHub Actions of one repository assume CiRole through OIDC; everything else in the account is created and deleted per workflow run. -Parameters: - GitHubRepository: - Type: String - Description: The owner/name of the GitHub repository whose workflows may assume the CI role. -Resources: - GitHubOidcProvider: - Type: AWS::IAM::OIDCProvider - Properties: - Url: https://token.actions.githubusercontent.com - ClientIdList: [sts.amazonaws.com] - ThumbprintList: [6938fd4d98bab03faadb97b34396831e3780aea1] - CiRole: - Type: AWS::IAM::Role - Properties: - RoleName: github-actions-e2e - MaxSessionDuration: 7200 - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: { Federated: !GetAtt GitHubOidcProvider.Arn } - Action: sts:AssumeRoleWithWebIdentity - Condition: - StringEquals: { 'token.actions.githubusercontent.com:aud': sts.amazonaws.com } - StringLike: { 'token.actions.githubusercontent.com:sub': !Sub 'repo:${GitHubRepository}:*' } - ManagedPolicyArns: [arn:aws:iam::aws:policy/AdministratorAccess] - LambdaExecutionRole: - Type: AWS::IAM::Role - Properties: - RoleName: e2e-lambda-exec - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: { Service: lambda.amazonaws.com } - Action: sts:AssumeRole - ManagedPolicyArns: [arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole] -Outputs: - CiRoleArn: { Value: !GetAtt CiRole.Arn } - LambdaExecutionRoleArn: { Value: !GetAtt LambdaExecutionRole.Arn }