diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..f9b7e57 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,419 @@ +name: "E2E" + +# Deploys the CloudFormation template into a throwaway AWS account the way a customer does: creates an AWS source, +# deploys the stack with that source's ClusterId/SourceToken/SourceId, pastes the stack outputs back into the source, +# checks that the sync auto-subscribes a new log group and leaves recreated ones alone (frozen behaviour, see the step +# naming T-21948 and T-21949), subscribes those through the API, then produces logs, a custom metric, an X-Ray trace and +# a CloudTrail management event in AWS and reads all of them back through the team's SQL API connection with exact row +# counts. Pull requests and pushes to main deploy the template from the checkout; the daily schedule deploys the +# published one and fails when it differs from main. Everything the run creates in AWS and Better Stack is named after +# RUN_ID and deleted in the cleanup step, together with leftovers of earlier runs. Within an hour of the previous run's +# stack deletion, the static group, aws/spans and CloudTrail delivering nothing is a warning instead of a failure (see +# the step that measures it). The docs check fails when the docs article stops naming a template toggle or output. +# Pull requests from forks are skipped: they get no OIDC token. +# See the docs guide in https://betterstack.com/docs/logs/aws-cloudformation/ +# +# AWS account: better-stack-e2e-aws-stack (251917137055), a member of the Better Stack organization that holds nothing +# but this test. Root sign-in: 1Password, Engineering vault, item "AWS better-stack-e2e-aws-stack". The one standing +# resource is the CloudFormation stack `e2e-bootstrap`: a GitHub OIDC provider and IAM role `github-actions-e2e` with +# AdministratorAccess trusting this repository. The template deploys IAM roles, Lambdas, Firehose streams, CloudTrail +# and S3, and its role and bucket names are fixed per account, so administrator access in an otherwise empty account is +# the honest scope and only one run may deploy at a time (the concurrency group below). +# Secrets: TEST_TEAM_TELEMETRY_API, TEST_TEAM_CONNECTION_USERNAME, TEST_TEAM_CONNECTION_PASSWORD (the E2E testing team), +# LINEAR_* (see notify_linear_failure.yml). + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: '15 5 * * *' + workflow_dispatch: + inputs: + template_url: + description: 'Template to deploy instead of the checkout (any URL)' + type: string + +permissions: + id-token: write + contents: read + actions: read + +# The template names its IAM roles and buckets globally, so one stack per account at a time. +concurrency: + group: e2e-aws-stack + cancel-in-progress: false + +env: + AWS_REGION: us-east-1 + AWS_ROLE_ARN: arn:aws:iam::251917137055:role/github-actions-e2e + API: https://telemetry.betterstack.com + DOCS_URL: https://betterstack.com/docs/logs/aws-cloudformation/ + PUBLISHED_TEMPLATE_URL: https://better-stack-cloudformation.s3.amazonaws.com/better-stack-full.yaml + TEMPLATE_URL_OVERRIDE: ${{ inputs.template_url }} + RUN_ID: gh${{ github.run_id }}a${{ github.run_attempt }} + SOURCE_PREFIX: e2e-aws-stack + STACK_NAME: e2e-gh${{ github.run_id }}a${{ github.run_attempt }} + LOG_GROUP: /e2e/gh${{ github.run_id }}a${{ github.run_attempt }} + # Recreated under the same name by every run, so Better Stack sees it with the history of the previous runs. + STATIC_LOG_GROUP: /e2e/static + MARKER: e2e run_id=gh${{ github.run_id }}a${{ github.run_attempt }} + TEST_TEAM_TELEMETRY_API: ${{ secrets.TEST_TEAM_TELEMETRY_API }} + TEST_TEAM_CONNECTION_USERNAME: ${{ secrets.TEST_TEAM_CONNECTION_USERNAME }} + TEST_TEAM_CONNECTION_PASSWORD: ${{ secrets.TEST_TEAM_CONNECTION_PASSWORD }} + +jobs: + test: + name: Stack round trip + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: "[Setup] Checkout code" + uses: actions/checkout@v7 + + - name: "[Check] Pick the template, docs article names every toggle and output, published template vs main" + run: | + if [ -n "$TEMPLATE_URL_OVERRIDE" ]; then + curl -sSfL "$TEMPLATE_URL_OVERRIDE" -o template.yaml; TEMPLATE_FROM="$TEMPLATE_URL_OVERRIDE" + elif [ "${{ github.event_name }}" = schedule ]; then + curl -sSfL "$PUBLISHED_TEMPLATE_URL" -o template.yaml; TEMPLATE_FROM="$PUBLISHED_TEMPLATE_URL" + else + cp cloudformation/full/better-stack-full.yaml template.yaml; TEMPLATE_FROM="this checkout" + fi + echo "TEMPLATE_FROM=$TEMPLATE_FROM ($(git rev-parse --short HEAD))" >> "$GITHUB_ENV" + curl -sSfL "$PUBLISHED_TEMPLATE_URL" -o template-published.yaml + if ! diff -q template-published.yaml cloudformation/full/better-stack-full.yaml >/dev/null; then + if [ "${{ github.event_name }}" = schedule ]; then + echo "::error::$PUBLISHED_TEMPLATE_URL differs from cloudformation/full/better-stack-full.yaml on main: the published template was not updated after a merge" + diff template-published.yaml cloudformation/full/better-stack-full.yaml | head -40 + exit 1 + fi + echo "::warning::$PUBLISHED_TEMPLATE_URL differs from the template in this checkout (expected while a change is unreleased)" + fi + TOGGLES=$(awk '/^Parameters:/{p=1;next} /^[A-Z][A-Za-z]*:/{p=0} p && /^ [A-Za-z]+:$/{name=substr($1,1,length($1)-1)} p && /^ - .false.$/{print name}' template.yaml | sort -u) + OUTPUTS=$(awk '/^Outputs:/{p=1;next} p && /^ [A-Za-z]+:$/{print substr($1,1,length($1)-1)}' template.yaml) + [ -n "$TOGGLES" ] && [ -n "$OUTPUTS" ] || { echo "::error::found no true/false parameters or no outputs in the template, this check needs updating"; exit 1; } + curl -sSfL "$DOCS_URL" -o docs.html + MISSING="" + for name in $TOGGLES $OUTPUTS https://github.com/BetterStackHQ/aws; do grep -q "$name" docs.html || MISSING="$MISSING $name"; done + [ -z "$MISSING" ] || { echo "::error::$DOCS_URL does not mention:$MISSING"; exit 1; } + echo "template from $TEMPLATE_FROM; docs article mentions every toggle ($(echo $TOGGLES)) and output ($(echo $OUTPUTS))" + + - name: "[Setup] AWS credentials (OIDC)" + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + # Stack creation and deletion can take 20 min each; the role allows 2 h sessions. + role-duration-seconds: 7200 + + - name: "[Check] Template is valid CloudFormation" + run: aws cloudformation validate-template --template-body file://template.yaml >/dev/null + + # A cancelled job gets only a short grace period, so its cleanup can be killed while the stack is still being + # deleted; sweeping first keeps such leftovers from failing this run. + - name: "[Setup] Sweep leftovers of earlier runs" + run: bash e2e/cleanup.sh + + # Each run deletes the template's IAM roles and the next run recreates them under the same names, while AWS keeps + # using sessions of the deleted roles for up to an hour. CloudWatch Logs holds the subscription role's session per + # log group name: delivery with a deleted role's session fails with a non-retryable error, the filter pauses for + # 10 minutes, skips whatever arrives meanwhile, and fails again on every retry until the session expires, even + # when the log group is recreated with a new filter. aws/spans, /aws/cloudtrail/better-stack and the static group + # have the same names in every run. CloudTrail holds its role's session for writing into + # /aws/cloudtrail/better-stack the same way and writes nothing until it expires. Customers keep their roles. + # So within that hour the read-back waits only for the run's own log group and the metric, and the static group, + # aws/spans or CloudTrail delivering nothing is a warning. 65 min, because the roles go a minute or two after the + # stack deletion starts. + - name: "[Setup] Measure how long ago the previous run deleted its stack" + run: | + LAST=$(aws cloudformation list-stacks --stack-status-filter DELETE_COMPLETE --query "max(StackSummaries[?starts_with(StackName, 'e2e-gh')].DeletionTime)" --output text) + [ "$LAST" != None ] || { echo "no earlier stack in this account, everything is checked strictly"; exit 0; } + AGE=$((($(date +%s) - $(date -d "$LAST" +%s)) / 60)) + [ "$AGE" -lt 65 ] || { echo "the previous stack was deleted $AGE min ago, everything is checked strictly"; exit 0; } + echo "RECENT_STACK_MIN=$AGE" >> "$GITHUB_ENV" + echo "the previous stack was deleted $AGE min ago: the static group, aws/spans and CloudTrail delivering nothing will be a warning" + + - name: "[Setup] Create AWS source" + run: | + RES=$(curl -sS -X POST "$API/api/v1/sources" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg name "$SOURCE_PREFIX $RUN_ID" '{name: $name, platform: "aws"}')") + jq -e '.data.id' <<<"$RES" >/dev/null || { echo "::error::creating the source failed: $RES"; exit 1; } + SOURCE_TOKEN=$(jq -r '.data.attributes.token' <<<"$RES") + echo "::add-mask::$SOURCE_TOKEN" + { + echo "SOURCE_ID=$(jq -r '.data.id' <<<"$RES")" + echo "SOURCE_TOKEN=$SOURCE_TOKEN" + echo "DATA_REGION=$(jq -r '.data.attributes.data_region' <<<"$RES")" + echo "INGESTING_HOST=$(jq -r '.data.attributes.ingesting_host' <<<"$RES")" + echo "COLLECTION=t$(jq -r '.data.attributes.team_id' <<<"$RES")_$(jq -r '.data.attributes.table_name' <<<"$RES")" + echo "CONNECT_HOST=$(jq -r '.data.attributes.data_region' <<<"$RES")-connect.betterstackdata.com" + echo "ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)" + } >> "$GITHUB_ENV" + echo "source $(jq -r '.data.id' <<<"$RES") on $(jq -r '.data.attributes.data_region' <<<"$RES")" + + # A fresh token reaches the ingesters with a delay; until then they answer 401 and Firehose would retry its + # batches. Send warm-up lines until 20 in a row are accepted, then let it settle. + - name: "[Setup] Warm up the source until ingest accepts its token" + run: | + STREAK=0 + DEADLINE=$((SECONDS + 300)) + while [ "$STREAK" -lt 20 ]; do + CODE=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "https://$INGESTING_HOST/" -H "Authorization: Bearer $SOURCE_TOKEN" -H 'Content-Type: application/json' -d "{\"message\":\"e2e warm-up run_id=$RUN_ID\"}") + if [ "$CODE" = 202 ]; then STREAK=$((STREAK + 1)); else echo "ingest answered HTTP $CODE after $STREAK accepted"; STREAK=0; fi + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::ingest did not accept the source token 20 times in a row within 5 min"; exit 1; } + sleep 1 + done + sleep 15 + echo "ingest accepts the token" + + - name: "[Setup] Create the two log groups before the stack, so the first sync finds them" + run: | + for group in "$LOG_GROUP" "$STATIC_LOG_GROUP"; do + aws logs create-log-group --log-group-name "$group" + aws logs create-log-stream --log-group-name "$group" --log-stream-name run + done + + - name: "[Test] Deploy the CloudFormation stack" + run: | + aws cloudformation create-stack --stack-name "$STACK_NAME" --template-body file://template.yaml --capabilities CAPABILITY_NAMED_IAM \ + --parameters "ParameterKey=ClusterId,ParameterValue=$DATA_REGION" "ParameterKey=SourceToken,ParameterValue=$SOURCE_TOKEN" "ParameterKey=SourceId,ParameterValue=$SOURCE_ID" + if ! aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME"; then + echo "::error::stack creation failed: $(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --query 'Stacks[0].StackStatus' --output text)" + aws cloudformation describe-stack-events --stack-name "$STACK_NAME" --query "StackEvents[?ends_with(ResourceStatus, 'FAILED')].[LogicalResourceId,ResourceStatus,ResourceStatusReason]" --output table + exit 1 + fi + OUTPUTS=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --query 'Stacks[0].Outputs' --output json) + ROLE_ARN=$(jq -r '.[] | select(.OutputKey == "IntegrationRoleArn") | .OutputValue' <<<"$OUTPUTS") + EXTERNAL_ID=$(jq -r '.[] | select(.OutputKey == "ExternalId") | .OutputValue' <<<"$OUTPUTS") + [ -n "$ROLE_ARN" ] && [ -n "$EXTERNAL_ID" ] || { echo "::error::stack outputs lack IntegrationRoleArn or ExternalId: $OUTPUTS"; exit 1; } + # The external id is half of what Better Stack needs to assume the integration role; this repo is public. + echo "::add-mask::$EXTERNAL_ID" + echo "ROLE_ARN=$ROLE_ARN" >> "$GITHUB_ENV" + echo "EXTERNAL_ID=$EXTERNAL_ID" >> "$GITHUB_ENV" + echo "stack outputs: IntegrationRoleArn $ROLE_ARN, ExternalId (masked)" + + # X-Ray creates the aws/spans log group when it processes the first span, and the sync subscribes only groups + # that exist, so a throwaway trace goes in before the source is connected. + - name: "[Test] Warm-up trace so that aws/spans exists before the first sync" + run: | + NOW=$(date +%s) + TRACE_ID="1-$(printf '%x' "$NOW")-$(openssl rand -hex 12)" + SEGMENT=$(jq -nc --arg name "e2e-warm-up" --arg id "$(openssl rand -hex 8)" --arg trace "$TRACE_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \ + '{name: $name, id: $id, trace_id: $trace, start_time: $start, end_time: $end}') + aws xray put-trace-segments --trace-segment-documents "$SEGMENT" --query 'UnprocessedTraceSegments' --output json | jq -e 'length == 0' >/dev/null + + # aws_account_id would select an already connected account; a new connection is the role ARN + external id only. + - name: "[Test] Paste the stack outputs into the source (connects the AWS account, enqueues the sync)" + run: | + BODY=$(jq -nc --arg r "$ROLE_ARN" --arg e "$EXTERNAL_ID" '{aws_role_arn: $r, aws_external_id: $e}') + for attempt in 1 2 3; do + CODE=$(curl -sS -o connect.json -w '%{http_code}' -X PATCH "$API/api/v1/sources/$SOURCE_ID" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' -d "$BODY") + [ "$CODE" = 200 ] && { echo "source connected to $ROLE_ARN"; exit 0; } + # IAM propagation occasionally refuses the first AssumeRole on a fresh role; one bounded retry on that answer only. + if [ "$CODE" = 422 ] && grep -qi assume connect.json && [ "$attempt" -lt 3 ]; then echo "attempt $attempt refused: $(cat connect.json), retrying in 20 s"; sleep 20; continue; fi + echo "::error::connecting the source failed: HTTP $CODE $(cat connect.json)"; exit 1 + done + + - name: "[Test] The sync subscribes the new log group by itself" + run: | + DEADLINE=$((SECONDS + 480)) + until aws logs describe-subscription-filters --log-group-name "$LOG_GROUP" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text | grep -q firehose; do + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::the sync did not subscribe $LOG_GROUP within 8 min"; exit 1; } + echo "waiting for the sync to subscribe $LOG_GROUP"; sleep 15 + done + echo "$LOG_GROUP carries the BetterStackLogs subscription filter" + + # Frozen behaviour. A fresh account would get aws/spans and the CloudTrail group subscribed by default and the + # static group by auto-subscribe. This sandbox account is permanent in Better Stack (T-21948), and all three + # groups are deleted and recreated by every run, which the sync's reconcile treats as an explicit unsubscribe + # (T-21949). So the sync that just subscribed the new group must have left these three alone. When either + # issue is fixed this step fails and the test moves on to the new behaviour. + - name: "[Test] The recreated static group, aws/spans and the CloudTrail group stay unsubscribed (see T-21948, T-21949), then get subscribed through the API" + run: | + DEADLINE=$((SECONDS + 180)) + until aws logs describe-log-groups --log-group-name-prefix aws/spans --query "logGroups[?logGroupName=='aws/spans'].logGroupName" --output text | grep -q spans; do + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::X-Ray did not create aws/spans within 3 min of the warm-up trace"; exit 1; } + sleep 10 + done + for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + if aws logs describe-subscription-filters --log-group-name "$group" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text | grep -q firehose; then + echo "::error::$group was subscribed by the sync although it was deleted and recreated: the behaviour this test freezes has changed (see T-21948, T-21949), update the test" + exit 1 + fi + echo "$group stays unsubscribed, as expected" + done + for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + CODE=$(curl -sS -o subscription.json -w '%{http_code}' -X POST "$API/api/v2/sources/$SOURCE_ID/aws-log-group-subscriptions" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg r "$AWS_REGION" --arg n "$group" '{region: $r, name: $n, subscribed: true}')") + [ "$CODE" = 201 ] || { echo "::error::subscribing $group failed: HTTP $CODE $(cat subscription.json)"; exit 1; } + echo "subscription requested for $group" + done + + - name: "[Test] Wait until the sync applied the three subscriptions" + run: | + DEADLINE=$((SECONDS + 480)) + while :; do + PENDING="" + for group in "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + aws logs describe-subscription-filters --log-group-name "$group" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text 2>/dev/null | grep -q firehose || PENDING="$PENDING $group" + done + [ -z "$PENDING" ] && { echo "all three log groups carry the BetterStackLogs subscription filter"; exit 0; } + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::no BetterStackLogs subscription filter after 8 min on:$PENDING"; exit 1; } + echo "waiting for subscriptions on:$PENDING"; sleep 15 + done + + - name: "[Test] Produce logs, a custom metric, a two-segment trace and a CloudTrail management event" + run: | + echo "SINCE=dt > toDateTime('$(date -u -d '-10 minutes' '+%F %T')')" >> "$GITHUB_ENV" + for group in "$LOG_GROUP" "$STATIC_LOG_GROUP"; do + jq -nc --arg m "$MARKER" --arg run "$RUN_ID" --arg g "$group" '[range(1; 21) as $n | {timestamp: ((now * 1000 | floor) + $n), message: ({message: $m, e2e: {run_id: $run, log_group: $g, n: $n}} | tojson)}]' > events.json + aws logs put-log-events --log-group-name "$group" --log-stream-name run --log-events file://events.json >/dev/null + done + # Values 1, 2 and 3 in three consecutive minutes; the read-back checks each minute's value. + T=$(date +%s) + jq -nc --arg run "$RUN_ID" --argjson t "$T" \ + '[range(3) | {MetricName: "RunMarker", Value: (. + 1), Unit: "Count", Timestamp: ($t - (2 - .) * 60 | todate), Dimensions: [{Name: "RunId", Value: $run}]}]' > metrics.json + aws cloudwatch put-metric-data --namespace E2eTest/AwsStack --metric-data file://metrics.json + echo "METRIC_MINUTES=$(((T - 120) / 60 * 60)) $(((T - 60) / 60 * 60)) $((T / 60 * 60))" >> "$GITHUB_ENV" + NOW=$(date +%s) + TRACE_ID="1-$(printf '%x' "$NOW")-$(openssl rand -hex 12)" + PARENT_ID=$(openssl rand -hex 8) + PARENT=$(jq -nc --arg name "e2e-parent-$RUN_ID" --arg id "$PARENT_ID" --arg trace "$TRACE_ID" --arg run "$RUN_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \ + '{name: $name, id: $id, trace_id: $trace, start_time: $start, end_time: $end, annotations: {run_id: $run}}') + CHILD=$(jq -nc --arg name "e2e-child-$RUN_ID" --arg id "$(openssl rand -hex 8)" --arg parent "$PARENT_ID" --arg trace "$TRACE_ID" --arg run "$RUN_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \ + '{name: $name, id: $id, parent_id: $parent, trace_id: $trace, start_time: ($start + 0.2), end_time: ($end - 0.2), annotations: {run_id: $run}}') + aws xray put-trace-segments --trace-segment-documents "$PARENT" "$CHILD" --query 'UnprocessedTraceSegments' --output json | jq -e 'length == 0' >/dev/null + # The X-Ray trace id 1-<8 hex epoch>-<24 hex> is stored as the 32-hex OpenTelemetry id. + echo "TRACE_HEX=$(cut -d- -f2,3 <<<"$TRACE_ID" | tr -d -)" >> "$GITHUB_ENV" + # A management event that happens after the trail exists and names this run. + aws logs put-retention-policy --log-group-name "$LOG_GROUP" --retention-in-days 1 + echo "20 log events into each of $LOG_GROUP and $STATIC_LOG_GROUP, 3 datapoints of E2eTest/AwsStack RunMarker, trace $TRACE_ID, PutRetentionPolicy on $LOG_GROUP" + + - name: "[Test] Read everything back through the SQL API connection" + run: | + set -o pipefail + sql() { + CODE=$(curl -sS -o response.json -w '%{http_code}' -u "$TEST_TEAM_CONNECTION_USERNAME:$TEST_TEAM_CONNECTION_PASSWORD" -H 'Content-Type: plain/text' --data-binary "$1 FORMAT JSONEachRow" "https://$CONNECT_HOST/") + # A source's logs, spans and metrics collections each exist only from their first stored row on; + # until then the query is "no rows yet". + [ "$CODE" = 500 ] && grep -q CLUSTER_DOESNT_EXIST response.json && return 0 + [ "$CODE" = 200 ] || { echo "::error::SQL API on $CONNECT_HOST answered HTTP $CODE: $(head -c 300 response.json)" >&2; return 1; } + cat response.json + } + count() { ROWS=$(sql "SELECT count() AS n FROM $1") || return 1; [ -n "$ROWS" ] && jq -r '.n' <<<"$ROWS" || echo 0; } + METRIC=e2e_test.aws_stack.run_marker + METRIC_FILTER="tags['run_id'] = '$RUN_ID' AND startsWith(name, '$METRIC')" + counts() { + LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$LOG_GROUP\"') > 0") + STATIC_LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$STATIC_LOG_GROUP\"') > 0") + # The mapper stores a CloudTrail event as {id, message: , context: {logGroup, ...}, dt}; this run's + # PutRetentionPolicy call names the run's log group. + CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'context', 'logGroup') = '/aws/cloudtrail/better-stack' AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'") + # The mapper stores each X-Ray span as {dt, resource, span: {trace_id, span_id, name, ...}}. + SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID')") + # The metric stream sends one aggregate per minute; the mapper stores it as the metric plus _max, _min, _sum and + # _count, so three minutes make 15 points. + METRICS=$(count "(SELECT name, dt FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND $METRIC_FILTER GROUP BY name, dt)") + echo "logs $LOGS/20 + static $STATIC_LOGS/20, spans $SPANS/2, metric points $METRICS/15, CloudTrail event $CLOUDTRAIL/1" + } + # What CloudWatch Logs did with each group in the last 40 min; DeliveryErrors mean the filter paused for 10 min + # and skipped what arrived meanwhile. + forwarding() { + for group in "$LOG_GROUP" "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do + for metric in IncomingLogEvents ForwardedLogEvents DeliveryErrors; do + DIMS="Name=LogGroupName,Value=$group" + [ "$metric" = IncomingLogEvents ] || DIMS="$DIMS Name=DestinationType,Value=Firehose Name=FilterName,Value=BetterStackLogs" + echo "$group $metric $(aws cloudwatch get-metric-statistics --namespace AWS/Logs --metric-name "$metric" --dimensions $DIMS \ + --start-time "$(date -u -d '-40 minutes' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 2400 --statistics Sum --query 'sum(Datapoints[].Sum)' --output text)" + done + done + } + # Within an hour of the previous stack deletion the static group, aws/spans and CloudTrail may deliver nothing + # at all, so only the run's own log group and the metric are waited for (see the step measuring that hour). + # Otherwise everything is: CloudTrail hands events to CloudWatch Logs within about 5 min of the call. + if [ -n "${RECENT_STACK_MIN:-}" ]; then + arrived() { [ "$LOGS" -ge 20 ] && [ "$METRICS" -ge 15 ]; } + else + arrived() { [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 15 ] && [ "$CLOUDTRAIL" -ge 1 ]; } + fi + DEADLINE=$((SECONDS + 1200)) + while :; do + counts + arrived && break + [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::not everything arrived within 20 min"; forwarding; mkdir -p out; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; exit 1; } + sleep 20 + done + # A last look 60 s later catches duplicate delivery. Every count must be exact; within that hour a fixed-name + # source that delivered nothing is a warning instead. + sleep 60 + counts + MISSING=() + for check in "logs:$LOGS:20" "static:$STATIC_LOGS:20" "spans:$SPANS:2" "cloudtrail:$CLOUDTRAIL:1"; do + IFS=: read -r name got want <<<"$check" + [ "$got" = "$want" ] && continue + if [ -n "${RECENT_STACK_MIN:-}" ] && [ "$name" != logs ] && [ "$got" = 0 ]; then MISSING+=("$name"); continue; fi + echo "::error::unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)" + exit 1 + done + # Each minute must hold exactly the value sent in it: 1, 2 and 3 for the metric, _max, _min and _sum, 1 for _count. + # The lowest and highest stored value per point catch a re-sent minute with a different value. Points are + # compared by minute, since AWS does not document whether the stream stamps a minute's start. + EXPECTED=$(jq -nc --arg n "$METRIC" --arg m "$METRIC_MINUTES" '[$m | split(" ") | to_entries[] | (.value | tonumber) as $t | (.key + 1) as $v + | ({name: $n, v: $v}, {name: "\($n)_max", v: $v}, {name: "\($n)_min", v: $v}, {name: "\($n)_sum", v: $v}, {name: "\($n)_count", v: 1}) + | {name, minute: $t, min: .v, max: .v}] | sort_by(.name, .minute)') + ACTUAL=$(sql "SELECT name, toUnixTimestamp(dt) AS t, minMerge(value_min) AS min, maxMerge(value_max) AS max FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND $METRIC_FILTER GROUP BY name, dt" \ + | jq -sc 'map({name, minute: (((.t | tonumber) / 60 | floor) * 60), min: (.min | tonumber), max: (.max | tonumber)}) | sort_by(.name, .minute)') + [ "$ACTUAL" = "$EXPECTED" ] || { echo "::error::metric points differ from the datapoints sent: expected $EXPECTED, got $ACTUAL"; exit 1; } + if [ "${#MISSING[@]}" -gt 0 ]; then + NAMES=$(printf '%s\n' "${MISSING[@]}" | sed -e "s#^static\$#the logs of $STATIC_LOG_GROUP#" -e 's#^spans$#the spans#' -e 's#^cloudtrail$#the CloudTrail event#' | paste -sd, - | sed 's/,/, /g') + WARNING="Not verified: $NAMES. The previous run deleted its stack $RECENT_STACK_MIN min before this one, and for up to an hour AWS keeps using sessions of the IAM roles every run recreates under the same names, so CloudWatch Logs and CloudTrail can skip this data. A run more than an hour after the previous one checks it." + echo "::warning::$WARNING" + forwarding + fi + mkdir -p out + sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json + if [ "$SPANS" = 2 ]; then + sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json + # The X-Ray trace id 1-- should be stored as the 32-hex id the mapper takes from Transaction Search. + STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ') + [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || { echo "::error::stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"; exit 1; } + fi + sql "SELECT name, dt, tags, avgMerge(value_avg) AS value FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND tags['run_id'] = '$RUN_ID' AND name = '$METRIC' GROUP BY name, dt, tags ORDER BY dt" > out/sample_metric.json + [ "$CLOUDTRAIL" = 0 ] || sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'" > out/sample_cloudtrail.json + { + echo "### AWS CloudFormation stack E2E \`$RUN_ID\`" + echo + echo "- template from $TEMPLATE_FROM, source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`" + echo "- \`$LOG_GROUP\` auto-subscribed; \`$STATIC_LOG_GROUP\`, \`aws/spans\` and \`/aws/cloudtrail/better-stack\` stayed unsubscribed after their recreation (T-21948, T-21949) and were subscribed through the API" + echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric points $METRICS with the values sent, CloudTrail event $CLOUDTRAIL" + echo + if [ "${#MISSING[@]}" -gt 0 ]; then + echo "⚠️ $WARNING" + echo + echo "✅ everything else round-tripped through the stack" + else + echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack" + fi + } > out/summary.md + + - name: "[Cleanup] Delete this run's resources in Better Stack and AWS" + if: ${{ always() }} + run: bash e2e/cleanup.sh + + - name: "[Report] Job summary" + if: ${{ always() }} + run: | + if [ -f out/summary.md ]; then cat out/summary.md >> "$GITHUB_STEP_SUMMARY"; else echo "No summary, the run failed before verification." >> "$GITHUB_STEP_SUMMARY"; fi + # Rows come back as JSONEachRow with the stored event as a JSON string in raw; render them as a pretty array. + for f in out/sample_*.json; do [ -f "$f" ] && { echo; echo "
$(basename "$f")"; echo; echo '```json'; jq -s 'map(if has("raw") then .raw |= (fromjson? // .) else . end)' "$f"; echo '```'; echo '
'; echo; } >> "$GITHUB_STEP_SUMMARY"; done + [ -f out/cleanup.md ] && { echo; cat out/cleanup.md; } >> "$GITHUB_STEP_SUMMARY" + true + + notify-linear: + if: ${{ failure() && (github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }} + needs: [test] + uses: ./.github/workflows/notify_linear_failure.yml + secrets: inherit diff --git a/.github/workflows/notify_linear_failure.yml b/.github/workflows/notify_linear_failure.yml new file mode 100644 index 0000000..529be28 --- /dev/null +++ b/.github/workflows/notify_linear_failure.yml @@ -0,0 +1,105 @@ +name: "Notify Linear on failure" + +on: + workflow_call: + +jobs: + create-linear-issue: + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: "Gather failure context" + id: ctx + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id || github.run_id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt || github.run_attempt }} + run: | + FAILED=$(gh api \ + "repos/${{ github.repository }}/actions/runs/$RUN_ID/attempts/$RUN_ATTEMPT/jobs" \ + --paginate \ + --jq '[.jobs[] + | select(.conclusion == "failure") + | "- **\(.name)** — failed at _\([.steps[] | select(.conclusion == "failure") | .name] | join(", "))_"] + | join("\n")') + + if [ -z "$FAILED" ]; then + FAILED="_(no failed jobs reported by the API yet)_" + fi + + { + echo "failed_jobs<> "$GITHUB_OUTPUT" + - name: "Create Linear issue" + env: + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} + LINEAR_TEAM_ID: ${{ secrets.LINEAR_TEAM_TELEMETRY_ID }} + LINEAR_STATUS_PLAN_ID: ${{ secrets.LINEAR_STATUS_PLAN_ID }} + LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID: ${{ secrets.LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID }} + LINEAR_ASSIGNEE_PETR_ID: ${{ secrets.LINEAR_ASSIGNEE_PETR_ID }} + WORKFLOW: ${{ github.event.workflow_run.name || github.workflow }} + REF_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }} + RUN_URL: ${{ github.event.workflow_run.html_url || format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt || github.run_attempt }} + EVENT_NAME: ${{ github.event.workflow_run.event || github.event_name }} + SHA: ${{ github.event.workflow_run.head_sha || github.sha }} + COMMIT_MESSAGE: ${{ github.event.workflow_run.head_commit.message || github.event.head_commit.message }} + FAILED_JOBS: ${{ steps.ctx.outputs.failed_jobs }} + run: | + SHORT_SHA="${SHA:0:7}" + + # head_commit is only populated on push events; skip the line otherwise + COMMIT_LINE="**Commit:** \`$SHORT_SHA\`" + if [ -n "$COMMIT_MESSAGE" ]; then + COMMIT_SUBJECT="$(printf '%s' "$COMMIT_MESSAGE" | head -n1)" + COMMIT_LINE="**Commit:** \`$SHORT_SHA\` — $COMMIT_SUBJECT" + fi + + ATTEMPT_NOTE="" + if [ "$RUN_ATTEMPT" != "1" ]; then + ATTEMPT_NOTE=" (attempt #$RUN_ATTEMPT)" + fi + + TITLE="E2E failure: $WORKFLOW" + DESCRIPTION=$(cat <a; e2e-bootstrap is the standing OIDC stack and must stay), the buckets the +# stack retains by design, aws/spans and /aws/application-signals/data (X-Ray Transaction Search creates them outside +# the stack) and the /e2e/* log groups. The workflow runs this twice: before a run, so leftovers of a killed cleanup +# never block the next run, and after it. One run at a time per account (the workflow's concurrency group), so nothing +# with these names belongs to a live run. +# Writes a markdown report to out/cleanup.md and exits 1 when something could not be listed or deleted. +# Env: API, TEST_TEAM_TELEMETRY_API, SOURCE_PREFIX, AWS_REGION (+ AWS credentials for the account). +set -euo pipefail + +mkdir -p out +: > out/cleanup.md +say() { echo "$1"; echo "$1" >> out/cleanup.md; } +could_not_list() { say "- ❌ could not list $1, so none were deleted"; touch cleanup.failed; } +rm -f cleanup.failed +say "### Cleanup" +say "" + +# Sources first: while the integration role still exists Better Stack can unsubscribe cleanly. +SOURCES=$(curl -sS --fail-with-body "$API/api/v1/sources?name=$SOURCE_PREFIX&per_page=250" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" \ + | jq -r --arg prefix "$SOURCE_PREFIX " '.data[] | select(.attributes.name | startswith($prefix)) | "\(.id) \(.attributes.name)"') || { could_not_list sources; SOURCES=""; } +while read -r id name; do + [ -n "$id" ] || continue + CODE=$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE "$API/api/v1/sources/$id" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API") + [ "$CODE" = 204 ] && say "- deleted source $id \`$name\`" || { say "- ❌ could not delete source $id \`$name\` (HTTP $CODE)"; touch cleanup.failed; } +done <<<"$SOURCES" + +STACKS=$(aws cloudformation describe-stacks --query "Stacks[?starts_with(StackName, 'e2e-gh')].StackName" --output text) || { could_not_list stacks; STACKS=""; } +for stack in $STACKS; do + aws cloudformation delete-stack --stack-name "$stack" + if aws cloudformation wait stack-delete-complete --stack-name "$stack"; then say "- deleted stack \`$stack\`"; else + say "- ❌ could not delete stack \`$stack\`: $(aws cloudformation describe-stack-events --stack-name "$stack" --query "StackEvents[?ends_with(ResourceStatus, 'FAILED')].[LogicalResourceId,ResourceStatusReason]" --output text | head -5)"; touch cleanup.failed + fi +done + +ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) || { could_not_list buckets; ACCOUNT_ID=""; } +for bucket in "better-stack-firehose-$ACCOUNT_ID-$AWS_REGION" "better-stack-cloudtrail-$ACCOUNT_ID-$AWS_REGION"; do + [ -n "$ACCOUNT_ID" ] && aws s3api head-bucket --bucket "$bucket" >/dev/null 2>&1 || continue + if aws s3 rb "s3://$bucket" --force >/dev/null; then say "- deleted bucket \`$bucket\`"; else say "- ❌ could not delete bucket \`$bucket\`"; touch cleanup.failed; fi +done + +LOG_GROUPS=$(aws logs describe-log-groups --query "logGroups[?logGroupName == 'aws/spans' || logGroupName == '/aws/application-signals/data' || starts_with(logGroupName, '/e2e/')].logGroupName" --output text) || { could_not_list "log groups"; LOG_GROUPS=""; } +for group in $LOG_GROUPS; do + if aws logs delete-log-group --log-group-name "$group"; then say "- deleted log group \`$group\`"; else say "- ❌ could not delete log group \`$group\`"; touch cleanup.failed; fi +done + +[ "$(wc -l < out/cleanup.md)" -gt 2 ] || say "- nothing to delete" +[ ! -f cleanup.failed ] || { say ""; say "❌ cleanup left resources behind"; exit 1; }