From ae1dbe680005dd0dce865fb14abd1be48ec3c132 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Tue, 18 Aug 2026 19:26:00 +0200 Subject: [PATCH 01/16] feat(postgres): add temporal service database role secrets --- src/development/postgres/compose.yaml | 11 +++++++++++ src/development/sqitch/compose.yaml | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/development/postgres/compose.yaml b/src/development/postgres/compose.yaml index 546cb761..b5c9f8a1 100644 --- a/src/development/postgres/compose.yaml +++ b/src/development/postgres/compose.yaml @@ -23,6 +23,12 @@ secrets: postgres-role-service-reccoom-username: # The username of the recommendation engine's database role. file: ~~/artifacts/secrets/postgres-role-service-reccoom-username.secret + postgres-role-service-temporal-password: + # The password of the workflow orchestrator's own database role (schema ownership, not app data). + file: ~~/artifacts/secrets/postgres-role-service-temporal-password.secret + postgres-role-service-temporal-username: + # The username of the workflow orchestrator's own database role (schema ownership, not app data). + file: ~~/artifacts/secrets/postgres-role-service-temporal-username.secret postgres-role-service-vibetype-password: # The `vibetype` database role's password. file: ~~/artifacts/secrets/postgres-role-service-vibetype-password.secret @@ -100,6 +106,11 @@ x-dargstack: type: random_string postgres-role-service-reccoom-username: type: wordlist_word + postgres-role-service-temporal-password: + special_characters: false + type: random_string + postgres-role-service-temporal-username: + type: wordlist_word postgres-role-service-vibetype-password: special_characters: false type: random_string diff --git a/src/development/sqitch/compose.yaml b/src/development/sqitch/compose.yaml index 12499b7b..ab36f09d 100644 --- a/src/development/sqitch/compose.yaml +++ b/src/development/sqitch/compose.yaml @@ -20,6 +20,8 @@ services: - postgres-role-service-postgraphile-username - postgres-role-service-reccoom-password - postgres-role-service-reccoom-username + - postgres-role-service-temporal-password + - postgres-role-service-temporal-username - postgres-role-service-vibetype-password - postgres-role-service-vibetype-username - postgres-role-service-zammad-username From a8a38dc45b2d4245d064cbdc4505a75d6a5a7abf Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Tue, 18 Aug 2026 19:58:00 +0200 Subject: [PATCH 02/16] feat(temporal): add self-hosted temporal server and ui services --- src/development/temporal/compose.yaml | 44 +++++++++++++++++++ .../temporal/configurations/entrypoint.sh | 10 +++++ src/production/temporal/compose.yaml | 13 ++++++ 3 files changed, 67 insertions(+) create mode 100644 src/development/temporal/compose.yaml create mode 100644 src/development/temporal/configurations/entrypoint.sh create mode 100644 src/production/temporal/compose.yaml diff --git a/src/development/temporal/compose.yaml b/src/development/temporal/compose.yaml new file mode 100644 index 00000000..ddd10b6e --- /dev/null +++ b/src/development/temporal/compose.yaml @@ -0,0 +1,44 @@ +services: + temporal: + # You cannot access the workflow orchestrator directly, use `temporal-ui`. + entrypoint: sh + command: ["/entrypoint.sh"] + environment: + DB: postgres12 + DB_PORT: "5432" + POSTGRES_SEEDS: postgres + PROMETHEUS_ENDPOINT: 0.0.0.0:8000 + healthcheck: + test: ["CMD-SHELL", "nc -z 127.0.0.1 7233 || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + image: temporalio/auto-setup:1.29.1 + secrets: + - postgres-role-service-temporal-password + - postgres-role-service-temporal-username + volumes: + - ./configurations/entrypoint.sh:/entrypoint.sh:ro + temporal-ui: + # You can access the workflow orchestrator's ui at [temporal.app.localhost](https://temporal.app.localhost/). + deploy: + labels: + - traefik.enable=true + - traefik.http.routers.temporal.entryPoints=web + - traefik.http.routers.temporal.middlewares=redirectscheme # dargstack:dev-only + - traefik.http.routers.temporal.rule=Host(`temporal.${STACK_DOMAIN}`) + - traefik.http.routers.temporal-secure.entryPoints=web-secure + - traefik.http.routers.temporal-secure.rule=Host(`temporal.${STACK_DOMAIN}`) + - traefik.http.routers.temporal-secure.tls.options=mintls13@file # dargstack:dev-only + - traefik.http.services.temporal.loadbalancer.server.port=8080 + environment: + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_CORS_ORIGINS: https://temporal.${STACK_DOMAIN} + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8080/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + image: temporalio/ui:2.40.1 diff --git a/src/development/temporal/configurations/entrypoint.sh b/src/development/temporal/configurations/entrypoint.sh new file mode 100644 index 00000000..83c60719 --- /dev/null +++ b/src/development/temporal/configurations/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +# The auto-setup image has no support for Docker's `_FILE`-suffixed secret +# convention, so read the mounted secret files ourselves and hand the plain +# values to the image's own entrypoint. +export POSTGRES_USER="$(cat /run/secrets/postgres-role-service-temporal-username)" +export POSTGRES_PWD="$(cat /run/secrets/postgres-role-service-temporal-password)" + +exec /etc/temporal/entrypoint.sh autosetup diff --git a/src/production/temporal/compose.yaml b/src/production/temporal/compose.yaml new file mode 100644 index 00000000..9f643d85 --- /dev/null +++ b/src/production/temporal/compose.yaml @@ -0,0 +1,13 @@ +services: + temporal: + deploy: + update_config: + order: stop-first + temporal-ui: + deploy: + labels: + - (( append )) + - traefik.http.routers.temporal.middlewares=redirectscheme + - traefik.http.routers.temporal-secure.tls.certresolver=default + update_config: + order: start-first From f448fa05a6975e0421623436de03571ef2f50cca Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Tue, 18 Aug 2026 20:41:00 +0200 Subject: [PATCH 03/16] feat(prometheus,grafana): scrape temporal metrics and alert on workflow failures --- .../alert-temporal-workflow-failures.yaml | 84 +++++++++++++++++++ .../prometheus/configurations/prometheus.yaml | 4 + 2 files changed, 88 insertions(+) create mode 100644 src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml diff --git a/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml b/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml new file mode 100644 index 00000000..d9c2a19f --- /dev/null +++ b/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml @@ -0,0 +1,84 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: Temporal + folder: Infrastructure + interval: 1m + repeat_interval: 604800 # 7 days + rules: + - uid: een6tworkflowfail + title: Workflow failures + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: sum(increase(workflow_failed[10m])) + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + range: false + refId: A + - refId: B + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: + summary: Temporal reported one or more failed workflow executions in the last 10 minutes. + labels: {} + isPaused: false + notification_settings: + receiver: 'Discord' diff --git a/src/development/prometheus/configurations/prometheus.yaml b/src/development/prometheus/configurations/prometheus.yaml index 5c499ac4..16f19f75 100644 --- a/src/development/prometheus/configurations/prometheus.yaml +++ b/src/development/prometheus/configurations/prometheus.yaml @@ -33,3 +33,7 @@ scrape_configs: - targets: - redpanda:9644 metrics_path: /public_metrics + + - job_name: "temporal" + static_configs: + - targets: ["temporal:8000"] From 0ecf3ec9ad2f6834ffe051817428cf62946b9ab3 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Tue, 18 Aug 2026 21:19:00 +0200 Subject: [PATCH 04/16] feat(temporal-worker): add worker service scaffolding and regenerate docs --- artifacts/docs/README.md | 12 ++++++++ src/development/temporal-worker/compose.yaml | 29 ++++++++++++++++++++ src/production/temporal-worker/compose.yaml | 9 ++++++ 3 files changed, 50 insertions(+) create mode 100644 src/development/temporal-worker/compose.yaml create mode 100644 src/production/temporal-worker/compose.yaml diff --git a/artifacts/docs/README.md b/artifacts/docs/README.md index 4dd16bc8..9bfb7cb5 100644 --- a/artifacts/docs/README.md +++ b/artifacts/docs/README.md @@ -151,6 +151,18 @@ You can access the event streaming platform's ui at [redpanda.app.localhost](htt You cannot access the database migrations directly. +### temporal + +You cannot access the workflow orchestrator directly, use `temporal-ui`. + +### temporal-ui + +You can access the workflow orchestrator's ui at [temporal.app.localhost](https://temporal.app.localhost/). + +### temporal-worker + +You cannot access the workflow worker directly, use `temporal-ui`. + ### traefik You can access the reverse proxy's dashboard at [traefik.app.localhost](https://traefik.app.localhost/). diff --git a/src/development/temporal-worker/compose.yaml b/src/development/temporal-worker/compose.yaml new file mode 100644 index 00000000..28f06e1b --- /dev/null +++ b/src/development/temporal-worker/compose.yaml @@ -0,0 +1,29 @@ +services: + temporal-worker: + # You cannot access the workflow worker directly, use `temporal-ui`. + deploy: + labels: + - dargstack.development.git.ssh=git@github.com:maevsi/temporal-worker.git + - dargstack.development.git.https=https://github.com/maevsi/temporal-worker.git + environment: + AWS_SHARED_CREDENTIALS_FILE: /run/secrets/jobber-aws-credentials + POSTGRES_DB_FILE: /run/secrets/postgres-db + POSTGRES_HOST: postgres + TEMPORAL_ADDRESS: temporal:7233 + image: vibetype/temporal-worker:development + secrets: + - jobber-aws-bucket + - jobber-aws-credentials + - source: jobber-aws-configuration + target: /home/node/.aws/config + - postgres-db + - postgres-role-service-jobber-password + - postgres-role-service-jobber-username + volumes: + - ~~~/temporal-worker/:/srv/app/ # dargstack:dev-only + - pnpm-data:/srv/.pnpm-store/ # dargstack:dev-only + - temporal-worker-data:/srv/app/node_modules # dargstack:dev-only +volumes: + temporal-worker-data: + # The workflow worker's data. + {} diff --git a/src/production/temporal-worker/compose.yaml b/src/production/temporal-worker/compose.yaml new file mode 100644 index 00000000..54c0fe63 --- /dev/null +++ b/src/production/temporal-worker/compose.yaml @@ -0,0 +1,9 @@ +services: + temporal-worker: + deploy: + update_config: + order: start-first + environment: + SENTRY_CRONS: ${SENTRY_CRONS} + SENTRY_CRONS_OUTBOX_PURGE: ${SENTRY_CRONS_OUTBOX_PURGE} + image: ghcr.io/maevsi/temporal-worker:0.1.0 From 8ec4531c51bc3fd79f878406da2bde71e248fd60 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Tue, 18 Aug 2026 21:57:00 +0200 Subject: [PATCH 05/16] refactor(jobber): remove jobber, migrated to temporal-worker --- artifacts/docs/README.md | 4 - src/development/jobber/compose.yaml | 106 ------------------ src/development/jobber/configurations/.jobber | 12 -- .../configurations/sinks/sentry/error.sh | 3 - .../sinks/sentry/in-progress.sh | 3 - .../jobber/configurations/sinks/sentry/ok.sh | 3 - src/development/postgres/compose.yaml | 10 ++ src/development/temporal-worker/compose.yaml | 61 +++++++++- src/production/jobber/compose.yaml | 10 -- src/production/jobber/configurations/.jobber | 20 ---- 10 files changed, 66 insertions(+), 166 deletions(-) delete mode 100644 src/development/jobber/compose.yaml delete mode 100644 src/development/jobber/configurations/.jobber delete mode 100755 src/development/jobber/configurations/sinks/sentry/error.sh delete mode 100755 src/development/jobber/configurations/sinks/sentry/in-progress.sh delete mode 100755 src/development/jobber/configurations/sinks/sentry/ok.sh delete mode 100644 src/production/jobber/compose.yaml delete mode 100644 src/production/jobber/configurations/.jobber diff --git a/artifacts/docs/README.md b/artifacts/docs/README.md index 9bfb7cb5..c2ff2d15 100644 --- a/artifacts/docs/README.md +++ b/artifacts/docs/README.md @@ -78,10 +78,6 @@ You cannot access the ip geolocator via a web interface. You can access the observation dashboard at [grafana.app.localhost](https://grafana.app.localhost/). See `docs/grafana.md` for how to configure datasources, alerting, and dashboards after first login. -### jobber - -You cannot access the jobber via a web interface. - ### memcached You cannot access the caching system via a web interface. diff --git a/src/development/jobber/compose.yaml b/src/development/jobber/compose.yaml deleted file mode 100644 index 8b0ff062..00000000 --- a/src/development/jobber/compose.yaml +++ /dev/null @@ -1,106 +0,0 @@ -secrets: - jobber-aliases: - # The job scheduler's SMTP client mail alias. - file: ~~/artifacts/secrets/jobber-aliases.secret - jobber-aws-bucket: - # The job scheduler's AWS bucket name. - file: ~~/artifacts/secrets/jobber-aws-bucket.secret - jobber-aws-credentials: - # The job scheduler's AWS credentials. - file: ~~/artifacts/secrets/jobber-aws-credentials.secret - jobber-aws-configuration: - # The job scheduler's AWS configuration. - file: ~~/artifacts/secrets/jobber-aws-configuration.secret - jobber-msmtprc: - # The job scheduler's SMTP client configuration. - file: ~~/artifacts/secrets/jobber-msmtprc.secret -services: - jobber: - # You cannot access the jobber via a web interface. - environment: - AWS_SHARED_CREDENTIALS_FILE: /run/secrets/jobber-aws-credentials - image: ghcr.io/dargmuesli/jobber-aws-msmtp:1.3.0 - secrets: - - source: jobber-aliases - target: /etc/aliases - - jobber-aws-bucket - - jobber-aws-credentials - - source: jobber-aws-configuration - target: /home/jobberuser/.aws/config - - source: jobber-msmtprc - target: /etc/msmtprc - volumes: - - ./configurations/.jobber:/home/jobberuser/.jobber:ro - - ./configurations/sinks:/srv/sinks:ro -x-dargstack: - secrets: - jobber-aliases: - type: insecure_default - insecure_default: "default: mail@localhost" - jobber-aws-bucket: - type: wordlist_word - jobber-aws-credentials: - type: insecure_default - insecure_default: | - [default] - aws_access_key_id = s3user - aws_secret_access_key = s3password - jobber-aws-configuration: - type: template - template: | - [default] - region = {{secret:jobber-aws-configuration-region}} - - s3 = - endpoint_url = {{secret:jobber-aws-configuration-s3-endpoint}} - signature_version = s3v4 - max_concurrent_requests = 100 - max_queue_size = 1000 - multipart_threshold = 50MB - # Edit the multipart_chunksize value according to the file sizes that you want to upload. The present configuration allows to upload files up to 10 GB (100 requests * 10MB). For example setting it to 5GB allows you to upload files up to 5TB. - multipart_chunksize = 10MB - s3api = - endpoint_url = {{secret:jobber-aws-configuration-s3-endpoint}} - - [plugins] - endpoint = awscli_plugin_endpoint - jobber-aws-configuration-region: - type: third_party - jobber-aws-configuration-s3-endpoint: - type: third_party - jobber-msmtprc: - type: template - template: | - # Set default values for all following accounts. - defaults - auth on - tls on - tls_trust_file /etc/ssl/certs/ca-certificates.crt - syslog on - - # - account {{secret:jobber-msmtprc-account}} - from {{secret:jobber-msmtprc-from}} - host {{secret:jobber-msmtprc-host}} - password {{secret:jobber-msmtprc-password}} - port {{secret:jobber-msmtprc-port}} - user {{secret:jobber-msmtprc-user}} - # or - #auth off - #tls off - - # Set a default account - account default : - aliases /etc/aliases - jobber-msmtprc-account: - type: third_party - jobber-msmtprc-from: - type: third_party - jobber-msmtprc-host: - type: third_party - jobber-msmtprc-password: - type: third_party - jobber-msmtprc-port: - type: third_party - jobber-msmtprc-user: - type: third_party diff --git a/src/development/jobber/configurations/.jobber b/src/development/jobber/configurations/.jobber deleted file mode 100644 index 906ff841..00000000 --- a/src/development/jobber/configurations/.jobber +++ /dev/null @@ -1,12 +0,0 @@ -version: 1.4 - -jobs: - DBBackup: - cmd: aws s3 sync /backups s3://$(cat /run/secrets/jobber-aws-bucket)/backups - time: 0 0 0 * - notifyOnSuccess: - - type: system-email - notifyOnError: - - type: system-email - notifyOnFailure: - - type: system-email diff --git a/src/development/jobber/configurations/sinks/sentry/error.sh b/src/development/jobber/configurations/sinks/sentry/error.sh deleted file mode 100755 index 242ad6aa..00000000 --- a/src/development/jobber/configurations/sinks/sentry/error.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -curl "${SENTRY_CRONS}?status=error" diff --git a/src/development/jobber/configurations/sinks/sentry/in-progress.sh b/src/development/jobber/configurations/sinks/sentry/in-progress.sh deleted file mode 100755 index aaa43302..00000000 --- a/src/development/jobber/configurations/sinks/sentry/in-progress.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -curl "${SENTRY_CRONS}?status=in_progress" diff --git a/src/development/jobber/configurations/sinks/sentry/ok.sh b/src/development/jobber/configurations/sinks/sentry/ok.sh deleted file mode 100755 index 0b18e7c7..00000000 --- a/src/development/jobber/configurations/sinks/sentry/ok.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -curl "${SENTRY_CRONS}?status=ok" diff --git a/src/development/postgres/compose.yaml b/src/development/postgres/compose.yaml index b5c9f8a1..a8e480e9 100644 --- a/src/development/postgres/compose.yaml +++ b/src/development/postgres/compose.yaml @@ -11,6 +11,16 @@ secrets: postgres-role-service-grafana-username: # The username of the observation dashboard's database role. file: ~~/artifacts/secrets/postgres-role-service-grafana-username.secret + postgres-role-service-jobber-password: + # The password of the workflow worker's outbox-purge database role. Kept + # under the `jobber` name jobber (the role's original owner) used, + # renaming would make dargstack generate a new random value and orphan + # the real grant already made on this role in the sqitch repo. + file: ~~/artifacts/secrets/postgres-role-service-jobber-password.secret + postgres-role-service-jobber-username: + # The username of the workflow worker's outbox-purge database role. Kept + # under the `jobber` name for the same reason as the password above. + file: ~~/artifacts/secrets/postgres-role-service-jobber-username.secret postgres-role-service-postgraphile-password: # The password of the GraphQL API database wrapper's database role. file: ~~/artifacts/secrets/postgres-role-service-postgraphile-password.secret diff --git a/src/development/temporal-worker/compose.yaml b/src/development/temporal-worker/compose.yaml index 28f06e1b..35572a4b 100644 --- a/src/development/temporal-worker/compose.yaml +++ b/src/development/temporal-worker/compose.yaml @@ -1,3 +1,18 @@ +secrets: + jobber-aws-bucket: + # The workflow worker's AWS bucket name for DBBackup. Kept under the + # `jobber` name jobber originally used, renaming would make dargstack + # generate a new random value and orphan the real S3 bucket already in + # use in production. + file: ~~/artifacts/secrets/jobber-aws-bucket.secret + jobber-aws-credentials: + # The workflow worker's AWS credentials for DBBackup. Kept under the + # `jobber` name for the same reason as `jobber-aws-bucket`. + file: ~~/artifacts/secrets/jobber-aws-credentials.secret + jobber-aws-configuration: + # The workflow worker's AWS configuration for DBBackup. Kept under the + # `jobber` name for the same reason as `jobber-aws-bucket`. + file: ~~/artifacts/secrets/jobber-aws-configuration.secret services: temporal-worker: # You cannot access the workflow worker directly, use `temporal-ui`. @@ -7,18 +22,21 @@ services: - dargstack.development.git.https=https://github.com/maevsi/temporal-worker.git environment: AWS_SHARED_CREDENTIALS_FILE: /run/secrets/jobber-aws-credentials - POSTGRES_DB_FILE: /run/secrets/postgres-db POSTGRES_HOST: postgres TEMPORAL_ADDRESS: temporal:7233 image: vibetype/temporal-worker:development secrets: - - jobber-aws-bucket + - source: jobber-aws-bucket + target: /run/environment-variables/AWS_BUCKET - jobber-aws-credentials - source: jobber-aws-configuration target: /home/node/.aws/config - - postgres-db - - postgres-role-service-jobber-password - - postgres-role-service-jobber-username + - source: postgres-db + target: /run/environment-variables/POSTGRES_DATABASE + - source: postgres-role-service-jobber-password + target: /run/environment-variables/POSTGRES_PASSWORD + - source: postgres-role-service-jobber-username + target: /run/environment-variables/POSTGRES_USER volumes: - ~~~/temporal-worker/:/srv/app/ # dargstack:dev-only - pnpm-data:/srv/.pnpm-store/ # dargstack:dev-only @@ -27,3 +45,36 @@ volumes: temporal-worker-data: # The workflow worker's data. {} +x-dargstack: + secrets: + jobber-aws-bucket: + type: wordlist_word + jobber-aws-credentials: + type: insecure_default + insecure_default: | + [default] + aws_access_key_id = s3user + aws_secret_access_key = s3password + jobber-aws-configuration: + type: template + template: | + [default] + region = {{secret:jobber-aws-configuration-region}} + + s3 = + endpoint_url = {{secret:jobber-aws-configuration-s3-endpoint}} + signature_version = s3v4 + max_concurrent_requests = 100 + max_queue_size = 1000 + multipart_threshold = 50MB + # Edit the multipart_chunksize value according to the file sizes that you want to upload. The present configuration allows to upload files up to 10 GB (100 requests * 10MB). For example setting it to 5GB allows you to upload files up to 5TB. + multipart_chunksize = 10MB + s3api = + endpoint_url = {{secret:jobber-aws-configuration-s3-endpoint}} + + [plugins] + endpoint = awscli_plugin_endpoint + jobber-aws-configuration-region: + type: third_party + jobber-aws-configuration-s3-endpoint: + type: third_party diff --git a/src/production/jobber/compose.yaml b/src/production/jobber/compose.yaml deleted file mode 100644 index 063fbc2f..00000000 --- a/src/production/jobber/compose.yaml +++ /dev/null @@ -1,10 +0,0 @@ -services: - jobber: - deploy: - update_config: - order: stop-first - environment: - SENTRY_CRONS: ${SENTRY_CRONS} - volumes: - - (( append )) - - postgres-backup-data:/backups/ diff --git a/src/production/jobber/configurations/.jobber b/src/production/jobber/configurations/.jobber deleted file mode 100644 index 162ba232..00000000 --- a/src/production/jobber/configurations/.jobber +++ /dev/null @@ -1,20 +0,0 @@ -version: 1.4 - -jobs: - DBBackup: - cmd: | - /srv/sinks/sentry/in-progress.sh - source .venv/bin/activate - aws s3 sync /backups s3://$(cat /run/secrets/jobber-aws-bucket)/backups - deactivate - sleep 3 - time: 0 0 0 * - notifyOnSuccess: - - type: program - path: /srv/sinks/sentry/ok.sh - notifyOnError: - - type: program - path: /srv/sinks/sentry/error.sh - notifyOnFailure: - - type: program - path: /srv/sinks/sentry/error.sh From 40926baf6446d4ccfa205725b50eb8f7a3e6a8cd Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Sun, 30 Aug 2026 02:02:20 +0200 Subject: [PATCH 06/16] fix(secrets): rename the workflow worker's database role --- src/development/postgres/compose.yaml | 27 +++++++++++++++------------ src/development/sqitch/compose.yaml | 2 ++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/development/postgres/compose.yaml b/src/development/postgres/compose.yaml index a8e480e9..424482bd 100644 --- a/src/development/postgres/compose.yaml +++ b/src/development/postgres/compose.yaml @@ -11,16 +11,6 @@ secrets: postgres-role-service-grafana-username: # The username of the observation dashboard's database role. file: ~~/artifacts/secrets/postgres-role-service-grafana-username.secret - postgres-role-service-jobber-password: - # The password of the workflow worker's outbox-purge database role. Kept - # under the `jobber` name jobber (the role's original owner) used, - # renaming would make dargstack generate a new random value and orphan - # the real grant already made on this role in the sqitch repo. - file: ~~/artifacts/secrets/postgres-role-service-jobber-password.secret - postgres-role-service-jobber-username: - # The username of the workflow worker's outbox-purge database role. Kept - # under the `jobber` name for the same reason as the password above. - file: ~~/artifacts/secrets/postgres-role-service-jobber-username.secret postgres-role-service-postgraphile-password: # The password of the GraphQL API database wrapper's database role. file: ~~/artifacts/secrets/postgres-role-service-postgraphile-password.secret @@ -34,11 +24,19 @@ secrets: # The username of the recommendation engine's database role. file: ~~/artifacts/secrets/postgres-role-service-reccoom-username.secret postgres-role-service-temporal-password: - # The password of the workflow orchestrator's own database role (schema ownership, not app data). + # The password of the workflow orchestrator's database role. + # The role owns the orchestrator's own two databases and holds none of the application's data. file: ~~/artifacts/secrets/postgres-role-service-temporal-password.secret postgres-role-service-temporal-username: - # The username of the workflow orchestrator's own database role (schema ownership, not app data). + # The username of the workflow orchestrator's database role. file: ~~/artifacts/secrets/postgres-role-service-temporal-username.secret + postgres-role-service-temporal-worker-password: + # The password of the workflow worker's database role. + # The role exists for the outbox purge workflow, which stays disabled until the outbox table lands in `sqitch`. + file: ~~/artifacts/secrets/postgres-role-service-temporal-worker-password.secret + postgres-role-service-temporal-worker-username: + # The username of the workflow worker's database role. + file: ~~/artifacts/secrets/postgres-role-service-temporal-worker-username.secret postgres-role-service-vibetype-password: # The `vibetype` database role's password. file: ~~/artifacts/secrets/postgres-role-service-vibetype-password.secret @@ -121,6 +119,11 @@ x-dargstack: type: random_string postgres-role-service-temporal-username: type: wordlist_word + postgres-role-service-temporal-worker-password: + special_characters: false + type: random_string + postgres-role-service-temporal-worker-username: + type: wordlist_word postgres-role-service-vibetype-password: special_characters: false type: random_string diff --git a/src/development/sqitch/compose.yaml b/src/development/sqitch/compose.yaml index ab36f09d..742d5b4c 100644 --- a/src/development/sqitch/compose.yaml +++ b/src/development/sqitch/compose.yaml @@ -22,6 +22,8 @@ services: - postgres-role-service-reccoom-username - postgres-role-service-temporal-password - postgres-role-service-temporal-username + - postgres-role-service-temporal-worker-password + - postgres-role-service-temporal-worker-username - postgres-role-service-vibetype-password - postgres-role-service-vibetype-username - postgres-role-service-zammad-username From 387b9006b1ff00af8f566238c82283e0eaad13cc Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Sun, 30 Aug 2026 02:02:20 +0200 Subject: [PATCH 07/16] fix(grafana): keep the workflow failure alert quiet without failures --- .../provisioning/alerting/alert-temporal-workflow-failures.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml b/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml index d9c2a19f..c533e240 100644 --- a/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml +++ b/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml @@ -17,7 +17,7 @@ groups: datasourceUid: prometheus model: editorMode: code - expr: sum(increase(workflow_failed[10m])) + expr: sum(increase(workflow_failed[10m])) or vector(0) instant: true intervalMs: 1000 maxDataPoints: 43200 From e6128df6f122419035928307a26ea5baf281a79c Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Sun, 30 Aug 2026 02:02:20 +0200 Subject: [PATCH 08/16] fix(prometheus): scrape the workflow worker --- src/development/prometheus/configurations/prometheus.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/development/prometheus/configurations/prometheus.yaml b/src/development/prometheus/configurations/prometheus.yaml index 16f19f75..6b692fd7 100644 --- a/src/development/prometheus/configurations/prometheus.yaml +++ b/src/development/prometheus/configurations/prometheus.yaml @@ -37,3 +37,7 @@ scrape_configs: - job_name: "temporal" static_configs: - targets: ["temporal:8000"] + + - job_name: "temporal-worker" + static_configs: + - targets: ["temporal-worker:9090"] From 91ad24477334f61d633b76bb5faf60995f7c8521 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Sun, 30 Aug 2026 02:02:20 +0200 Subject: [PATCH 09/16] feat(traefik): protect the redpanda and temporal uis with basic authentication --- src/development/traefik/compose.yaml | 22 ++++++++++++++++++++++ src/production/redpanda/compose.yaml | 3 +++ 2 files changed, 25 insertions(+) diff --git a/src/development/traefik/compose.yaml b/src/development/traefik/compose.yaml index 0aeee6e6..e0198ae2 100644 --- a/src/development/traefik/compose.yaml +++ b/src/development/traefik/compose.yaml @@ -1,3 +1,10 @@ +secrets: + redpanda-console-users: + # The event streaming platform ui's basic authentication users, in htpasswd format. + file: ~~/artifacts/secrets/redpanda-console-users.secret + temporal-ui-users: + # The workflow orchestrator ui's basic authentication users, in htpasswd format. + file: ~~/artifacts/secrets/temporal-ui-users.secret services: traefik: # You can access the reverse proxy's dashboard at [traefik.app.localhost](https://traefik.app.localhost/). @@ -18,6 +25,10 @@ services: - dargstack.profiles=default,analytics,event-streaming,recommendation,upload,zammad - traefik.enable=true - traefik.http.middlewares.redirectscheme.redirectscheme.scheme=https # dargstack:dev-only + # These two uis have no login of their own, so the reverse proxy supplies one. + # Both the middlewares and their user files live here because it is the reverse proxy that has to read them, not the services they protect. + - traefik.http.middlewares.redpanda-console-auth.basicauth.usersfile=/run/secrets/redpanda-console-users + - traefik.http.middlewares.temporal-ui-auth.basicauth.usersfile=/run/secrets/temporal-ui-users - traefik.http.routers.traefik.entryPoints=web - traefik.http.routers.traefik.middlewares=redirectscheme # dargstack:dev-only - traefik.http.routers.traefik.rule=Host(`traefik.${STACK_DOMAIN}`) @@ -47,7 +58,18 @@ services: protocol: tcp # dargstack:dev-only published: 443 # dargstack:dev-only target: 443 # dargstack:dev-only + secrets: + - redpanda-console-users + - temporal-ui-users volumes: - /var/run/docker.sock:/var/run/docker.sock - ~~/artifacts/certificates/:/etc/traefik/acme/ # dargstack:dev-only - ./configurations/dynamic.yml:/dynamic.yml:ro # dargstack:dev-only +x-dargstack: + secrets: + redpanda-console-users: + type: insecure_default + insecure_default: "redpanda:$2y$05$5M8cW1XyOSaT9bnOyKGvmuLHd8tbM0uB4gNuh8p7E0Ee0rQUMOFwy" + temporal-ui-users: + type: insecure_default + insecure_default: "temporal:$2y$05$gM4UgMFoaY7P3h/9X.2TCOM3MXoEeA7bvbIbUTJIbghoKf8CQWnqa" diff --git a/src/production/redpanda/compose.yaml b/src/production/redpanda/compose.yaml index 8d384353..6e53e9fa 100644 --- a/src/production/redpanda/compose.yaml +++ b/src/production/redpanda/compose.yaml @@ -7,6 +7,9 @@ services: deploy: labels: - (( append )) + # The ui has no login of its own, so both routers sit behind basic authentication. + - traefik.http.routers.redpanda.middlewares=redpanda-console-auth + - traefik.http.routers.redpanda-secure.middlewares=redpanda-console-auth - traefik.http.routers.redpanda-secure.tls.certresolver=default update_config: order: start-first From e828110fb78f9c6ebac43dcc25dcdcc7a2fc637f Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Sun, 30 Aug 2026 02:02:20 +0200 Subject: [PATCH 10/16] fix(temporal): correct the worker's repository, configuration and profiles --- artifacts/docs/README.md | 2 +- src/development/.env.template | 3 + src/development/temporal-worker/compose.yaml | 80 -------------------- src/development/temporal/compose.yaml | 62 ++++++++++++++- src/production/temporal-worker/compose.yaml | 9 --- src/production/temporal/compose.yaml | 16 +++- 6 files changed, 80 insertions(+), 92 deletions(-) delete mode 100644 src/development/temporal-worker/compose.yaml delete mode 100644 src/production/temporal-worker/compose.yaml diff --git a/artifacts/docs/README.md b/artifacts/docs/README.md index c2ff2d15..11e1b01a 100644 --- a/artifacts/docs/README.md +++ b/artifacts/docs/README.md @@ -8,7 +8,7 @@ Profiles group services so you can deploy subsets on demand. Activate with `darg ### analytics -Services: cadvisor, grafana, node-exporter, postgres, prometheus, sqitch, traefik +Services: cadvisor, grafana, node-exporter, postgres, prometheus, sqitch, temporal, temporal-ui, temporal-worker, traefik ### default diff --git a/src/development/.env.template b/src/development/.env.template index 546de5a7..65584a0b 100644 --- a/src/development/.env.template +++ b/src/development/.env.template @@ -1,4 +1,7 @@ RECCOOM_ADMIN_ACCOUNT_IDS= +TEMPORAL_WORKER_S3_ENDPOINT=http://minio:9000 +TEMPORAL_WORKER_S3_REGION=eu-central-1 +TEMPORAL_WORKER_S3_USE_PATH_STYLE=true TUSD_BUCKET=vibetype-images TUSD_ENDPOINT=http://minio:9000 TUSD_MAX_SIZE=10485760 diff --git a/src/development/temporal-worker/compose.yaml b/src/development/temporal-worker/compose.yaml deleted file mode 100644 index 35572a4b..00000000 --- a/src/development/temporal-worker/compose.yaml +++ /dev/null @@ -1,80 +0,0 @@ -secrets: - jobber-aws-bucket: - # The workflow worker's AWS bucket name for DBBackup. Kept under the - # `jobber` name jobber originally used, renaming would make dargstack - # generate a new random value and orphan the real S3 bucket already in - # use in production. - file: ~~/artifacts/secrets/jobber-aws-bucket.secret - jobber-aws-credentials: - # The workflow worker's AWS credentials for DBBackup. Kept under the - # `jobber` name for the same reason as `jobber-aws-bucket`. - file: ~~/artifacts/secrets/jobber-aws-credentials.secret - jobber-aws-configuration: - # The workflow worker's AWS configuration for DBBackup. Kept under the - # `jobber` name for the same reason as `jobber-aws-bucket`. - file: ~~/artifacts/secrets/jobber-aws-configuration.secret -services: - temporal-worker: - # You cannot access the workflow worker directly, use `temporal-ui`. - deploy: - labels: - - dargstack.development.git.ssh=git@github.com:maevsi/temporal-worker.git - - dargstack.development.git.https=https://github.com/maevsi/temporal-worker.git - environment: - AWS_SHARED_CREDENTIALS_FILE: /run/secrets/jobber-aws-credentials - POSTGRES_HOST: postgres - TEMPORAL_ADDRESS: temporal:7233 - image: vibetype/temporal-worker:development - secrets: - - source: jobber-aws-bucket - target: /run/environment-variables/AWS_BUCKET - - jobber-aws-credentials - - source: jobber-aws-configuration - target: /home/node/.aws/config - - source: postgres-db - target: /run/environment-variables/POSTGRES_DATABASE - - source: postgres-role-service-jobber-password - target: /run/environment-variables/POSTGRES_PASSWORD - - source: postgres-role-service-jobber-username - target: /run/environment-variables/POSTGRES_USER - volumes: - - ~~~/temporal-worker/:/srv/app/ # dargstack:dev-only - - pnpm-data:/srv/.pnpm-store/ # dargstack:dev-only - - temporal-worker-data:/srv/app/node_modules # dargstack:dev-only -volumes: - temporal-worker-data: - # The workflow worker's data. - {} -x-dargstack: - secrets: - jobber-aws-bucket: - type: wordlist_word - jobber-aws-credentials: - type: insecure_default - insecure_default: | - [default] - aws_access_key_id = s3user - aws_secret_access_key = s3password - jobber-aws-configuration: - type: template - template: | - [default] - region = {{secret:jobber-aws-configuration-region}} - - s3 = - endpoint_url = {{secret:jobber-aws-configuration-s3-endpoint}} - signature_version = s3v4 - max_concurrent_requests = 100 - max_queue_size = 1000 - multipart_threshold = 50MB - # Edit the multipart_chunksize value according to the file sizes that you want to upload. The present configuration allows to upload files up to 10 GB (100 requests * 10MB). For example setting it to 5GB allows you to upload files up to 5TB. - multipart_chunksize = 10MB - s3api = - endpoint_url = {{secret:jobber-aws-configuration-s3-endpoint}} - - [plugins] - endpoint = awscli_plugin_endpoint - jobber-aws-configuration-region: - type: third_party - jobber-aws-configuration-s3-endpoint: - type: third_party diff --git a/src/development/temporal/compose.yaml b/src/development/temporal/compose.yaml index ddd10b6e..f0425dd6 100644 --- a/src/development/temporal/compose.yaml +++ b/src/development/temporal/compose.yaml @@ -1,13 +1,30 @@ +secrets: + jobber-aws-bucket: + # The name of the S3 bucket the workflow worker syncs database backups to. + # This secret keeps the name jobber gave it because dargstack generated its value and that value is the name of the bucket that already holds production's backups. + # Renaming it would generate a new word and point the backups at a bucket that does not exist. + file: ~~/artifacts/secrets/jobber-aws-bucket.secret + temporal-worker-s3-access-key-id: + # The access key id the workflow worker uses to reach the backup bucket. + file: ~~/artifacts/secrets/temporal-worker-s3-access-key-id.secret + temporal-worker-s3-secret-access-key: + # The secret access key the workflow worker uses to reach the backup bucket. + file: ~~/artifacts/secrets/temporal-worker-s3-secret-access-key.secret services: temporal: # You cannot access the workflow orchestrator directly, use `temporal-ui`. - entrypoint: sh command: ["/entrypoint.sh"] + deploy: + labels: + - dargstack.profiles=analytics + entrypoint: sh environment: DB: postgres12 DB_PORT: "5432" POSTGRES_SEEDS: postgres PROMETHEUS_ENDPOINT: 0.0.0.0:8000 + # `sqitch` owns the two databases, so the role does not need `CREATEDB`. + SKIP_DB_CREATE: "true" healthcheck: test: ["CMD-SHELL", "nc -z 127.0.0.1 7233 || exit 1"] interval: 30s @@ -24,6 +41,7 @@ services: # You can access the workflow orchestrator's ui at [temporal.app.localhost](https://temporal.app.localhost/). deploy: labels: + - dargstack.profiles=analytics - traefik.enable=true - traefik.http.routers.temporal.entryPoints=web - traefik.http.routers.temporal.middlewares=redirectscheme # dargstack:dev-only @@ -42,3 +60,45 @@ services: retries: 3 start_period: 15s image: temporalio/ui:2.40.1 + temporal-worker: + # You cannot access the workflow worker directly, use `temporal-ui`. + deploy: + labels: + - dargstack.development.git.ssh=git@github.com:maevsi/temporal.git + - dargstack.development.git.https=https://github.com/maevsi/temporal.git + - dargstack.profiles=analytics + environment: + S3_ENDPOINT: ${TEMPORAL_WORKER_S3_ENDPOINT} + S3_REGION: ${TEMPORAL_WORKER_S3_REGION} + S3_USE_PATH_STYLE: ${TEMPORAL_WORKER_S3_USE_PATH_STYLE} + TEMPORAL_HOST_PORT: temporal:7233 + image: vibetype/temporal:development + secrets: + - source: jobber-aws-bucket + target: /run/environment-variables/S3_BUCKET + - source: temporal-worker-s3-access-key-id + target: /run/environment-variables/S3_ACCESS_KEY_ID + - source: temporal-worker-s3-secret-access-key + target: /run/environment-variables/S3_SECRET_ACCESS_KEY + volumes: # dargstack:dev-only + - ~~~/temporal/:/srv/app/ # dargstack:dev-only + - temporal-worker-data:/home/go/go/pkg/mod # dargstack:dev-only + - temporal-worker-backup-data:/backups/ # dargstack:dev-only +volumes: + temporal-worker-backup-data: + # The workflow worker's backup source directory. + # Production mounts the database backup's data here instead; development keeps an empty volume so the backup workflow is a no-op rather than a recurring failure. + {} + temporal-worker-data: + # The workflow worker's Go module cache. + {} +x-dargstack: + secrets: + jobber-aws-bucket: + type: wordlist_word + temporal-worker-s3-access-key-id: + type: insecure_default + insecure_default: s3user + temporal-worker-s3-secret-access-key: + type: insecure_default + insecure_default: s3password diff --git a/src/production/temporal-worker/compose.yaml b/src/production/temporal-worker/compose.yaml deleted file mode 100644 index 54c0fe63..00000000 --- a/src/production/temporal-worker/compose.yaml +++ /dev/null @@ -1,9 +0,0 @@ -services: - temporal-worker: - deploy: - update_config: - order: start-first - environment: - SENTRY_CRONS: ${SENTRY_CRONS} - SENTRY_CRONS_OUTBOX_PURGE: ${SENTRY_CRONS_OUTBOX_PURGE} - image: ghcr.io/maevsi/temporal-worker:0.1.0 diff --git a/src/production/temporal/compose.yaml b/src/production/temporal/compose.yaml index 9f643d85..4f54b612 100644 --- a/src/production/temporal/compose.yaml +++ b/src/production/temporal/compose.yaml @@ -7,7 +7,21 @@ services: deploy: labels: - (( append )) - - traefik.http.routers.temporal.middlewares=redirectscheme + # The ui has no login of its own and can terminate or signal workflows, so both routers sit behind basic authentication. + - traefik.http.routers.temporal.middlewares=temporal-ui-auth + - traefik.http.routers.temporal-secure.middlewares=temporal-ui-auth - traefik.http.routers.temporal-secure.tls.certresolver=default update_config: order: start-first + temporal-worker: + deploy: + update_config: + order: start-first + environment: + SENTRY_CRONS: ${SENTRY_CRONS} + image: ghcr.io/maevsi/temporal:0.1.0 + volumes: + - postgres-backup-data:/backups/ +volumes: + temporal-worker-backup-data: (( prune )) + temporal-worker-data: (( prune )) From ae4d6bef049a9bf3c74d109d8915429821f6abbb Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Thu, 3 Sep 2026 12:52:23 +0200 Subject: [PATCH 11/16] docs(temporal): start each comment sentence on its own line --- src/development/temporal/compose.yaml | 3 ++- src/development/temporal/configurations/entrypoint.sh | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/development/temporal/compose.yaml b/src/development/temporal/compose.yaml index f0425dd6..77874ecb 100644 --- a/src/development/temporal/compose.yaml +++ b/src/development/temporal/compose.yaml @@ -87,7 +87,8 @@ services: volumes: temporal-worker-backup-data: # The workflow worker's backup source directory. - # Production mounts the database backup's data here instead; development keeps an empty volume so the backup workflow is a no-op rather than a recurring failure. + # Production mounts the database backup's data here instead. + # Development keeps an empty volume so the backup workflow is a no-op rather than a recurring failure. {} temporal-worker-data: # The workflow worker's Go module cache. diff --git a/src/development/temporal/configurations/entrypoint.sh b/src/development/temporal/configurations/entrypoint.sh index 83c60719..59563e98 100644 --- a/src/development/temporal/configurations/entrypoint.sh +++ b/src/development/temporal/configurations/entrypoint.sh @@ -1,9 +1,7 @@ #!/bin/sh set -eu -# The auto-setup image has no support for Docker's `_FILE`-suffixed secret -# convention, so read the mounted secret files ourselves and hand the plain -# values to the image's own entrypoint. +# The auto-setup image has no support for Docker's `_FILE`-suffixed secret convention, so read the mounted secret files ourselves and hand the plain values to the image's own entrypoint. export POSTGRES_USER="$(cat /run/secrets/postgres-role-service-temporal-username)" export POSTGRES_PWD="$(cat /run/secrets/postgres-role-service-temporal-password)" From 3da2df4ac2b47f1fb529059dc553032193eb8f88 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Thu, 3 Sep 2026 12:52:28 +0200 Subject: [PATCH 12/16] docs(grafana): describe the workflow failure alert instead of provisioning it --- docs/grafana.md | 18 +++- .../alert-temporal-workflow-failures.yaml | 84 ------------------- 2 files changed, 17 insertions(+), 85 deletions(-) delete mode 100644 src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml diff --git a/docs/grafana.md b/docs/grafana.md index 41247513..9d8b1688 100644 --- a/docs/grafana.md +++ b/docs/grafana.md @@ -79,6 +79,21 @@ Add under **Alerting → Alert rules → New alert rule**, in the `Infrastructur - **Summary annotation**: "There are notifications which are not sent out, or at least not marked as acknowledged." - **Contact point**: `Discord` +### Alert rule: workflow failures + +Add under **Alerting → Alert rules → New alert rule**, in the `Infrastructure` folder. + +- **Query (A)**, on the Prometheus datasource, code mode, instant query over the last 10 minutes: `sum(increase(workflow_failed[10m])) or vector(0)` +- **Reduce (B)**: last of A +- **Threshold (C)**: B is above 0, this is the alert condition +- **Evaluation**: every `1m`, for `1m`, group `Temporal`, repeat every 7 days +- **No data / error state**: `NoData` / `Error` +- **Summary annotation**: "Temporal reported one or more failed workflow executions in the last 10 minutes." +- **Contact point**: `Discord` + +The `or vector(0)` keeps the query from going stale while no workflow has ever failed. +Without it `workflow_failed` does not exist yet, the rule evaluates to no data, and the alert fires on the `NoData` state instead of staying quiet. + ## Dashboards Add under **Dashboards → New → Import**. @@ -93,7 +108,8 @@ The following were provisioned from public community dashboards and can be reimp The `Redpanda Ops Dashboard`, `Grafana metrics`, and `Prometheus 2.0 Stats` dashboards were also provisioned from community sources, but without a recorded grafana.com ID. Search grafana.com's dashboard library by name, or recover the exact JSON that was previously provisioned from this repository's git history (`git log --diff-filter=D -- 'src/development/grafana/configurations/dashboards/**'`) and import it via **Upload dashboard JSON file**. -Two dashboards were specific to this project rather than imported from the community, both querying the PostgreSQL datasource. Recreate their panels as needed: +Two dashboards were specific to this project rather than imported from the community, both querying the PostgreSQL datasource. +Recreate their panels as needed: **KPIs** (folder `Management`): diff --git a/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml b/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml deleted file mode 100644 index c533e240..00000000 --- a/src/development/grafana/configurations/provisioning/alerting/alert-temporal-workflow-failures.yaml +++ /dev/null @@ -1,84 +0,0 @@ -apiVersion: 1 -groups: - - orgId: 1 - name: Temporal - folder: Infrastructure - interval: 1m - repeat_interval: 604800 # 7 days - rules: - - uid: een6tworkflowfail - title: Workflow failures - condition: C - data: - - refId: A - relativeTimeRange: - from: 600 - to: 0 - datasourceUid: prometheus - model: - editorMode: code - expr: sum(increase(workflow_failed[10m])) or vector(0) - instant: true - intervalMs: 1000 - maxDataPoints: 43200 - range: false - refId: A - - refId: B - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: [] - type: gt - operator: - type: and - query: - params: - - B - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - reducer: last - refId: B - type: reduce - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: B - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - noDataState: NoData - execErrState: Error - for: 1m - annotations: - summary: Temporal reported one or more failed workflow executions in the last 10 minutes. - labels: {} - isPaused: false - notification_settings: - receiver: 'Discord' From a3cb16b75383d9feb8b35f170824cdfb2fe224dd Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Thu, 3 Sep 2026 12:52:28 +0200 Subject: [PATCH 13/16] fix(traefik): keep the base secrets when overriding for production --- src/production/traefik/compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/production/traefik/compose.yaml b/src/production/traefik/compose.yaml index 32982339..25486df8 100644 --- a/src/production/traefik/compose.yaml +++ b/src/production/traefik/compose.yaml @@ -22,6 +22,7 @@ services: CF_DNS_API_TOKEN_FILE: /run/secrets/traefik-cf-dns-api-token CF_ZONE_API_TOKEN_FILE: /run/secrets/traefik-cf-zone-api-token secrets: + - (( append )) - traefik-cf-dns-api-token - traefik-cf-zone-api-token volumes: From 0182c104d4db2d6d954019db921f3406489be813 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Thu, 3 Sep 2026 12:52:33 +0200 Subject: [PATCH 14/16] feat(traefik): generate the uis' basic authentication credentials --- dargstack.yaml | 2 +- src/development/traefik/compose.yaml | 58 ++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/dargstack.yaml b/dargstack.yaml index 1d1d23b4..f82e1119 100644 --- a/dargstack.yaml +++ b/dargstack.yaml @@ -14,7 +14,7 @@ environment: # tag: latest # optional, defaults to "latest" metadata: - compatibility: ">=4.13.0-0 <5.0.0" + compatibility: ">=4.17.0-0 <5.0.0" name: "vibetype" source: name: "maevsi" diff --git a/src/development/traefik/compose.yaml b/src/development/traefik/compose.yaml index e0198ae2..aa81868e 100644 --- a/src/development/traefik/compose.yaml +++ b/src/development/traefik/compose.yaml @@ -1,10 +1,22 @@ secrets: - redpanda-console-users: - # The event streaming platform ui's basic authentication users, in htpasswd format. - file: ~~/artifacts/secrets/redpanda-console-users.secret - temporal-ui-users: - # The workflow orchestrator ui's basic authentication users, in htpasswd format. - file: ~~/artifacts/secrets/temporal-ui-users.secret + redpanda-console-auth: + # The event streaming platform ui's basic authentication credential, in htpasswd format. + file: ~~/artifacts/secrets/redpanda-console-auth.secret + redpanda-console-password: + # The password half of the event streaming platform ui's basic authentication credential, before hashing. + file: ~~/artifacts/secrets/redpanda-console-password.secret + redpanda-console-username: + # The username half of the event streaming platform ui's basic authentication credential. + file: ~~/artifacts/secrets/redpanda-console-username.secret + temporal-ui-auth: + # The workflow orchestrator ui's basic authentication credential, in htpasswd format. + file: ~~/artifacts/secrets/temporal-ui-auth.secret + temporal-ui-password: + # The password half of the workflow orchestrator ui's basic authentication credential, before hashing. + file: ~~/artifacts/secrets/temporal-ui-password.secret + temporal-ui-username: + # The username half of the workflow orchestrator ui's basic authentication credential. + file: ~~/artifacts/secrets/temporal-ui-username.secret services: traefik: # You can access the reverse proxy's dashboard at [traefik.app.localhost](https://traefik.app.localhost/). @@ -26,9 +38,9 @@ services: - traefik.enable=true - traefik.http.middlewares.redirectscheme.redirectscheme.scheme=https # dargstack:dev-only # These two uis have no login of their own, so the reverse proxy supplies one. - # Both the middlewares and their user files live here because it is the reverse proxy that has to read them, not the services they protect. - - traefik.http.middlewares.redpanda-console-auth.basicauth.usersfile=/run/secrets/redpanda-console-users - - traefik.http.middlewares.temporal-ui-auth.basicauth.usersfile=/run/secrets/temporal-ui-users + # Both the middlewares and their credential files live here because it is the reverse proxy that has to read them, not the services they protect. + - traefik.http.middlewares.redpanda-console-auth.basicauth.usersfile=/run/secrets/redpanda-console-auth + - traefik.http.middlewares.temporal-ui-auth.basicauth.usersfile=/run/secrets/temporal-ui-auth - traefik.http.routers.traefik.entryPoints=web - traefik.http.routers.traefik.middlewares=redirectscheme # dargstack:dev-only - traefik.http.routers.traefik.rule=Host(`traefik.${STACK_DOMAIN}`) @@ -59,17 +71,29 @@ services: published: 443 # dargstack:dev-only target: 443 # dargstack:dev-only secrets: - - redpanda-console-users - - temporal-ui-users + - redpanda-console-auth + - temporal-ui-auth volumes: - /var/run/docker.sock:/var/run/docker.sock - ~~/artifacts/certificates/:/etc/traefik/acme/ # dargstack:dev-only - ./configurations/dynamic.yml:/dynamic.yml:ro # dargstack:dev-only x-dargstack: secrets: - redpanda-console-users: - type: insecure_default - insecure_default: "redpanda:$2y$05$5M8cW1XyOSaT9bnOyKGvmuLHd8tbM0uB4gNuh8p7E0Ee0rQUMOFwy" - temporal-ui-users: - type: insecure_default - insecure_default: "temporal:$2y$05$gM4UgMFoaY7P3h/9X.2TCOM3MXoEeA7bvbIbUTJIbghoKf8CQWnqa" + # Only the hash reaches the reverse proxy, so both halves get a file of their own that no service mounts, because the generated credential would be unrecoverable otherwise. + # Read them back with `dargstack secret show` when you need to log in. + redpanda-console-auth: + source: redpanda-console-password + type: basic_auth + username: "{{secret:redpanda-console-username}}" + redpanda-console-password: + type: random_string + redpanda-console-username: + type: wordlist_word + temporal-ui-auth: + source: temporal-ui-password + type: basic_auth + username: "{{secret:temporal-ui-username}}" + temporal-ui-password: + type: random_string + temporal-ui-username: + type: wordlist_word From 3b207782dba87870bdaefe7df1c0d54ec3e19670 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Fri, 4 Sep 2026 15:45:20 +0200 Subject: [PATCH 15/16] feat(temporal): pin the orchestrator and worker to the storage node --- docs/multi-node.md | 6 +++--- src/production/temporal/compose.yaml | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/multi-node.md b/docs/multi-node.md index 2bc5186b..5c46fb2d 100644 --- a/docs/multi-node.md +++ b/docs/multi-node.md @@ -17,11 +17,11 @@ On any other node that path does not exist, and Docker silently creates an empty ## Which services are pinned -Services holding a production volume: `debezium`, `elasticsearch`, `grafana`, `jobber`, `portainer`, `postgres`, `postgres-backup`, `prometheus`, `reccoom-postgres`, `redis`, `redpanda`, `traefik`, `traefik-certs-dumper` and all six `zammad` services. +Services holding a production volume: `debezium`, `elasticsearch`, `grafana`, `portainer`, `postgres`, `postgres-backup`, `prometheus`, `reccoom-postgres`, `redis`, `redpanda`, `temporal-worker`, `traefik`, `traefik-certs-dumper` and all six `zammad` services. -Services holding only a `configurations/` bind mount: `adminer`, `debezium-postgres-connector`, `redpanda-console`. +Services holding only a `configurations/` bind mount: `adminer`, `debezium-postgres-connector`, `redpanda-console`, `temporal`. -Some of these also share a volume and therefore have to land on the same node as each other, which the shared label already guarantees: `postgres` with `postgres-backup` with `jobber`, `traefik` with `traefik-certs-dumper`, and the six `zammad` services with one another. +Some of these also share a volume and therefore have to land on the same node as each other, which the shared label already guarantees: `postgres` with `postgres-backup` with `temporal-worker`, `traefik` with `traefik-certs-dumper`, and the six `zammad` services with one another. ## Adding a service diff --git a/src/production/temporal/compose.yaml b/src/production/temporal/compose.yaml index 4f54b612..10df019a 100644 --- a/src/production/temporal/compose.yaml +++ b/src/production/temporal/compose.yaml @@ -1,6 +1,9 @@ services: temporal: deploy: + placement: + constraints: + - node.labels.vibetype.storage == true update_config: order: stop-first temporal-ui: @@ -15,6 +18,9 @@ services: order: start-first temporal-worker: deploy: + placement: + constraints: + - node.labels.vibetype.storage == true update_config: order: start-first environment: From 317598d7d1f057352b732d97f9154950c38b282f Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Fri, 4 Sep 2026 15:45:20 +0200 Subject: [PATCH 16/16] fix(temporal): source the worker's s3 credentials from minio --- src/development/temporal/compose.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/development/temporal/compose.yaml b/src/development/temporal/compose.yaml index 77874ecb..e9f857e7 100644 --- a/src/development/temporal/compose.yaml +++ b/src/development/temporal/compose.yaml @@ -98,8 +98,8 @@ x-dargstack: jobber-aws-bucket: type: wordlist_word temporal-worker-s3-access-key-id: - type: insecure_default - insecure_default: s3user + template: "{{secret:minio-root-user}}" + type: template temporal-worker-s3-secret-access-key: - type: insecure_default - insecure_default: s3password + template: "{{secret:minio-root-password}}" + type: template