diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1c257fb..5268040 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ version: 2 updates: # Dependabot's docker file fetcher matches any filename containing "dockerfile" or # "containerfile", case-insensitively (DOCKER_REGEXP in dependabot-core), so Dockerfile-17 - # is picked up without renaming it. + # and Dockerfile-18 are picked up without renaming them. # # Each PR here is a PostgreSQL minor release. Read the Dockerfile header before merging: # a bump below the minor version a live data directory runs is a downgrade PostgreSQL does @@ -16,6 +16,15 @@ updates: prefix: docker labels: - dependencies + # Each Dockerfile- stays on its own PostgreSQL major. A major bump would point the + # `` tag at a server that cannot read the data directories already using it, so + # only minor bumps are raised. Adding a new major means adding a new Dockerfile-. + # Dockerfile-upgrade-17-18 pins railwayapp-templates/postgres-ssl/upgrade, a separate + # dependency name, so this rule does not cover it. + ignore: + - dependency-name: railwayapp-templates/postgres-ssl + update-types: + - version-update:semver-major - package-ecosystem: github-actions directory: / diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71f3d24..30bfbd5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,9 @@ name: build # recognition: no Data tab, and `railway connect` answers "No supported database found in # service". A service pointed at a pre-built image keeps it. So this workflow publishes the # image and each Railway service stays image-sourced, with Source Image set to -# ghcr.io/fountain-bio/postgres:17. +# ghcr.io/fountain-bio/postgres:, for example ghcr.io/fountain-bio/postgres:17. +# Moving a service from 17 to 18 goes through the upgrade job image published as +# ghcr.io/fountain-bio/postgres:upgrade-17-18; Dockerfile-upgrade-17-18 has the runbook. # # The package has to be pullable by Railway. Make it public under the org's package settings, # or add registry credentials on each service. @@ -23,8 +25,17 @@ concurrency: cancel-in-progress: true jobs: - postgres-17: + postgres: + name: postgres-${{ matrix.image }} runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + # Each image publishes its own tags, so a failed 18 build should still let 17 publish. + fail-fast: false + matrix: + # Each entry builds from Dockerfile- and publishes the `` tag. `17` and + # `18` are the runtime images; `upgrade-17-18` is the one-shot in-place upgrade job + # described in Dockerfile-upgrade-17-18. + image: ['17', '18', 'upgrade-17-18'] permissions: contents: read packages: write @@ -46,16 +57,16 @@ jobs: uses: docker/metadata-action@v6.2.0 with: images: ghcr.io/fountain-bio/postgres - # `17` is the moving tag Railway points at. The sha tag gives every build a stable - # identity, so a bad rollout can be pinned back to a known-good digest. + # `` is the moving tag Railway points at. The sha tag gives every build a + # stable identity, so a bad rollout can be pinned back to a known-good digest. tags: | - type=raw,value=17 - type=raw,value=17-${{ github.sha }} + type=raw,value=${{ matrix.image }} + type=raw,value=${{ matrix.image }}-${{ github.sha }} - uses: useblacksmith/build-push-action@v2.3.0 with: context: . - file: ./Dockerfile-17 + file: ./Dockerfile-${{ matrix.image }} # Railway runs amd64. Building arm64 as well would double build time for an image # nothing pulls on that architecture. platforms: linux/amd64 diff --git a/Dockerfile-17 b/Dockerfile-17 index 5f50e60..aed9ab3 100644 --- a/Dockerfile-17 +++ b/Dockerfile-17 @@ -8,16 +8,13 @@ # `FATAL: could not access file "pg_cron"`. Baking the extension into the image is what # makes a redeploy, an image change, or a platform restart survivable. # -# pg_cron drives the Inngest run/trace retention job (`ops/inngest-prune.sql` in the -# fountain-bio monorepo), which is the only thing keeping those trace stores bounded. -# # The base tag is PINNED rather than floating on `:17` so Dependabot raises a visible PR for # each Postgres minor release. Two rules when that PR arrives: # 1. Never merge a bump that moves BELOW the minor version a live data directory runs. # PostgreSQL guarantees minor upgrades, not minor downgrades. # 2. Merging does not deploy. Railway pulls the new image on the next redeploy of each # service, so prod moves when you move it. -FROM ghcr.io/railwayapp-templates/postgres-ssl:17.10 +FROM ghcr.io/railwayapp-templates/postgres-ssl:17.11 # The package major version has to track the base image's major version. RUN apt-get update \ diff --git a/Dockerfile-18 b/Dockerfile-18 new file mode 100644 index 0000000..52f4272 --- /dev/null +++ b/Dockerfile-18 @@ -0,0 +1,22 @@ +# Railway's Postgres image with pg_cron added. +# +# Railway's stock Postgres ships no extensions, so pg_cron used to be apt-installed by hand +# inside the running container. That splits the install across two lifetimes: the +# `shared_preload_libraries = pg_stat_statements,pg_cron` setting lives in postgresql.conf +# inside PGDATA, which is on the volume, while pg_cron.so lives only in the container +# filesystem. Replace the container and Postgres refuses to start with +# `FATAL: could not access file "pg_cron"`. Baking the extension into the image is what +# makes a redeploy, an image change, or a platform restart survivable. +# +# The base tag is PINNED rather than floating on `:18` so Dependabot raises a visible PR for +# each Postgres minor release. Two rules when that PR arrives: +# 1. Never merge a bump that moves BELOW the minor version a live data directory runs. +# PostgreSQL guarantees minor upgrades, not minor downgrades. +# 2. Merging does not deploy. Railway pulls the new image on the next redeploy of each +# service, so prod moves when you move it. +FROM ghcr.io/railwayapp-templates/postgres-ssl:18.6 + +# The package major version has to track the base image's major version. +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql-18-cron \ + && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile-upgrade-17-18 b/Dockerfile-upgrade-17-18 new file mode 100644 index 0000000..0b49bb4 --- /dev/null +++ b/Dockerfile-upgrade-17-18 @@ -0,0 +1,47 @@ +# One-shot job image that upgrades a PostgreSQL 17 data directory to 18 in place. +# +# Railway's dashboard upgrade (Database > Config > Major Version Upgrade) only runs for +# services on Railway's own postgres-ssl image, so it is unavailable to services on +# ghcr.io/fountain-bio/postgres. This image runs the same upgrade job that flow uses: +# Railway's published 17-to-18 job image, which carries both majors' server binaries and +# runs `pg_upgrade --link` through upgrade-job.sh against the service's own volume. +# +# pg_cron has to be added for both majors. Our volumes preload pg_cron, so the 17 server +# cannot start without postgresql-17-cron, and pg_upgrade's loadable-libraries check fails +# unless postgresql-18-cron is present for the new cluster. upgrade-job.sh copies the old +# cluster's shared_preload_libraries onto the 18 server when the .so exists, and pg_cron +# skips starting its scheduler while pg_upgrade runs the servers in binary-upgrade mode. +# +# Runbook, per database service. The job runs as a deployment of the database service so it +# gets the service's volume and variables (PGDATA, POSTGRES_USER): +# 1. Take a backup of the service. +# 2. Set the service's restart policy to Never so a failed job is not rerun on its own. +# Set the service variable UPGRADE_JOB_MODE=check, set Source Image to +# ghcr.io/fountain-bio/postgres:upgrade-17-18, and deploy. The database is down from here +# until step 5. The job exits after one run; read the RAILWAY_UPGRADE_RESULT line in the +# deploy logs. Exit 1 lists blockers to fix on 17 before continuing. +# 3. Set UPGRADE_JOB_MODE=upgrade and redeploy. The job writes a completion marker on the +# volume once pg_upgrade succeeds, then swaps directories. Re-running upgrade mode after +# a crash resumes from the marker. +# 4. Remove UPGRADE_JOB_MODE and put the restart policy back. +# 5. Set Source Image to ghcr.io/fountain-bio/postgres:18 and deploy. The 18 image's +# wrapper refuses to boot while an unfinished marker exists or the data directory is +# on another major. +# To back out before step 3 commits, deploy with UPGRADE_JOB_MODE=recover, then return the +# Source Image to ghcr.io/fountain-bio/postgres:17. After the commit point, restore the backup. +# +# The job requires the volume mounted at /var/lib/postgresql/data with PGDATA beneath it, +# which is the layout Railway's 17 template created. +# +# The base is pinned by digest because Railway publishes this image only under the moving +# `17-18` tag. +FROM ghcr.io/railwayapp-templates/postgres-ssl/upgrade:17-18@sha256:b6ff5fcb8a45da6f132fa60c8369d33a562b28d5a0d7ef3a52404b8bf575531f + +# The base image's PGDG apt source already lists both the 17 and 18 components. PGDG builds +# each pg_cron package against the newest server minor and depends on it, so this install +# can raise postgresql-17 and postgresql-18 past the minors the base image bundles. The +# runtime images get the same raise from their own pg_cron install, so the 18 binaries that +# create the new cluster here match the 18 image that serves it afterwards. +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql-17-cron postgresql-18-cron \ + && rm -rf /var/lib/apt/lists/*