From 36240fcdf4b72847883a4c1c1b6a99ae68a60847 Mon Sep 17 00:00:00 2001 From: Nat Budin Date: Sat, 31 Oct 2026 12:10:44 -0700 Subject: [PATCH 1/6] Carry the framework-mode frontend in the Rails image (part of #12260) The Dockerfile builds the frontend in its own stage and the Rails image keeps the pristine build. bin/with_static_shell, which the entrypoint now runs (inside chamber, because ASSETS_HOST comes from there), makes the copy Rails serves with ASSETS_HOST substituted in, puts the hashed files in public/assets, and sets STATIC_SHELL_DIR. Rails can then serve the whole frontend without nginx or a CDN in front of it. The middleware that kept the unhashed data-mode entry files uncached, and the list it read, go with the build that needed them. Co-Authored-By: Claude Sonnet 5.5 --- .dockerignore | 3 ++ Dockerfile | 53 +++++++++++++++++++ bin/entrypoint.sh | 5 +- bin/with_static_shell | 18 +++++++ config/environments/production.rb | 7 --- config/nocache-files.json | 5 -- .../disable_caching_for_specific_assets.rb | 24 --------- script/prepareStaticShell.sh | 10 ++-- 8 files changed, 83 insertions(+), 42 deletions(-) create mode 100755 bin/with_static_shell delete mode 100644 config/nocache-files.json delete mode 100644 lib/intercode/disable_caching_for_specific_assets.rb diff --git a/.dockerignore b/.dockerignore index 6be17a2d261..757872ee0bb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -55,6 +55,9 @@ cookbooks /node_modules # /public/packs /public/packs-test +# The frontend is built inside the image, never taken from a build on the host +/build +/public/assets /node_modules # Stuff that's sometimes lying around in dev machines diff --git a/Dockerfile b/Dockerfile index 8553aeebd0c..eb17fa93bce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,13 @@ ARG RUBY_VERSION=4 ARG NODE_VERSION=24.11.1 ARG CHAMBER_VERSION=3.1.5 +# Optional, for the assets target: an earlier intercode-assets image (for example the last release's) whose hashed files +# are carried over, so that tabs and cached pages from the previous release can still load their chunks. The default, +# an empty image, carries nothing over. +ARG PREVIOUS_ASSETS_IMAGE=scratch +# Carried-over files are pruned this many days after the release that dropped them. Files that are part of this +# release are never pruned, however old. +ARG PREVIOUS_ASSETS_RETENTION_DAYS=30 ### dev @@ -30,6 +37,48 @@ RUN bundle config set without 'development test' \ COPY --chown=www:www . /usr/src/intercode +### frontend (the framework-mode build, which the Rails app serves as its shell and the assets image serves from nginx) + +FROM node:${NODE_VERSION}-slim AS frontend +ARG REVISION +ENV REVISION=${REVISION} +# husky, which wants a git repository +ENV HUSKY=0 +WORKDIR /usr/src/intercode +RUN apt-get update && apt-get install -y --no-install-recommends python3 build-essential && rm -rf /var/lib/apt/lists/* +COPY package.json yarn.lock .yarnrc.yml ./ +COPY .yarn .yarn +COPY doc-site/package.json doc-site/package.json +RUN yarn install --immutable +COPY . . +RUN yarn run build:framework + +### assets (docker build --target assets): the frontend served by nginx, as the intercode-assets image +# +# The build is made against a placeholder asset host. When the container starts it takes a fresh copy of the build and +# swaps in $ASSETS_HOST (or nothing, for same-origin assets), so one image serves any deployment. The app container +# needs the same ASSETS_HOST. With RAILS_UPSTREAM set, the container also proxies the Rails routes and crawlers to Rails. +# See https://intercode.interactiveliterature.org/docs/deploying + +FROM ${PREVIOUS_ASSETS_IMAGE} AS previous + +FROM nginx AS assets +ARG PREVIOUS_ASSETS_RETENTION_DAYS +# The pristine build. The entrypoint copies it into the web root on every start, before substituting ASSETS_HOST. +COPY --from=frontend /usr/src/intercode/build/framework/client /usr/share/intercode-assets +RUN rm /usr/share/nginx/html/index.* +RUN --mount=type=bind,from=previous,target=/previous \ + --mount=type=bind,source=script/mergePreviousAssets.sh,target=/mergePreviousAssets.sh \ + /mergePreviousAssets.sh /previous ${PREVIOUS_ASSETS_RETENTION_DAYS} /usr/share/intercode-assets assets +COPY script/substituteAssetsHost.sh /usr/local/bin/substituteAssetsHost.sh +COPY docker-assets/40-substitute-assets-host.sh /docker-entrypoint.d/40-substitute-assets-host.sh +COPY docker-assets/nginx.conf /etc/nginx/conf.d/default.conf +# The reference front server configuration, used when RAILS_UPSTREAM is set. The crawler map and the Rails locations are +# generated from the same route list as the CloudFront module. +COPY docker-assets/front-proxy.conf.template /etc/nginx/intercode/front-proxy.conf.template +COPY docker-assets/proxy_params.conf /etc/nginx/proxy_params.conf +COPY terraform/modules/cloudfront_intercode/nginx/crawler-map.conf terraform/modules/cloudfront_intercode/nginx/rails-locations.conf /etc/nginx/intercode/ + ### build FROM dev AS build @@ -41,6 +90,10 @@ RUN bundle exec bootsnap precompile --gemfile app/ lib/ RUN DATABASE_URL=postgresql://fakehost/not_a_real_database AWS_S3_BUCKET=fakebucket AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy bundle exec rails assets:precompile RUN rm -r doc-site +# The pristine framework-mode build. It is made against a placeholder asset host, because ASSETS_HOST is only known when +# the container starts; bin/with_static_shell makes the copy that Rails serves, with the real host substituted in. +COPY --from=frontend --chown=www:www /usr/src/intercode/build/framework/client /usr/src/intercode/build/framework/client + ### ld_preload trickery FROM ruby:${RUBY_VERSION}-slim AS amd64_jemalloc diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 0bac20662e6..9b03289f780 100755 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -45,7 +45,8 @@ process.stdin.on('end', () => { export AWS_ROLE_SESSION_NAME="chamber" unset CHAMBER_AWS_ROLE_ARN fi - drop_to_www chamber exec "$CHAMBER_SERVICE" -- "$@" + # with_static_shell goes inside chamber, because it needs ASSETS_HOST, which chamber sets + drop_to_www chamber exec "$CHAMBER_SERVICE" -- /usr/src/intercode/bin/with_static_shell "$@" else - drop_to_www "$@" + drop_to_www /usr/src/intercode/bin/with_static_shell "$@" fi diff --git a/bin/with_static_shell b/bin/with_static_shell new file mode 100755 index 00000000000..24236c6a6c5 --- /dev/null +++ b/bin/with_static_shell @@ -0,0 +1,18 @@ +#!/bin/sh +# Runs the command it is given with STATIC_SHELL_DIR set, so Rails serves the framework-mode build the image carries +# (see StaticSpaShell). The build is made against a placeholder asset host; this makes the copy with ASSETS_HOST +# substituted in (or same-origin assets when it is empty), and puts the hashed files where Rails serves them. It runs +# after chamber has set ASSETS_HOST, which is why bin/entrypoint.sh calls it from inside `chamber exec`. +# +# Does nothing when STATIC_SHELL_DIR is already set (a deployment that supplies its own shell), or when the image has no +# build, as in development. +set -eu + +root="$(cd "$(dirname "$0")/.." && pwd)" + +if [ -z "${STATIC_SHELL_DIR:-}" ] && [ -f "$root/build/framework/client/index.html" ]; then + STATIC_SHELL_DIR="$("$root/script/prepareStaticShell.sh" "$root/build/framework/client" "$root/tmp/static-shell" "${ASSETS_HOST:-}" | sed -n 's/^STATIC_SHELL_DIR=//p')" + export STATIC_SHELL_DIR +fi + +exec "$@" diff --git a/config/environments/production.rb b/config/environments/production.rb index 0d4e74786d3..cf59457edcd 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -require File.expand_path("lib/intercode/disable_caching_for_specific_assets", Rails.root) Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -76,12 +75,6 @@ "Expires" => 1.year.from_now.to_fs(:rfc822) } - if ENV["RAILS_SERVE_STATIC_FILES"].present? - config.middleware.insert_before ActionDispatch::Static, - Intercode::DisableCachingForSpecificAssets, - JSON.parse(File.read(File.expand_path("config/nocache-files.json", Rails.root))) - end - # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). config.i18n.fallbacks = true diff --git a/config/nocache-files.json b/config/nocache-files.json deleted file mode 100644 index 53ed17c5cf5..00000000000 --- a/config/nocache-files.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - "/packs/application.js", - "/packs/application-styles.js" -] - diff --git a/lib/intercode/disable_caching_for_specific_assets.rb b/lib/intercode/disable_caching_for_specific_assets.rb deleted file mode 100644 index 6a257be5d74..00000000000 --- a/lib/intercode/disable_caching_for_specific_assets.rb +++ /dev/null @@ -1,24 +0,0 @@ -# adapted from https://stackoverflow.com/a/52848885 -module Intercode - class DisableCachingForSpecificAssets - attr_reader :app, :asset_paths - - def initialize(app, asset_paths) - @app = app - @asset_paths = Set.new(asset_paths) - end - - def call(env) - # Let the next middleware classes & app do their thing first… - status, headers, response = app.call(env) - - # …and modify the response if a cache-disabled asset was fetched. - if asset_paths.include?(env['REQUEST_PATH']) - headers['Cache-Control'] = 'no-cache' - headers.except!('Expires') - end - - [status, headers, response] - end - end -end diff --git a/script/prepareStaticShell.sh b/script/prepareStaticShell.sh index 77a76c3a9d1..39c8917c998 100755 --- a/script/prepareStaticShell.sh +++ b/script/prepareStaticShell.sh @@ -4,9 +4,10 @@ # where Rails serves them. Run it again after every build; it replaces what it made before, including all of # public/assets, which nothing else uses. # -# Usage: prepareStaticShell.sh [build-dir] [shell-dir] -# build-dir the build's client directory (default build/framework/client) -# shell-dir where to put the shell (default tmp/static-shell) +# Usage: prepareStaticShell.sh [build-dir] [shell-dir] [assets-host] +# build-dir the build's client directory (default build/framework/client) +# shell-dir where to put the shell (default tmp/static-shell) +# assets-host where browsers load the hashed files from, as in ASSETS_HOST (default none: the app's own domain) # # Then start Rails (or run the system tests) with STATIC_SHELL_DIR set to the shell directory, which this prints. set -eu @@ -14,6 +15,7 @@ set -eu root="$(cd "$(dirname "$0")/.." && pwd)" build="${1:-$root/build/framework/client}" shell="${2:-$root/tmp/static-shell}" +assets_host="${3:-}" if [ ! -f "$build/index.html" ]; then echo "prepareStaticShell.sh: no $build/index.html; run yarn build:framework first" >&2 @@ -23,7 +25,7 @@ fi rm -rf "$shell" mkdir -p "$shell" cp -a "$build/." "$shell/" -"$root/script/substituteAssetsHost.sh" "$shell" +"$root/script/substituteAssetsHost.sh" "$shell" "$assets_host" rm -rf "$root/public/assets" mkdir -p "$root/public/assets" From 7d6d131d2fac22b8bc81f174f76a433d8a1df3a7 Mon Sep 17 00:00:00 2001 From: Nat Budin Date: Sat, 31 Oct 2026 12:10:54 -0700 Subject: [PATCH 2/6] Rename the framework-mode assets image to intercode-assets (part of #12260) The nginx image is now the assets stage of the main Dockerfile, built from the same frontend stage as the Rails image so the two always carry the same build, and it replaces the data-mode assets image under the same name. CI builds it on every push (on main it layers on the last release's image, so the previous release's files stay available for the retention period), smoke-tests both images with and without ASSETS_HOST, and checks that Rails serves the hashed files. The release tags one image instead of two. The assets bucket workflow now uploads the framework build's assets/ (with ASSETS_HOST substituted in) instead of packs/, and, in its publish phase, marks the files under packs/ obsolete so that tabs open on an 8.x release can load them until the bucket's retention runs out. The bundle size and GraphQL document checks read the framework build, and CI keeps checking the generated routes in the TypeScript job. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 215 ++++++++---------- .github/workflows/release.yml | 10 - .github/workflows/static.yml | 2 +- .github/workflows/upload-assets-to-bucket.yml | 30 ++- .../upload-framework-assets-to-bucket.yml | 18 +- Dockerfile.assets | 39 ---- Dockerfile.assets-framework | 44 ---- .../40-substitute-assets-host.sh | 0 .../front-proxy.conf.template | 0 .../nginx.conf | 0 .../proxy_params.conf | 0 nginx-assets.conf | 54 ----- script/genNginxNocache.js | 9 - script/mergePreviousAssets.sh | 6 +- script/uploadFrameworkAssets.sh | 12 +- script/uploadReleaseAssets.sh | 43 ---- scripts/check-bundle-size.mjs | 48 ++-- scripts/check-graphql-documents.mts | 10 +- 18 files changed, 158 insertions(+), 382 deletions(-) delete mode 100644 Dockerfile.assets delete mode 100644 Dockerfile.assets-framework rename {docker-assets-framework => docker-assets}/40-substitute-assets-host.sh (100%) rename {docker-assets-framework => docker-assets}/front-proxy.conf.template (100%) rename {docker-assets-framework => docker-assets}/nginx.conf (100%) rename {docker-assets-framework => docker-assets}/proxy_params.conf (100%) delete mode 100644 nginx-assets.conf delete mode 100644 script/genNginxNocache.js delete mode 100755 script/uploadReleaseAssets.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 049be72b047..ec7c58ab533 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: run: yarn run graphql:codegen - name: tsc on resulting generated files run: yarn run ts:check + # The framework build's routes are generated from AppRouter.tsx (see script/genRoutes.mts) until that is deleted + - name: Check that the generated routes are up to date + run: yarn run routes:check vitest: name: Vitest runs-on: ubuntu-latest @@ -53,8 +56,8 @@ jobs: uses: mikepenz/action-junit-report@v6 if: always() # always run even if the previous step fails with: - check_name: "Vitest Report" - report_paths: "test/reports/TEST-*.xml" + check_name: 'Vitest Report' + report_paths: 'test/reports/TEST-*.xml' detailed_summary: true skip_success_summary: true - name: Archive HTML test reports @@ -69,84 +72,6 @@ jobs: with: name: vitest-coverage path: coverage/* - # The framework-mode build (react-router.config.ts) while the migration is in progress. Off unless the repository - # variable BUILD_FRAMEWORK is set, and not required by anything else. - build-framework: - name: Framework mode build - if: vars.BUILD_FRAMEWORK != '' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v7 - - name: Read .node-version - id: node-version - run: echo "node-version=$(cat .node-version)" >> $GITHUB_OUTPUT - - name: install node - uses: actions/setup-node@v6 - with: - cache: yarn - node-version: ${{ steps.node-version.outputs.node-version }} - - name: yarn install - run: yarn install - - name: Check that the generated routes are up to date - run: yarn run routes:check - - name: Build - run: yarn run build:framework - - name: Check the output - run: | - test -f build/framework/client/index.html - ls build/framework/client/assets/*.js > /dev/null - - name: Build the assets image - run: > - docker build -f Dockerfile.assets-framework -t intercode-assets-framework:ci - --build-arg REVISION=${{ github.sha }} . - - name: Push the assets image (on main), for the release upload to the static shell bucket - if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - docker tag intercode-assets-framework:ci ghcr.io/neinteractiveliterature/intercode-assets-framework:${{ github.sha }} - docker push ghcr.io/neinteractiveliterature/intercode-assets-framework:${{ github.sha }} - - name: Check the assets image with and without ASSETS_HOST - run: | - docker run -d --name with-host -e ASSETS_HOST=assets.example.com:3135 -p 8081:80 intercode-assets-framework:ci - docker run -d --name without-host -p 8082:80 intercode-assets-framework:ci - sleep 5 - curl -sf http://localhost:8081/index.html | grep -q 'href="https://assets.example.com:3135/assets/' - curl -sf http://localhost:8082/index.html | grep -q 'href="/assets/' - for port in 8081 8082; do - if curl -sf http://localhost:$port/index.html | grep -q ASSETHOSTPLACEHOLDER; then - echo "placeholder left in index.html on port $port" - exit 1 - fi - manifest=$(curl -sf http://localhost:$port/index.html | grep -o '/assets/manifest-[a-z0-9]*\.js' | head -1) - curl -sf http://localhost:$port$manifest | grep -q __reactRouterManifest - done - - name: Check the assets image as a front server (RAILS_UPSTREAM) - run: | - cat > stub-rails.conf <<'EOF' - server { - listen 80; - location / { - default_type text/plain; - return 200 "RAILS uri=$request_uri host=$http_host proto=$http_x_forwarded_proto\n"; - } - } - EOF - docker network create front - docker run -d --name stub-rails --network front -v "$PWD/stub-rails.conf:/etc/nginx/conf.d/default.conf:ro" nginx - docker run -d --name front --network front -e RAILS_UPSTREAM=stub-rails:80 -p 8083:80 intercode-assets-framework:ci - sleep 5 - # Browsers get the static shell, for any path that isn't a Rails route - curl -sf http://localhost:8083/events/4130 -A 'Mozilla/5.0' | grep -q 'rel="modulepreload"' - # Crawlers get Rails' Open Graph document for the path, with the original Host - curl -sf "http://localhost:8083/events/a%20b/c?x=1" -A 'facebookexternalhit/1.1' -H 'Host: con.example.org' | grep -q 'uri=/og-shell?path=/events/a%20b/c host=con.example.org' - # The Rails routes are proxied - curl -sf -X POST http://localhost:8083/graphql | grep -q 'uri=/graphql' - curl -sf http://localhost:8083/oauth_session/refresh -X POST | grep -q 'uri=/oauth_session/refresh' - # A missing asset is a 404, not the shell - test "$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8083/assets/nope.js)" = 404 bundle-size: name: Bundle size check runs-on: ubuntu-latest @@ -163,7 +88,7 @@ jobs: - name: yarn install run: yarn install - name: Build frontend - run: yarn run build:web + run: yarn run build:framework - name: Check bundle size run: yarn run check:bundle-size minitest: @@ -208,8 +133,8 @@ jobs: uses: mikepenz/action-junit-report@v6 if: always() # always run even if the previous step fails with: - check_name: "Minitest Report" - report_paths: "test/reports/TEST-*.xml" + check_name: 'Minitest Report' + report_paths: 'test/reports/TEST-*.xml' detailed_summary: true skip_success_summary: true - name: Archive HTML test reports @@ -267,28 +192,20 @@ jobs: - name: yarn install run: yarn install - name: build - run: yarn run build + run: yarn run build:framework && yarn run build:cli + # Rails serves the framework-mode build as its static shell, as it does in production + - name: Prepare the static shell + run: script/prepareStaticShell.sh | tee -a "$GITHUB_ENV" - name: Database setup run: bundle exec rails db:create db:migrate - name: Run tests run: TERM=xterm-color bundle exec rails test:system - # The framework-mode build, with Rails serving its static shell, until it replaces the build above. Its routes are - # generated from AppRouter.tsx (see script/genRoutes.mts), so all the system tests should pass against it too. - - name: Build framework mode - if: vars.BUILD_FRAMEWORK != '' - run: yarn run build:framework - - name: Prepare the static shell - if: vars.BUILD_FRAMEWORK != '' - run: script/prepareStaticShell.sh | tee -a "$GITHUB_ENV" - - name: Run system tests against the framework-mode shell - if: vars.BUILD_FRAMEWORK != '' - run: TERM=xterm-color bundle exec rails test:system - name: Publish Test Report uses: mikepenz/action-junit-report@v6 if: always() # always run even if the previous step fails with: - check_name: "Minitest System Test Report" - report_paths: "test/reports/TEST-*.xml" + check_name: 'Minitest System Test Report' + report_paths: 'test/reports/TEST-*.xml' detailed_summary: true skip_success_summary: true - name: Archive HTML test reports @@ -318,23 +235,15 @@ jobs: - name: Read .node-version id: node-version run: echo "node-version=$(cat .node-version)" >> $GITHUB_OUTPUT - - name: install node - uses: actions/setup-node@v6 - with: - cache: yarn - node-version: ${{ steps.node-version.outputs.node-version }} - - name: yarn install - run: yarn install - name: Log in to registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Build frontend bundle - run: yarn run build - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master - name: Read .ruby-version id: ruby-version run: echo "ruby-version=$(cat .ruby-version)" >> $GITHUB_OUTPUT + # The frontend (yarn build:framework) is built inside the image, in a stage that both images below share - name: Build (and push to registry, if on main) uses: docker/build-push-action@v7 with: @@ -356,22 +265,82 @@ jobs: -e AWS_S3_BUCKET=fakebucket ghcr.io/neinteractiveliterature/intercode:${{ github.sha }} bin/rails runner 'puts "Hello, world!"' - - name: Build assets only (and push to registry, if on main) - uses: docker/build-push-action@v7 + - name: Check that the app image carries the frontend, with and without ASSETS_HOST + run: | + app=ghcr.io/neinteractiveliterature/intercode:${{ github.sha }} + docker run --rm -e DATABASE_URL=postgresql://fakehost/not_a_real_database -e AWS_S3_BUCKET=fakebucket \ + -e ASSETS_HOST=assets.example.com:3135 $app sh -c ' + grep -q "href=\"https://assets.example.com:3135/assets/" "$STATIC_SHELL_DIR/index.html" && + ! grep -rq ASSETHOSTPLACEHOLDER "$STATIC_SHELL_DIR" public/assets' + docker run --rm -e DATABASE_URL=postgresql://fakehost/not_a_real_database -e AWS_S3_BUCKET=fakebucket $app sh -c ' + grep -q "href=\"/assets/" "$STATIC_SHELL_DIR/index.html" && + ! grep -rq ASSETHOSTPLACEHOLDER "$STATIC_SHELL_DIR" public/assets && + ls public/assets/*.js > /dev/null' + - name: Check that Rails serves the frontend's files + run: | + docker run -d --name rails-serving -p 3000:3000 -e RAILS_SERVE_STATIC_FILES=enabled -e INTERCODE_HOST=localhost \ + -e DATABASE_URL=postgresql://fakehost/not_a_real_database -e AWS_S3_BUCKET=fakebucket \ + ghcr.io/neinteractiveliterature/intercode:${{ github.sha }} + file=$(docker exec rails-serving sh -c 'cd public/assets && ls entry-*.css | head -1') + for i in $(seq 1 30); do + curl -sf -o /dev/null -H 'X-Forwarded-Proto: https' http://localhost:3000/assets/$file && break + sleep 2 + done + curl -sfD - -o /dev/null -H 'X-Forwarded-Proto: https' http://localhost:3000/assets/$file | grep -i 'access-control-allow-origin: \*' + # The nginx image with the same build. On main it layers on the last released image, so that the previous + # release's files stay available for tabs that are still open (see the assets stage of the Dockerfile); latest is + # only retagged when a release is published. + - name: Build the assets image + run: > + docker buildx build --load --target assets -t intercode-assets:ci + --build-arg NODE_VERSION=${{ steps.node-version.outputs.node-version }} + --build-arg REVISION=${{ github.sha }} + ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' && '--build-arg PREVIOUS_ASSETS_IMAGE=ghcr.io/neinteractiveliterature/intercode-assets:latest' || '' }} + . + - name: Check the assets image with and without ASSETS_HOST + run: | + docker run -d --name with-host -e ASSETS_HOST=assets.example.com:3135 -p 8081:80 intercode-assets:ci + docker run -d --name without-host -p 8082:80 intercode-assets:ci + sleep 5 + curl -sf http://localhost:8081/index.html | grep -q 'href="https://assets.example.com:3135/assets/' + curl -sf http://localhost:8082/index.html | grep -q 'href="/assets/' + for port in 8081 8082; do + if curl -sf http://localhost:$port/index.html | grep -q ASSETHOSTPLACEHOLDER; then + echo "placeholder left in index.html on port $port" + exit 1 + fi + manifest=$(curl -sf http://localhost:$port/index.html | grep -o '/assets/manifest-[a-z0-9]*\.js' | head -1) + curl -sf http://localhost:$port$manifest | grep -q __reactRouterManifest + done + - name: Check the assets image as a front server (RAILS_UPSTREAM) + run: | + cat > stub-rails.conf <<'EOF' + server { + listen 80; + location / { + default_type text/plain; + return 200 "RAILS uri=$request_uri host=$http_host proto=$http_x_forwarded_proto\n"; + } + } + EOF + docker network create front + docker run -d --name stub-rails --network front -v "$PWD/stub-rails.conf:/etc/nginx/conf.d/default.conf:ro" nginx + docker run -d --name front --network front -e RAILS_UPSTREAM=stub-rails:80 -p 8083:80 intercode-assets:ci + sleep 5 + # Browsers get the static shell, for any path that isn't a Rails route + curl -sf http://localhost:8083/events/4130 -A 'Mozilla/5.0' | grep -q 'rel="modulepreload"' + # Crawlers get Rails' Open Graph document for the path, with the original Host + curl -sf "http://localhost:8083/events/a%20b/c?x=1" -A 'facebookexternalhit/1.1' -H 'Host: con.example.org' | grep -q 'uri=/og-shell?path=/events/a%20b/c host=con.example.org' + # The Rails routes are proxied + curl -sf -X POST http://localhost:8083/graphql | grep -q 'uri=/graphql' + curl -sf http://localhost:8083/oauth_session/refresh -X POST | grep -q 'uri=/oauth_session/refresh' + # A missing asset is a 404, not the shell + test "$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8083/assets/nope.js)" = 404 + - name: Push the assets image (on main) if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' - with: - context: . - file: Dockerfile.assets - push: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' }} - platforms: linux/amd64 - tags: | - ghcr.io/neinteractiveliterature/intercode-assets:${{ github.sha }} - # Layer on the last released assets image, so the previous release's files stay available (see - # Dockerfile.assets). latest is only retagged when a release is published. - build-args: | - BASEIMAGE=ghcr.io/neinteractiveliterature/intercode:${{ github.sha }} - PREVIOUS_ASSETS_IMAGE=ghcr.io/neinteractiveliterature/intercode-assets:latest - PREVIOUS_ASSETS_RETENTION_DAYS=30 + run: | + docker tag intercode-assets:ci ghcr.io/neinteractiveliterature/intercode-assets:${{ github.sha }} + docker push ghcr.io/neinteractiveliterature/intercode-assets:${{ github.sha }} doc-site: name: Build doc site runs-on: ubuntu-latest @@ -453,9 +422,9 @@ jobs: id: code_coverage_report_action if: ${{ github.actor != 'dependabot[bot]'}} with: - filename: "merged-coverage.xml" + filename: 'merged-coverage.xml' fail_on_negative_difference: false - artifact_download_workflow_names: "Continuous integration" + artifact_download_workflow_names: 'Continuous integration' only_list_changed_files: true - name: Add Coverage PR Comment if: steps.code_coverage_report_action.outputs.file != '' && github.event_name == 'pull_request' && (success() || failure()) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef6efdd872c..d237fc6eb18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,16 +28,6 @@ jobs: run: docker tag ghcr.io/neinteractiveliterature/intercode-assets:${{ github.sha }} ghcr.io/neinteractiveliterature/intercode-assets:latest - name: Push to registry run: docker push ghcr.io/neinteractiveliterature/intercode-assets:latest && docker push ghcr.io/neinteractiveliterature/intercode-assets:${{ github.event.release.name }} - # And for the framework-mode assets image, which is only built when BUILD_FRAMEWORK is set (see ci.yml) - - name: Tag and push the framework-mode assets image - if: vars.BUILD_FRAMEWORK != '' - run: | - image=ghcr.io/neinteractiveliterature/intercode-assets-framework - docker pull $image:${{ github.sha }} - docker tag $image:${{ github.sha }} $image:${{ github.event.release.name }} - docker tag $image:${{ github.sha }} $image:latest - docker push $image:latest - docker push $image:${{ github.event.release.name }} # Keeps each release's built assets in an S3 bucket, without deleting earlier releases' files, so that stale # tabs and cached pages can still load their chunks. Skipped unless ASSETS_BUCKET is set; see # terraform/modules/assets_bucket_intercode and upload-assets-to-bucket.yml. diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index 9ded990e51d..d1850f3e4fb 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -43,7 +43,7 @@ jobs: with: cache: yarn - run: yarn install - - run: yarn build:web + - run: yarn build:framework - run: yarn check-graphql-documents eslint: name: eslint diff --git a/.github/workflows/upload-assets-to-bucket.yml b/.github/workflows/upload-assets-to-bucket.yml index 4d470fba4ff..bcb7ab74537 100644 --- a/.github/workflows/upload-assets-to-bucket.yml +++ b/.github/workflows/upload-assets-to-bucket.yml @@ -1,7 +1,7 @@ name: Upload assets to bucket -# Uploads a commit's built assets (from its intercode-assets image) to the assets bucket, and marks the files that -# commit's release dropped as obsolete. See terraform/modules/assets_bucket_intercode. Called by the release workflow, +# Uploads a commit's built frontend (from its intercode-assets image) to the assets bucket, which is what ASSETS_HOST +# serves to the pages that Rails serves, and marks the files that commit's release dropped as obsolete. See terraform/modules/assets_bucket_intercode. Called by the release workflow, # and can be run by hand to upload an already-built commit. # # The role it assumes only trusts the refs listed in the module's github_oidc_subjects (release tags by default), so a @@ -45,8 +45,8 @@ jobs: steps: - name: Check that the bucket is configured run: | - if [ -z "${{ vars.ASSETS_BUCKET }}" ] || [ -z "${{ vars.ASSETS_UPLOAD_ROLE_ARN }}" ]; then - echo "ASSETS_BUCKET and ASSETS_UPLOAD_ROLE_ARN must be set as repository variables" >&2 + if [ -z "${{ vars.ASSETS_BUCKET }}" ] || [ -z "${{ vars.ASSETS_UPLOAD_ROLE_ARN }}" ] || [ -z "${{ vars.ASSETS_HOST }}" ]; then + echo "ASSETS_BUCKET, ASSETS_UPLOAD_ROLE_ARN and ASSETS_HOST must be set as repository variables" >&2 exit 1 fi - name: Check the commit @@ -58,16 +58,20 @@ jobs: - uses: actions/checkout@v7 - name: Log in to registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Extract assets from image + - name: Extract the build from the image run: | docker pull ghcr.io/neinteractiveliterature/intercode-assets:$ASSETS_SHA container=$(docker create ghcr.io/neinteractiveliterature/intercode-assets:$ASSETS_SHA) - docker cp "$container:/usr/share/nginx/html/packs" packs + docker cp "$container:/usr/share/intercode-assets" build docker cp "$container:/etc/intercode-assets/current-files.txt" current-files.txt docker rm "$container" - # The image also carries over earlier releases' files (see Dockerfile.assets). Only this release's own - # files count as current, so the rest can be marked obsolete in the bucket. - (cd packs && find . -type f | sort) | comm -23 - current-files.txt | (cd packs && xargs -r rm) + # The image also carries over earlier releases' files (see the assets stage of the Dockerfile). Only this + # release's own files count as current, so the rest can be marked obsolete in the bucket. + (cd build/assets && find . -type f | sort) | comm -23 - current-files.txt | (cd build/assets && xargs -r rm) + - name: Point the build at the assets host + # The build is made against a placeholder asset host. Pages that Rails serves load these files from + # ASSETS_HOST, which is the value the app containers have, so the URLs have to name it. + run: script/substituteAssetsHost.sh build "${{ vars.ASSETS_HOST }}" - name: Install boto3 (used to mark obsolete files) run: python3 -m pip install --user 'boto3>=1.35,<2' - uses: aws-actions/configure-aws-credentials@v6 @@ -75,4 +79,10 @@ jobs: role-to-assume: ${{ vars.ASSETS_UPLOAD_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} - name: Upload the assets (all of them, or the phase asked for) - run: script/uploadReleaseAssets.sh "${{ vars.ASSETS_BUCKET }}" packs "$PHASE" + run: UPLOAD_INDEX=false script/uploadFrameworkAssets.sh "${{ vars.ASSETS_BUCKET }}" build "$PHASE" + # The bucket still holds the files of the releases before v9.0.0, under packs/. The frontend no longer has any, so + # all of them are obsolete: this starts their countdown to expiry (the bucket's retention period), and tabs open + # on an old release can load their files until then. It does nothing once they are all marked. + - name: Mark the files of earlier releases' packs/ obsolete + if: env.PHASE != 'hashed' + run: python3 script/mark_obsolete_assets.py "${{ vars.ASSETS_BUCKET }}" packs/ < /dev/null diff --git a/.github/workflows/upload-framework-assets-to-bucket.yml b/.github/workflows/upload-framework-assets-to-bucket.yml index eed6afb594d..c180603372d 100644 --- a/.github/workflows/upload-framework-assets-to-bucket.yml +++ b/.github/workflows/upload-framework-assets-to-bucket.yml @@ -1,10 +1,9 @@ name: Upload framework-mode build to bucket -# Uploads a commit's framework-mode build (from its intercode-assets-framework image) to the static shell bucket, and +# Uploads a commit's framework-mode build (from its intercode-assets image) to the static shell bucket, and # marks the files that commit's release dropped as obsolete. See terraform/modules/assets_bucket_intercode and the static # shell mode of terraform/modules/cloudfront_intercode. Called by the release workflow, and can be run by hand to upload -# an already-built commit. The image is only built (and pushed, on main) when the repository variable BUILD_FRAMEWORK -# is set; see the build-framework job in ci.yml. +# an already-built commit. # # The role it assumes only trusts the refs listed in the module's github_oidc_subjects (release tags by default), so a # manual run has to be started from a ref that role trusts. @@ -13,7 +12,7 @@ on: workflow_call: inputs: sha: - description: Commit whose intercode-assets-framework image to upload + description: Commit whose intercode-assets image to upload required: true type: string phase: @@ -24,7 +23,7 @@ on: workflow_dispatch: inputs: sha: - description: Commit whose intercode-assets-framework image to upload (full SHA; defaults to the commit this runs from) + description: Commit whose intercode-assets image to upload (full SHA; defaults to the commit this runs from) required: false type: string phase: @@ -62,15 +61,12 @@ jobs: run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: Extract the build from the image run: | - if ! docker pull ghcr.io/neinteractiveliterature/intercode-assets-framework:$ASSETS_SHA; then - echo "No intercode-assets-framework image for $ASSETS_SHA. It is only built when BUILD_FRAMEWORK is set." >&2 - exit 1 - fi - container=$(docker create ghcr.io/neinteractiveliterature/intercode-assets-framework:$ASSETS_SHA) + docker pull ghcr.io/neinteractiveliterature/intercode-assets:$ASSETS_SHA + container=$(docker create ghcr.io/neinteractiveliterature/intercode-assets:$ASSETS_SHA) docker cp "$container:/usr/share/intercode-assets" build docker cp "$container:/etc/intercode-assets/current-files.txt" current-files.txt docker rm "$container" - # The image also carries over earlier releases' files (see Dockerfile.assets-framework). Only this release's + # The image also carries over earlier releases' files (see the assets stage of the Dockerfile). Only this release's # own files count as current, so the rest can be marked obsolete in the bucket. (cd build/assets && find . -type f | sort) | comm -23 - current-files.txt | (cd build/assets && xargs -r rm) - name: Point the build at the same origin diff --git a/Dockerfile.assets b/Dockerfile.assets deleted file mode 100644 index 54bc9e7f214..00000000000 --- a/Dockerfile.assets +++ /dev/null @@ -1,39 +0,0 @@ -ARG BASEIMAGE -# Optional: an earlier intercode-assets image (e.g. ghcr.io/neinteractiveliterature/intercode-assets:latest) whose -# hashed files are carried over into this image, so that tabs and cached pages from the previous release can still -# load their chunks. The default, an empty image, carries nothing over and leaves this build unchanged. -ARG PREVIOUS_ASSETS_IMAGE=scratch -# Carried-over files are pruned this many days after the release that dropped them. Files that are part of this -# release are never pruned, however old. -ARG PREVIOUS_ASSETS_RETENTION_DAYS=30 -# -# Whoever builds this image passes these (for example --build-arg PREVIOUS_ASSETS_IMAGE=); nothing in this repository's workflows does. The merge step goes by file ages, which BuildKit's cache -# doesn't see, so don't expect --cache-from to be safe for rebuilding the same inputs at a later date. - -FROM ${PREVIOUS_ASSETS_IMAGE} as previous - -FROM ${BASEIMAGE} as code -# Make assets readable -USER root -RUN chown root.root -R /usr/src/intercode/public - -# Create an nginx snippet here, to keep final image -# as small as possible. -RUN node script/genNginxNocache.js > /usr/src/intercode/public/cache.conf - - # Copy into http server -FROM nginx -RUN rm /usr/share/nginx/html/index.* -COPY --from=code /usr/src/intercode/public /usr/share/nginx/html -ARG PREVIOUS_ASSETS_RETENTION_DAYS -# Carry over the previous image's files, and prune the ones that have been obsolete for too long. -RUN --mount=type=bind,from=previous,target=/previous \ - --mount=type=bind,source=script/mergePreviousAssets.sh,target=/mergePreviousAssets.sh \ - /mergePreviousAssets.sh /previous ${PREVIOUS_ASSETS_RETENTION_DAYS} -RUN touch /usr/share/nginx/html/index.html && \ - chmod 644 /usr/share/nginx/html/index.html && \ - chown www-data.root -R /usr/share/nginx/html && \ - mv /usr/share/nginx/html/cache.conf /etc/nginx/ && \ - mkdir -p /var/run/nginx -COPY nginx-assets.conf /etc/nginx/conf.d/default.conf diff --git a/Dockerfile.assets-framework b/Dockerfile.assets-framework deleted file mode 100644 index a58ab23e8bc..00000000000 --- a/Dockerfile.assets-framework +++ /dev/null @@ -1,44 +0,0 @@ -# The framework-mode (react-router.config.ts) assets image, built only when BUILD_FRAMEWORK is set in CI while the -# migration is in progress. It is a standalone build: it compiles the framework build itself instead of starting from -# the app image, because the app image doesn't contain it. -# -# The build is made against a placeholder asset host. When the container starts it takes a fresh copy of the build and -# swaps in $ASSETS_HOST (or nothing, for same-origin assets), so one image serves any deployment. The app container -# needs the same ASSETS_HOST. With RAILS_UPSTREAM set, the container also proxies the Rails routes and crawlers to Rails. -ARG NODE_VERSION=24.11.1 -# Optional: an earlier intercode-assets-framework image whose hashed files are carried over, as in Dockerfile.assets. -ARG PREVIOUS_ASSETS_IMAGE=scratch -ARG PREVIOUS_ASSETS_RETENTION_DAYS=30 - -FROM ${PREVIOUS_ASSETS_IMAGE} AS previous - -FROM node:${NODE_VERSION}-slim AS build -ARG REVISION -ENV REVISION=${REVISION} -# husky, which wants a git repository -ENV HUSKY=0 -WORKDIR /usr/src/intercode -RUN apt-get update && apt-get install -y --no-install-recommends python3 build-essential && rm -rf /var/lib/apt/lists/* -COPY package.json yarn.lock .yarnrc.yml ./ -COPY .yarn .yarn -COPY doc-site/package.json doc-site/package.json -RUN yarn install --immutable -COPY . . -RUN yarn run build:framework - -FROM nginx -ARG PREVIOUS_ASSETS_RETENTION_DAYS -# The pristine build. The entrypoint copies it into the web root on every start, before substituting ASSETS_HOST. -COPY --from=build /usr/src/intercode/build/framework/client /usr/share/intercode-assets -RUN rm /usr/share/nginx/html/index.* -RUN --mount=type=bind,from=previous,target=/previous \ - --mount=type=bind,source=script/mergePreviousAssets.sh,target=/mergePreviousAssets.sh \ - /mergePreviousAssets.sh /previous ${PREVIOUS_ASSETS_RETENTION_DAYS} /usr/share/intercode-assets assets -COPY script/substituteAssetsHost.sh /usr/local/bin/substituteAssetsHost.sh -COPY docker-assets-framework/40-substitute-assets-host.sh /docker-entrypoint.d/40-substitute-assets-host.sh -COPY docker-assets-framework/nginx.conf /etc/nginx/conf.d/default.conf -# The reference front server configuration, used when RAILS_UPSTREAM is set (see https://intercode.interactiveliterature.org/docs/deploying). The crawler map and -# the Rails locations are generated from the same route list as the CloudFront module. -COPY docker-assets-framework/front-proxy.conf.template /etc/nginx/intercode/front-proxy.conf.template -COPY docker-assets-framework/proxy_params.conf /etc/nginx/proxy_params.conf -COPY terraform/modules/cloudfront_intercode/nginx/crawler-map.conf terraform/modules/cloudfront_intercode/nginx/rails-locations.conf /etc/nginx/intercode/ diff --git a/docker-assets-framework/40-substitute-assets-host.sh b/docker-assets/40-substitute-assets-host.sh similarity index 100% rename from docker-assets-framework/40-substitute-assets-host.sh rename to docker-assets/40-substitute-assets-host.sh diff --git a/docker-assets-framework/front-proxy.conf.template b/docker-assets/front-proxy.conf.template similarity index 100% rename from docker-assets-framework/front-proxy.conf.template rename to docker-assets/front-proxy.conf.template diff --git a/docker-assets-framework/nginx.conf b/docker-assets/nginx.conf similarity index 100% rename from docker-assets-framework/nginx.conf rename to docker-assets/nginx.conf diff --git a/docker-assets-framework/proxy_params.conf b/docker-assets/proxy_params.conf similarity index 100% rename from docker-assets-framework/proxy_params.conf rename to docker-assets/proxy_params.conf diff --git a/nginx-assets.conf b/nginx-assets.conf deleted file mode 100644 index 7fe9a3ec938..00000000000 --- a/nginx-assets.conf +++ /dev/null @@ -1,54 +0,0 @@ - -server { - listen 80; - listen unix:/var/run/nginx/http_in.sock; - - server_name localhost; - - root /usr/share/nginx/html; - include /etc/nginx/cache.conf; - location / { - expires 180d; - ### Wide open CORS from the intertudes - https://michielkalkman.com/snippets/nginx-cors-open-configuration/ - if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*'; - # - # Cookie control - # - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - - # - # Custom headers and headers various browsers *should* be OK with but aren't - # - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; - - # - # Tell client that this pre-flight info is valid for 20 days - # - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain charset=UTF-8'; - add_header 'Content-Length' 0; - return 204; - } - if ($request_method = 'POST') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; - } - if ($request_method = 'GET') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; - } - if ($request_method = 'HEAD') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; - } - - } -} diff --git a/script/genNginxNocache.js b/script/genNginxNocache.js deleted file mode 100644 index a0bc8ce1850..00000000000 --- a/script/genNginxNocache.js +++ /dev/null @@ -1,9 +0,0 @@ -fs=require('fs'); - -for (const fname of JSON.parse(fs.readFileSync("config/nocache-files.json"))) { - process.stdout.write(`location ${fname} { - add_header "Cache-Control" "no-cache"; - add_header "Access-Control-Allow-Origin" "*"; - }\n` - ); -} diff --git a/script/mergePreviousAssets.sh b/script/mergePreviousAssets.sh index 61b519d4f06..efd3298b3f1 100755 --- a/script/mergePreviousAssets.sh +++ b/script/mergePreviousAssets.sh @@ -1,12 +1,12 @@ #!/bin/sh -# Run by Dockerfile.assets: carries the previous assets image's files over into this one, and prunes the ones that have +# Run by the assets stage of the Dockerfile: carries the previous assets image's files over into this one, and prunes the ones that have # been obsolete for longer than the retention period. A file becomes obsolete when a release stops containing it, so # its age is counted from the build that dropped it, not from when it was first built. # # Usage: mergePreviousAssets.sh [web-root] [subdirectory] # -# The web root and subdirectory default to the ones Dockerfile.assets uses (/usr/share/nginx/html and packs). -# Dockerfile.assets-framework passes its own, since its hashed files live in assets/ in a different directory. +# The web root and subdirectory default to /usr/share/nginx/html and packs, which the pre-9.0.0 assets image used. +# The assets stage of the Dockerfile passes its own, since its hashed files live in assets/ in a different directory. set -eu previous_root="$1" diff --git a/script/uploadFrameworkAssets.sh b/script/uploadFrameworkAssets.sh index 4d316c38887..452d1095a5f 100755 --- a/script/uploadFrameworkAssets.sh +++ b/script/uploadFrameworkAssets.sh @@ -12,6 +12,9 @@ # publish uploads index.html, which makes the new frontend live, then marks the dropped files obsolete # The default, all, runs both in that order. # +# The assets bucket, which only serves the hashed files to pages that Rails serves, has no use for index.html: set +# UPLOAD_INDEX=false and the publish phase only marks the obsolete files. +# # INDEX_MAX_AGE (seconds, default 60) is the Cache-Control of index.html; keep it in step with the distribution's # static_shell_ttl. The hashed files in assets/ never change, so they are cached for a long time. set -euo pipefail @@ -19,6 +22,7 @@ set -euo pipefail bucket="$1" build_dir="${2%/}" index_max_age="${INDEX_MAX_AGE:-60}" +upload_index="${UPLOAD_INDEX:-true}" phase="${3:-all}" case "$phase" in @@ -36,9 +40,11 @@ if [ "$phase" != publish ]; then fi if [ "$phase" != hashed ]; then - # index.html goes last, so that it never names a file that isn't in the bucket yet - aws s3 cp "$build_dir/index.html" "s3://$bucket/index.html" \ - --cache-control "public, max-age=$index_max_age" --content-type "text/html" + if [ "$upload_index" = true ]; then + # index.html goes last, so that it never names a file that isn't in the bucket yet + aws s3 cp "$build_dir/index.html" "s3://$bucket/index.html" \ + --cache-control "public, max-age=$index_max_age" --content-type "text/html" + fi (cd "$build_dir" && find assets -type f) | "$(dirname "$0")/mark_obsolete_assets.py" "$bucket" assets/ fi diff --git a/script/uploadReleaseAssets.sh b/script/uploadReleaseAssets.sh deleted file mode 100755 index b23dbd223b7..00000000000 --- a/script/uploadReleaseAssets.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Uploads a release's built packs to the assets bucket (see terraform/modules/assets_bucket_intercode), then marks the -# files that earlier releases left behind and this release no longer contains as obsolete, which starts the bucket's -# lifecycle rule counting down to their expiry. -# -# Usage: script/uploadReleaseAssets.sh [all|hashed|publish] -# -# The release workflow runs it in two phases, so that the new frontend only goes live once the new backend is: -# hashed uploads the hashed files, which nothing refers to until the entry files do, so this can run alongside the -# backend deploy -# publish uploads the unhashed entry files, which makes the new frontend live, then marks the dropped files obsolete -# The default, all, runs both in that order. -set -euo pipefail - -bucket="$1" -packs_dir="${2%/}" -phase="${3:-all}" -nocache_json="$(dirname "$0")/../config/nocache-files.json" - -case "$phase" in - all | hashed | publish) ;; - *) - echo "Unknown phase: $phase (expected all, hashed or publish)" >&2 - exit 1 - ;; -esac - -if [ "$phase" != publish ]; then - # Upload every file, not only new ones. Uploading replaces the object, which also clears an `obsolete` tag left on a - # file that has come back into the release. - aws s3 cp "$packs_dir" "s3://$bucket/packs" --recursive \ - --exclude "application.js" --exclude "application-styles.js" \ - --cache-control "public, max-age=15552000" -fi - -if [ "$phase" != hashed ]; then - # The unhashed entry files are replaced each release, so they must always be revalidated. - for file in $(jq -r '.[]' "$nocache_json"); do - aws s3 cp "$packs_dir/$(basename "$file")" "s3://$bucket$file" --cache-control "no-cache" - done - - (cd "$packs_dir" && find . -type f | sed 's#^\./#packs/#') | "$(dirname "$0")/mark_obsolete_assets.py" "$bucket" packs/ -fi diff --git a/scripts/check-bundle-size.mjs b/scripts/check-bundle-size.mjs index bc1cb90631f..372cda1ba28 100644 --- a/scripts/check-bundle-size.mjs +++ b/scripts/check-bundle-size.mjs @@ -1,12 +1,14 @@ #!/usr/bin/env node // Checks the gzip size of all JS/CSS assets in the initial page load against a -// budget. The entry file (application.js) embeds the full preload list in its -// __vite__mapDeps array, so we parse that instead of requiring a Vite manifest. +// budget. The initial page load is whatever the shell (index.html in the +// framework-mode build) preloads or links, so we read that. // // Without a large eagerly-loaded dependency like Monaco editor, the initial -// payload is well under 500 KB compressed. With Monaco in the initial bundle -// (the regression this guards against), it exceeds 900 KB. The threshold is +// payload is well under the budget. With Monaco in the initial bundle (the +// regression this guards against), it is far over. The threshold is // intentionally set between those two states with headroom on each side. +// +// Run after `yarn build:framework`. import { readFileSync } from 'fs'; import { createGzip } from 'zlib'; @@ -16,30 +18,31 @@ import { pipeline } from 'stream/promises'; import { Readable } from 'stream'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); -const PACKS_DIR = join(ROOT, 'public', 'packs'); -const ENTRY_FILE = join(PACKS_DIR, 'application.js'); +const BUILD_DIR = join(ROOT, 'build', 'framework', 'client'); +const INDEX_FILE = join(BUILD_DIR, 'index.html'); const THRESHOLD_BYTES = 700_000; -const entrySource = readFileSync(ENTRY_FILE, 'utf8'); +const indexSource = readFileSync(INDEX_FILE, 'utf8'); -// Extract asset paths from the __vite__mapDeps inline array in the entry file. -const assetPaths = [...entrySource.matchAll(/"(assets\/[^"]+)"/g)].map((m) => m[1]); +// The files the shell preloads, links as stylesheets or loads as scripts. Their URLs are built against a placeholder +// asset host (see vite.framework.config.mts), which stands for the root of the build directory. +const assetPaths = [ + ...new Set( + [...indexSource.matchAll(/<(?:link|script)\b[^>]*?\b(?:href|src)="[^"]*?\/(assets\/[^"#?]+)/g)].map((m) => m[1]), + ), +]; if (assetPaths.length === 0) { - console.error('ERROR: No asset paths found in application.js — has the build format changed?'); + console.error('ERROR: No asset paths found in index.html — has the build format changed?'); process.exit(1); } async function gzippedSize(filePath) { const content = readFileSync(filePath); let bytes = 0; - await pipeline( - Readable.from(content), - createGzip(), - async function* (source) { - for await (const chunk of source) bytes += chunk.length; - }, - ); + await pipeline(Readable.from(content), createGzip(), async function* (source) { + for await (const chunk of source) bytes += chunk.length; + }); return bytes; } @@ -53,16 +56,11 @@ const results = []; let totalBytes = 0; for (const asset of assetPaths) { - const size = await gzippedSize(join(PACKS_DIR, asset)); + const size = await gzippedSize(join(BUILD_DIR, asset)); totalBytes += size; results.push({ asset, size }); } -// Include the entry file itself (tiny, but complete the picture) -const entrySize = await gzippedSize(ENTRY_FILE); -totalBytes += entrySize; -results.push({ asset: 'application.js', size: entrySize }); - results.sort((a, b) => b.size - a.size); console.log(`Initial load files (${results.length} total):`); @@ -74,9 +72,7 @@ if (results.length > 15) console.log(` ... and ${results.length - 15} smaller f console.log(`\nTotal compressed: ${humanBytes(totalBytes)} (limit: ${humanBytes(THRESHOLD_BYTES)})`); if (totalBytes > THRESHOLD_BYTES) { - console.error( - `\nFAIL: Initial JS + CSS payload is ${humanBytes(totalBytes - THRESHOLD_BYTES)} over budget.`, - ); + console.error(`\nFAIL: Initial JS + CSS payload is ${humanBytes(totalBytes - THRESHOLD_BYTES)} over budget.`); console.error('A large dependency may have snuck into the initial bundle.'); process.exit(1); } else { diff --git a/scripts/check-graphql-documents.mts b/scripts/check-graphql-documents.mts index 68b6ab56946..dec21d2c460 100644 --- a/scripts/check-graphql-documents.mts +++ b/scripts/check-graphql-documents.mts @@ -1,6 +1,6 @@ /** * Verifies that GraphQL document constants from *.generated.ts files are present - * in the production build output. Run after `yarn build:web`. + * in the production build output. Run after `yarn build:framework`. * * GraphQL operation names (e.g. "AppRootQuery") are embedded as string literals * in the serialized document AST and survive minification, making them a reliable @@ -18,9 +18,7 @@ const KNOWN_ABSENT: ReadonlySet = new Set(); async function findFiles(dir: string, suffix: string): Promise { const entries = await readdir(dir, { withFileTypes: true, recursive: true }); - return entries - .filter((e) => e.isFile() && e.name.endsWith(suffix)) - .map((e) => join(e.parentPath ?? e.path, e.name)); + return entries.filter((e) => e.isFile() && e.name.endsWith(suffix)).map((e) => join(e.parentPath ?? e.path, e.name)); } async function main() { @@ -37,10 +35,10 @@ async function main() { } // Combine all built JS chunks into one searchable string. - const assetsDir = join(root, 'public/packs/assets'); + const assetsDir = join(root, 'build/framework/client/assets'); const builtFiles = await findFiles(assetsDir, '.js'); if (builtFiles.length === 0) { - console.error('No built files found in public/packs/assets/. Run yarn build:web first.'); + console.error('No built files found in build/framework/client/assets/. Run yarn build:framework first.'); process.exit(1); } const builtContent = (await Promise.all(builtFiles.map((f) => readFile(f, 'utf-8')))).join(''); From 732f61467991a900a0d05b2cd3edd28c38443ee5 Mon Sep 17 00:00:00 2001 From: Nat Budin Date: Sat, 31 Oct 2026 12:11:02 -0700 Subject: [PATCH 3/6] Take the print report styles from the framework-mode shell (part of #12260) The printable reports and the CMS layout preview loaded /packs/application-styles.js, which only the data-mode build makes. They now link the stylesheets the shell's index.html links, which StaticSpaShell reads. Without a shell (in development) they go unstyled, until the reports move into the SPA (#12322). Co-Authored-By: Claude Sonnet 5.5 --- app/presenters/cms_rendering_context.rb | 5 ++-- app/services/static_spa_shell.rb | 17 ++++++++++++ app/views/layouts/print_reports.html.erb | 4 ++- test/controllers/reports_controller_test.rb | 19 +++++++++++++ test/services/static_spa_shell_test.rb | 30 +++++++++++++++++++++ 5 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 test/services/static_spa_shell_test.rb diff --git a/app/presenters/cms_rendering_context.rb b/app/presenters/cms_rendering_context.rb index a9bb880daf3..3bb210ec487 100644 --- a/app/presenters/cms_rendering_context.rb +++ b/app/presenters/cms_rendering_context.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/ClassLength class CmsRenderingContext include Cadmus::RenderingHelper include Cadmus::Renderable @@ -120,9 +119,9 @@ def liquid_assigns_for_single_page_app(cms_layout) end def liquid_assigns_for_placeholder_template - styles_url = url_with_possible_host("/packs/application-styles.js", ENV.fetch("ASSETS_HOST", nil)) + stylesheet_links = StaticSpaShell.stylesheet_hrefs.map { |href| tag.link(rel: "stylesheet", href: href) } liquid_assigns.merge( - "content_for_head" => "#{javascript_include_tag styles_url, type: "module"}{{ content_for_head }}", + "content_for_head" => "#{stylesheet_links.join}{{ content_for_head }}", "content_for_navbar" => "{{ content_for_navbar }}", "content_for_layout" => "{{ content_for_layout }}" ) diff --git a/app/services/static_spa_shell.rb b/app/services/static_spa_shell.rb index 46ef6224d78..ca68a4d8ade 100644 --- a/app/services/static_spa_shell.rb +++ b/app/services/static_spa_shell.rb @@ -24,6 +24,23 @@ def index_path File.join(directory, "index.html") end + # The stylesheets the shell links, for the pages Rails renders itself (the printable reports, the CMS layout + # preview) that need the app's styles. Empty without a shell, as in development, where the pages go unstyled. + def stylesheet_hrefs + return [] unless enabled? + + @stylesheet_hrefs ||= {} + @stylesheet_hrefs[directory] ||= File + .read(index_path) + .scan(/]*>/) + .filter_map do |link| + next unless link.match?(/\brel="stylesheet"/) + + link[/\bhref="([^"]+)"/, 1] + end + .freeze + end + def crawler?(user_agent) CRAWLER_USER_AGENT_REGEX.match?(user_agent.to_s) end diff --git a/app/views/layouts/print_reports.html.erb b/app/views/layouts/print_reports.html.erb index 1399afb6f5f..ed88fbf2312 100644 --- a/app/views/layouts/print_reports.html.erb +++ b/app/views/layouts/print_reports.html.erb @@ -5,7 +5,9 @@ <%= page_title %> - <%= javascript_include_tag url_with_possible_host('/packs/application-styles.js', ENV['ASSETS_HOST']), type: 'module' %> + <% StaticSpaShell.stylesheet_hrefs.each do |href| %> + <%= tag.link rel: 'stylesheet', href: href %> + <% end %> diff --git a/test/controllers/reports_controller_test.rb b/test/controllers/reports_controller_test.rb index fe79c7860be..17ca4e79a9c 100644 --- a/test/controllers/reports_controller_test.rb +++ b/test/controllers/reports_controller_test.rb @@ -31,6 +31,25 @@ class ReportsControllerTest < ActionDispatch::IntegrationTest sign_in con_admin end + describe "styles" do + after { ENV.delete("STATIC_SHELL_DIR") } + + it "links the stylesheets of the static shell" do + Dir.mktmpdir do |dir| + File.write( + File.join(dir, "index.html"), + '' + ) + ENV["STATIC_SHELL_DIR"] = dir + + get reports_per_event_path + + assert_response :ok + assert_select 'link[rel="stylesheet"][href="https://assets.example.com/assets/entry-abc.css"]' + end + end + end + describe "GET per_event" do it "renders without error for an event with a waitlisted signup requesting a bucket" do dogs_bucket = bucket_with_key(event.registration_policy, "dogs") diff --git a/test/services/static_spa_shell_test.rb b/test/services/static_spa_shell_test.rb new file mode 100644 index 00000000000..70c925a768c --- /dev/null +++ b/test/services/static_spa_shell_test.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true +require "test_helper" + +class StaticSpaShellTest < ActiveSupport::TestCase + describe ".stylesheet_hrefs" do + after { ENV.delete("STATIC_SHELL_DIR") } + + it "is empty without a shell" do + ENV.delete("STATIC_SHELL_DIR") + + assert_equal [], StaticSpaShell.stylesheet_hrefs + end + + it "lists the stylesheets the shell links, and nothing else" do + Dir.mktmpdir do |dir| + File.write(File.join(dir, "index.html"), <<~HTML) + + + + + + HTML + ENV["STATIC_SHELL_DIR"] = dir + + assert_equal %w[https://assets.example.com/assets/entry-abc.css /assets/other-def.css#], + StaticSpaShell.stylesheet_hrefs + end + end + end +end From 83a1561931e5dd62457576b2284280e47613f546 Mon Sep 17 00:00:00 2001 From: Nat Budin Date: Sat, 31 Oct 2026 12:11:03 -0700 Subject: [PATCH 4/6] Document the 9.0.0 image layout for deployers (part of #12260) The deployment guide now says that Rails serves the frontend itself and nginx is optional, that intercode-assets is the new name of the framework-mode image (and what it was before 9.0.0), and that both containers use ASSETS_HOST to build what they serve. The staged rollout guide gets a note on what 9.0.0 changes for it. Co-Authored-By: Claude Sonnet 5.5 --- agent-docs/testing.md | 4 +- doc-site/docs/deploying/index.md | 96 ++++++++++--------- doc-site/docs/deploying/staged-rollout.md | 8 ++ .../deploying/docker-compose.example.yml | 8 +- .../modules/cloudfront_intercode/README.md | 2 +- 5 files changed, 68 insertions(+), 50 deletions(-) diff --git a/agent-docs/testing.md b/agent-docs/testing.md index 652e5b8a77d..235a254167d 100644 --- a/agent-docs/testing.md +++ b/agent-docs/testing.md @@ -28,7 +28,7 @@ PARALLEL_WORKERS=4 bin/rails test ## System tests against the framework-mode build -While the migration to React Router framework mode (#12219) is in progress, the system tests also run against the framework-mode build, in a real browser (Cuprite) with Rails serving its prebuilt shell (`StaticSpaShell`) when `STATIC_SHELL_DIR` is set. `test/system/framework_shell_test.rb` only runs in that mode and checks that the build boots (it uses the placeholder `/framework-check` page); the other system tests run in both. +The system tests run in a real browser (Cuprite), and CI runs them against the framework-mode build with Rails serving its prebuilt shell (`StaticSpaShell`) when `STATIC_SHELL_DIR` is set, which is how production runs. Locally they use the data-mode build unless you set that up (below). `test/system/framework_shell_test.rb` only runs in the framework mode and checks that the build boots (it uses the placeholder `/framework-check` page); the other system tests run in both. ```sh yarn build:framework @@ -36,7 +36,7 @@ script/prepareStaticShell.sh # prints STATIC_SHELL_DIR=...; also fills pu STATIC_SHELL_DIR=$PWD/tmp/static-shell bin/rails test:system ``` -Run `script/prepareStaticShell.sh` again after each build. CI does the same in the `Minitest system tests` job when the repository variable `BUILD_FRAMEWORK` is set. The framework build's routes are generated from `AppRouter.tsx` by `script/genRoutes.mts` (`yarn routes:generate`; CI runs `yarn routes:check`), so change routes in `AppRouter.tsx` and regenerate until the cutover (#12260) deletes both. +Run `script/prepareStaticShell.sh` again after each build. CI does the same in the `Minitest system tests` job. The framework build's routes are generated from `AppRouter.tsx` by `script/genRoutes.mts` (`yarn routes:generate`; CI runs `yarn routes:check`), so change routes in `AppRouter.tsx` and regenerate until the cutover (#12260) deletes both. ## End-to-End Testing with Playwright diff --git a/doc-site/docs/deploying/index.md b/doc-site/docs/deploying/index.md index 2fd38369432..9a969dfedab 100644 --- a/doc-site/docs/deploying/index.md +++ b/doc-site/docs/deploying/index.md @@ -4,8 +4,9 @@ sidebar_position: 1 # Deploying Intercode -This is for people running their own Intercode, with the published Docker images. It covers the framework-mode build -(the React Router single-page app that `yarn build:framework` makes) and the nginx configuration that serves it. +This is for people running their own Intercode, with the published Docker images. It covers the frontend (the React +Router single-page app that `yarn build:framework` makes), which the Rails image carries and serves itself, and the +nginx image that can serve it instead. [`docker-compose.example.yml`](pathname:///deploying/docker-compose.example.yml) (with its [`example.env`](pathname:///deploying/example.env)) is a complete working example. For CloudFront and S3 instead of nginx, see [terraform/modules/cloudfront_intercode](https://github.com/neinteractiveliterature/intercode/blob/main/terraform/modules/cloudfront_intercode/README.md). @@ -14,57 +15,65 @@ requests to the same places. ## What runs -| Piece | Image | What it does | -| ---------- | ------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| **Rails** | `ghcr.io/neinteractiveliterature/intercode` | The API (`/graphql`), sign-in, uploads, and the pages that need per-page metadata for link previews (`/og-shell`). It needs PostgreSQL and S3. | -| **web** | `ghcr.io/neinteractiveliterature/intercode-assets-framework` | nginx. Serves the static build (`index.html` and the hashed files in `assets/`), proxies Rails' routes to Rails, and sends crawlers to Rails. | -| PostgreSQL | `postgres` | The database. | +| Piece | Image | What it does | +| ---------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Rails** | `ghcr.io/neinteractiveliterature/intercode` | The API (`/graphql`), sign-in, uploads, and the pages that need per-page metadata for link previews (`/og-shell`). It also serves the frontend (`index.html` and the hashed files in `assets/`) itself. It needs PostgreSQL and S3. | +| **web** | `ghcr.io/neinteractiveliterature/intercode-assets` | Optional. nginx. Serves the same build, proxies Rails' routes to Rails, and sends crawlers to Rails. | +| PostgreSQL | `postgres` | The database. | -Visitors talk to **web**, never to Rails directly. Intercode only works over HTTPS (Rails forces SSL, and the refresh-token -cookie has a `__Host-` prefix), so put a TLS-terminating proxy or load balancer in front of web that sends -`X-Forwarded-Proto: https`. web passes that header on, and passes the original `Host` header on too, which is how Rails -picks the convention for a request. +**Rails alone is enough**: put a TLS-terminating proxy or load balancer in front of it. Intercode only works over HTTPS +(Rails forces SSL, and the refresh-token cookie has a `__Host-` prefix), so the proxy has to send `X-Forwarded-Proto: +https`, and has to pass the original `Host` header on, which is how Rails picks the convention for a request. + +Run **web** as well when you want nginx to serve the frontend's files, or to run the front server described below. The +example in this guide does. Visitors then talk to web, never to Rails directly, and web passes those headers on. ## Images and tags Every commit on `main` is built as a `:` tag. A GitHub release adds `:` (for example `:8.7.0`) and `:latest`. -| Image | Tags | Notes | -| ---------------------------- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `intercode` | `:`, `:`, `:latest` | The Rails app. | -| `intercode-assets-framework` | `:`, `:`, `:latest` | The framework-mode build and nginx. Only built when the repository variable `BUILD_FRAMEWORK` is set. **This image is renamed `intercode-assets` when framework mode becomes the default.** | -| `intercode-assets` | `:`, `:`, `:latest` | The previous, non-framework build's assets, until that cutover. | +| Image | Tags | Notes | +| ------------------ | --------------------------------- | ----------------------------------------------------------------------------------- | +| `intercode` | `:`, `:`, `:latest` | The Rails app, with the frontend. | +| `intercode-assets` | `:`, `:`, `:latest` | The frontend and nginx. It has the same build as the `intercode` image at that tag. | -Run both images at the **same tag**. The shell and the Rails API are released together, and mixing versions is not +Run both images at the **same tag**. The frontend and the Rails API are released together, and mixing versions is not supported. +**Upgrading from 8.x.** `intercode-assets` before 9.0.0 was a different image: nginx serving the previous build from +`packs/`, with no `ASSETS_HOST` substitution. From 9.0.0 it is what `intercode-assets-framework` was in the 8.x +releases (which published it as a preview); that name is no longer updated. The Rails containers now need nothing new +but `ASSETS_HOST`, which they use to build the page they serve (see below). + ## Configuration -| Variable | Container | Meaning | -| ------------------------------------------------------------------------------ | --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ASSETS_HOST` | **Rails and web, the same value** | Where browsers load the build's files from: a bare host name, optionally with a port (`assets.example.org`), no scheme. **Empty or unset serves the files from the same domain as the app**, which is what nginx does on its own. See below. | -| `RAILS_UPSTREAM` | web | `host:port` of Rails (for example `rails:3000`). Turns the container into the front server described below. Without it, the container only serves the build. | -| `INTERCODE_HOST` | Rails | The domain the root site is served from. | -| `DATABASE_URL` | Rails | The PostgreSQL connection URL. | -| `SECRET_KEY_BASE`, `DEVISE_SECRET_KEY` | Rails | Random secrets (`openssl rand -hex 64`). Changing them signs everyone out. | -| `OPENID_CONNECT_SIGNING_KEY` | Rails | An RSA private key for signing OpenID Connect tokens, on one line with `\n` for line breaks. | -| `AWS_S3_BUCKET`, `AWS_S3_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | Rails | Where uploaded files are stored. Rails won't boot without a bucket name. | +| Variable | Container | Meaning | +| ------------------------------------------------------------------------------ | --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ASSETS_HOST` | **Rails and web, the same value** | Where browsers load the build's files from: a bare host name, optionally with a port (`assets.example.org`), no scheme. **Empty or unset serves the files from the same domain as the app**, which Rails and nginx both do. See below. | +| `RAILS_UPSTREAM` | web | `host:port` of Rails (for example `rails:3000`). Turns the container into the front server described below. Without it, the container only serves the build. | +| `INTERCODE_HOST` | Rails | The domain the root site is served from. | +| `DATABASE_URL` | Rails | The PostgreSQL connection URL. | +| `SECRET_KEY_BASE`, `DEVISE_SECRET_KEY` | Rails | Random secrets (`openssl rand -hex 64`). Changing them signs everyone out. | +| `OPENID_CONNECT_SIGNING_KEY` | Rails | An RSA private key for signing OpenID Connect tokens, on one line with `\n` for line breaks. | +| `AWS_S3_BUCKET`, `AWS_S3_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | Rails | Where uploaded files are stored. Rails won't boot without a bucket name. | `example.env` lists these with comments. Rails reads more variables (error reporting, email, Stripe and so on); see the configuration files for them. ### `ASSETS_HOST` has to match on both -The build is made against a placeholder host, because the real one is only known when the container starts. When the web -container starts it takes a fresh copy of the build and replaces the placeholder in every file that mentions it -(`index.html`, the route manifest, the CSS) with `https://$ASSETS_HOST/`, or with `/` if `ASSETS_HOST` is empty. So: +The build is made against a placeholder host, because the real one is only known when the container starts. When a +Rails or web container starts it takes a fresh copy of the build and replaces the placeholder in every file that +mentions it (`index.html`, the route manifest, the CSS) with `https://$ASSETS_HOST/`, or with `/` if `ASSETS_HOST` is +empty. So: - **One server, same-origin assets (the example).** Leave `ASSETS_HOST` empty everywhere. -- **A separate asset host.** Run the image a second time on that host, without `RAILS_UPSTREAM`, and give **both** - containers the same `ASSETS_HOST`. If the asset host's container starts without it, its manifest points at the wrong - place and the app won't boot. The asset host also answers with `Access-Control-Allow-Origin: *`, which convention - domains need to load module scripts from it. +- **A separate asset host.** Put the build's `assets/` directory on that host, either by running the + `intercode-assets` image there (without `RAILS_UPSTREAM`) or by uploading the files to a bucket behind a CDN. Give + **every** container, Rails and web, the same `ASSETS_HOST`. An asset host that was given the files without the + substitution has a manifest that points at the wrong place, and the app won't boot. The asset host also answers with + `Access-Control-Allow-Origin: *`, which convention domains need to load module scripts from it. Forgetting `ASSETS_HOST` doesn't fail loudly: the files are just requested from the app's own domain. @@ -105,16 +114,17 @@ static shell one at a time: the order, what to check, what to watch and how to r Everything in `assets/` is named by a hash of its contents and cached for 180 days. `index.html` is the one file whose name doesn't change, so it isn't cached: browsers revalidate it, and a new release is picked up on the next page load. -A browser tab opened before a release may still ask for files that the new image doesn't have. The app recovers: when a -file the shell names fails to load it reloads once to get the current shell, and then shows a message with a reload -button if that doesn't help. The image can carry earlier releases' files forward to avoid even that (build it with -`--build-arg PREVIOUS_ASSETS_IMAGE=`; `PREVIOUS_ASSETS_RETENTION_DAYS`, default 30, sets how long -they're kept), but the published images are not built that way, so **they contain only their own release's files**. -Deployers who care can build the image themselves with that argument, or keep the files in a bucket as the -CloudFront setup does. - -The old non-framework build had a `nocache-files.json` list of files to serve uncached. That doesn't apply to the -framework build; `index.html` is the only such file. +A browser tab opened before a release may still ask for files that the new release doesn't have. The app recovers: when +a file the shell names fails to load it reloads once to get the current shell, and then shows a message with a reload +button if that doesn't help. Two things can avoid even that: + +- The published **`intercode-assets` image** is built on the previous release's image, so it carries earlier releases' + files forward and prunes them 30 days after the release that dropped them. To do the same in an image you build + yourself, use `docker build --target assets --build-arg PREVIOUS_ASSETS_IMAGE=`; + `PREVIOUS_ASSETS_RETENTION_DAYS` sets the number of days. +- **Rails serves only its own release's files**, so a deployment that serves the frontend from Rails has the reload + and nothing more. Put the files in a bucket (as the CloudFront setup does) or run the `intercode-assets` image as the + asset host to keep the earlier ones. ## The example diff --git a/doc-site/docs/deploying/staged-rollout.md b/doc-site/docs/deploying/staged-rollout.md index 7e1d5df88bd..ff6b60a6f64 100644 --- a/doc-site/docs/deploying/staged-rollout.md +++ b/doc-site/docs/deploying/staged-rollout.md @@ -10,6 +10,14 @@ convention domain at a time. It is part of the migration tracked in [#12219](https://github.com/neinteractiveliterature/intercode/issues/12219) and [#12260](https://github.com/neinteractiveliterature/intercode/issues/12260). +:::note +This guide was written for the 8.8 releases, which could still build the old (data-mode) frontend. From 9.0.0 the Rails +image carries the framework build and serves it itself, so a Rails origin already serves the same frontend as the static +shell, and there is no data-mode build to move away from. Static shell mode is still an option for a CloudFront +distribution, and the checks and monitoring below still apply when you turn it on, but rolling back from 9.0.0 to the +old frontend means deploying an 8.x image. +::: + Nothing here removes the data-mode build. Every release still builds and publishes both, so each step can be undone by changing one Terraform variable. The code cutover (v9.0.0), which makes the framework build the only one and gives the Rails image its own copy of the shell, comes after this rollout is finished. Domains that Rails serves directly, with no diff --git a/doc-site/static/deploying/docker-compose.example.yml b/doc-site/static/deploying/docker-compose.example.yml index 25429961c40..380d64b3aea 100644 --- a/doc-site/static/deploying/docker-compose.example.yml +++ b/doc-site/static/deploying/docker-compose.example.yml @@ -1,4 +1,4 @@ -# An example of running Intercode from the published images, with nginx (the framework-mode assets image) in front. +# An example of running Intercode from the published images, with nginx (the assets image) in front. # See https://intercode.interactiveliterature.org/docs/deploying for what each piece is, and for what a real deployment needs on top of this. # # cp example.env .env # then edit it; see the comments there @@ -16,8 +16,8 @@ x-intercode: &intercode SECRET_KEY_BASE: ${SECRET_KEY_BASE} DEVISE_SECRET_KEY: ${DEVISE_SECRET_KEY} OPENID_CONNECT_SIGNING_KEY: ${OPENID_CONNECT_SIGNING_KEY} - # Where the browser loads the build from. Leave it empty to serve assets from the same domain as the app (nginx does - # that); set it to a separate asset host's name, and give that host the same value (see web below). + # Where the browser loads the build from. Leave it empty to serve assets from the same domain as the app (nginx and + # Rails both do that); set it to a separate asset host's name, and give that host the same value (see web below). ASSETS_HOST: ${ASSETS_HOST:-} RAILS_LOG_TO_STDOUT: 'true' AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} @@ -87,7 +87,7 @@ services: # The framework-mode build, served by nginx, which also proxies the Rails routes to Rails and sends crawlers to # Rails' /og-shell. Pin INTERCODE_VERSION to the same release for both images. web: - image: ghcr.io/neinteractiveliterature/intercode-assets-framework:${INTERCODE_VERSION:-latest} + image: ghcr.io/neinteractiveliterature/intercode-assets:${INTERCODE_VERSION:-latest} environment: ASSETS_HOST: ${ASSETS_HOST:-} RAILS_UPSTREAM: rails:3000 diff --git a/terraform/modules/cloudfront_intercode/README.md b/terraform/modules/cloudfront_intercode/README.md index 1d0e079910f..d55341bdab6 100644 --- a/terraform/modules/cloudfront_intercode/README.md +++ b/terraform/modules/cloudfront_intercode/README.md @@ -149,7 +149,7 @@ block) and `nginx/rails-locations.conf` (include in the `server` block) are gene second expects an upstream named `rails` and `/etc/nginx/proxy_params.conf`. Send crawlers to `/og-shell?path=` from `location /` when `$is_crawler` is set. -The `intercode-assets-framework` image ships a complete nginx configuration built from these files, with a docker-compose +The `intercode-assets` image ships a complete nginx configuration built from these files, with a docker-compose example and notes for deployers: see [the deployment guide](https://intercode.interactiveliterature.org/docs/deploying). ## Upgrading from the Lambda@Edge router From 2ddcbfe6db897c1318303ee8e2085f92442c8336 Mon Sep 17 00:00:00 2001 From: Nat Budin Date: Sat, 31 Oct 2026 12:11:04 -0700 Subject: [PATCH 5/6] Add a Fly canary config for the 9.0.0 cutover (part of #12260) Temporary: delete this file before merging the branch. Co-Authored-By: Claude Sonnet 5.5 --- fly.canary.toml | 59 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 fly.canary.toml diff --git a/fly.canary.toml b/fly.canary.toml new file mode 100644 index 00000000000..dc453d54f89 --- /dev/null +++ b/fly.canary.toml @@ -0,0 +1,59 @@ +# A canary for the 9.0.0 cutover (#12260): the Rails image of the 12260-cutover branch, serving one real domain, so the +# new way Rails serves the frontend (bin/with_static_shell, public/assets) sees real traffic before a release. Delete +# this file before merging the branch. The steps are in the pull request. +# +# It shares everything with production (the database, the SQS queues, the secrets from chamber) except the app, so: +# - only the web process runs: no shoryuken, or the canary would take jobs off the queues +# - there is no release_command: no migrations, and no release notifications +# - deploy it only from a branch that is rebased on the released main, so it never runs ahead of the schema +app = "intercode-canary" +primary_region = "iad" +swap_size_mb = 512 +console_command = "/usr/src/intercode/bin/entrypoint.sh /bin/bash" + +[build] +dockerfile = "Dockerfile" +build-target = "production" + +[build.args] +# What CI passes (.ruby-version and .node-version), because the Dockerfile's defaults float +RUBY_VERSION = "4.0.6" +NODE_VERSION = "24.22.2" +REVISION = "canary" + +[env] +AWS_REGION = "us-east-1" +CHAMBER_SERVICE = "intercode_production" +CHAMBER_AWS_ROLE_ARN = "arn:aws:iam::689053117832:role/intercode-chamber" +JSON_LOGGING = "1" +MALLOC_ARENA_MAX = "2" +RACK_ENV = "production" +RAILS_ENV = "production" +RAILS_LOG_TO_STDOUT = "enabled" +RAILS_MAX_THREADS = "3" +RAILS_SERVE_STATIC_FILES = "enabled" +RUBY_YJIT_ENABLE = "1" +WEB_CONCURRENCY = "0" + +[processes] +web = "bundle exec bin/rails server -p 3000 -b 0.0.0.0" + +[http_service] +force_https = true +internal_port = 3000 +min_machines_running = 1 +processes = ["web"] + +[[http_service.checks]] +grace_period = "15s" +interval = "30s" +method = "GET" +path = "/healthz" +protocol = "http" +timeout = "5s" + +[[vm]] +cpu_kind = "shared" +cpus = 1 +memory_mb = 1024 +processes = ["web"] From 0dfd73e4bf7760fca9e0efc45633017548ce1d25 Mon Sep 17 00:00:00 2001 From: Nat Budin Date: Sat, 31 Oct 2026 12:21:35 -0700 Subject: [PATCH 6/6] Serve the Devise pages from the static shell, and fix what CI found (part of #12260) SessionsController, PasswordsController and RegistrationsController rendered the data-mode shell for the sign-in, sign-up and password pages, which loads packs/application.js. CI used to hide that by also building the data-mode bundle into the image; with that gone, sign-in broke. They now serve the static shell when there is one (render_spa_shell). The upload script tests drop the deleted packs script and cover UPLOAD_INDEX=false and marking packs/ obsolete, and the CI check of Rails serving assets waits for the container to make public/assets before it looks. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UKBUsy46smVMUGFsNFhy1B --- .github/workflows/ci.yml | 5 +- app/controllers/application_controller.rb | 11 +++ app/controllers/passwords_controller.rb | 4 +- app/controllers/registrations_controller.rb | 4 +- app/controllers/sessions_controller.rb | 2 +- .../static_spa_shell_controller_test.rb | 23 +++++ test/scripts/upload_assets_test.rb | 96 ++++++++----------- 7 files changed, 82 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec7c58ab533..620ad8ae104 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,9 +281,10 @@ jobs: docker run -d --name rails-serving -p 3000:3000 -e RAILS_SERVE_STATIC_FILES=enabled -e INTERCODE_HOST=localhost \ -e DATABASE_URL=postgresql://fakehost/not_a_real_database -e AWS_S3_BUCKET=fakebucket \ ghcr.io/neinteractiveliterature/intercode:${{ github.sha }} - file=$(docker exec rails-serving sh -c 'cd public/assets && ls entry-*.css | head -1') + # The container makes public/assets when it starts, so wait for the file as well as the server for i in $(seq 1 30); do - curl -sf -o /dev/null -H 'X-Forwarded-Proto: https' http://localhost:3000/assets/$file && break + file=$(docker exec rails-serving sh -c 'cd public/assets && ls entry-*.css | head -1' 2>/dev/null || true) + [ -n "$file" ] && curl -sf -o /dev/null -H 'X-Forwarded-Proto: https' http://localhost:3000/assets/$file && break sleep 2 done curl -sfD - -o /dev/null -H 'X-Forwarded-Proto: https' http://localhost:3000/assets/$file | grep -i 'access-control-allow-origin: \*' diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index dfd41c9036f..c9ebe02d6b6 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -71,6 +71,17 @@ def app_component_props protected + # The page for a route that the SPA renders but that Rails serves the URL of (the Devise pages). With a static shell + # (see StaticSpaShell) that is the prebuilt framework-mode shell; otherwise the Rails-rendered one. + def render_spa_shell + if StaticSpaShell.enabled? + response.headers["Cache-Control"] = "no-cache" + send_file StaticSpaShell.index_path, type: "text/html", disposition: "inline" + else + render html: "", layout: "application" + end + end + # Lograge's custom_options (see config/initializers/json_logging.rb) reads these keys # straight off the process_action.action_controller event payload, so they need to be # added here rather than via config.lograge.custom_payload (which nests them instead). diff --git a/app/controllers/passwords_controller.rb b/app/controllers/passwords_controller.rb index 9fa25fe71a8..55ae7e07b8d 100644 --- a/app/controllers/passwords_controller.rb +++ b/app/controllers/passwords_controller.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true class PasswordsController < Devise::PasswordsController def new - render html: "", layout: "application" + render_spa_shell end def edit - render html: "", layout: "application" + render_spa_shell end def create diff --git a/app/controllers/registrations_controller.rb b/app/controllers/registrations_controller.rb index 56abebee106..56b30b5e5a9 100644 --- a/app/controllers/registrations_controller.rb +++ b/app/controllers/registrations_controller.rb @@ -6,11 +6,11 @@ class RegistrationsController < Devise::RegistrationsController prepend_before_action :disable_destroy, only: [:destroy] # rubocop:disable Rails/LexicallyScopedActionFilter def new - render html: "", layout: "application" + render_spa_shell end def edit - render html: "", layout: "application" + render_spa_shell end private diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index da30f728d01..2d45ac4420c 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -4,7 +4,7 @@ class SessionsController < Devise::SessionsController prepend_before_action :set_return_to, only: [:new] def new - render html: "", layout: "application" + render_spa_shell end # Override to allow cross-host redirect back to the convention subdomain after sign-in. diff --git a/test/controllers/static_spa_shell_controller_test.rb b/test/controllers/static_spa_shell_controller_test.rb index 768fe09e774..eada77b3a6f 100644 --- a/test/controllers/static_spa_shell_controller_test.rb +++ b/test/controllers/static_spa_shell_controller_test.rb @@ -101,4 +101,27 @@ class StaticSpaShellControllerTest < ActionDispatch::IntegrationTest assert_not StaticSpaShell.crawler?(nil) end end + + describe "the Devise pages" do + after { ENV.delete("STATIC_SHELL_DIR") } + + %w[/users/sign_in /users/sign_up /users/password/new /users/password/edit?reset_password_token=abc].each do |path| + it "serves #{path} from the static shell when there is one" do + ENV["STATIC_SHELL_DIR"] = @shell_dir + + get path, headers: { "User-Agent" => browser } + + assert_response :ok + assert_equal index_html, response.body + assert_equal "no-cache", response.headers["Cache-Control"] + end + + it "renders #{path} itself without one" do + get path, headers: { "User-Agent" => browser } + + assert_response :ok + assert_not_includes response.body, "static-shell-marker" + end + end + end end diff --git a/test/scripts/upload_assets_test.rb b/test/scripts/upload_assets_test.rb index 825724469f9..e87aef53247 100644 --- a/test/scripts/upload_assets_test.rb +++ b/test/scripts/upload_assets_test.rb @@ -74,9 +74,11 @@ def stub_env(dir, bucket_keys, tagged) } end + # A Hash as the last argument is added to the script's environment def run_script(script, *args, bucket_keys:, tagged: [], fail_key: nil) + env = args.last.is_a?(Hash) ? args.pop : {} Dir.mktmpdir do |dir| - full_env = stub_env(dir, bucket_keys, tagged) + full_env = stub_env(dir, bucket_keys, tagged).merge(env) full_env["AWS_STUB_FAIL_KEY"] = fail_key if fail_key resolved_args = args.map { |arg| arg.respond_to?(:call) ? arg.call(dir) : arg } _out, err, status = Open3.capture3(full_env, Rails.root.join(script).to_s, *resolved_args) @@ -160,75 +162,57 @@ def phase_calls(phase) end end - describe "script/uploadReleaseAssets.sh" do - it "still marks files dropped from packs/ as obsolete and leaves other prefixes alone" do - calls = - run_script( - "script/uploadReleaseAssets.sh", - "the-bucket", - lambda do |dir| - packs = File.join(dir, "packs") - FileUtils.mkdir_p(packs) - File.write(File.join(packs, "current-AAA.js"), "x") - packs - end, - bucket_keys: %w[packs/current-AAA.js packs/gone-BBB.js assets/other.js] - ) + describe "script/uploadFrameworkAssets.sh without index.html (the assets bucket)" do + def calls_without_index(phase) + run_script( + "script/uploadFrameworkAssets.sh", + "the-bucket", + method(:framework_build), + phase, + { "UPLOAD_INDEX" => "false" }, + bucket_keys: %w[assets/shared-SAME.js assets/old-OLD.js] + ) + end - copies = calls.select { |call| call.start_with?("s3api copy-object") } + it "uploads no index.html in the publish phase, but still marks the dropped files" do + calls = calls_without_index("publish") - assert_equal 1, copies.size - assert_includes copies.first, "--key packs/gone-BBB.js" + assert(calls.none? { |call| call.start_with?("s3 cp") }) + assert(calls.any? { |call| call.start_with?("s3api copy-object") && call.include?("--key assets/old-OLD.js") }) + end + + it "still uploads the hashed files in the hashed phase" do + calls = calls_without_index("hashed") + + assert_equal 1, calls.size + assert_includes calls.first, "s3://the-bucket/assets --recursive" end it "fails when marking a file fails" do run_script( - "script/uploadReleaseAssets.sh", + "script/uploadFrameworkAssets.sh", "the-bucket", - lambda do |dir| - packs = File.join(dir, "packs") - FileUtils.mkdir_p(packs) - File.write(File.join(packs, "current-AAA.js"), "x") - packs - end, - bucket_keys: %w[packs/current-AAA.js packs/gone-BBB.js], - fail_key: "packs/gone-BBB.js" + method(:framework_build), + "publish", + bucket_keys: %w[assets/old-OLD.js], + fail_key: "assets/old-OLD.js" ) end + end - describe "in phases" do - def phase_calls(phase) + describe "script/mark_obsolete_assets.py with no current files (what the cutover does to packs/)" do + it "marks every untagged file under the prefix and nothing else" do + calls = run_script( - "script/uploadReleaseAssets.sh", + "script/mark_obsolete_assets.py", "the-bucket", - lambda do |dir| - packs = File.join(dir, "packs") - FileUtils.mkdir_p(packs) - %w[application.js application-styles.js current-AAA.js].each do |name| - File.write(File.join(packs, name), "x") - end - packs - end, - phase, - bucket_keys: %w[packs/current-AAA.js packs/gone-BBB.js] + "packs/", + bucket_keys: %w[packs/a-AAA.js packs/b-BBB.js packs/c-TAGGED.js assets/current-CCC.js], + tagged: ["packs/c-TAGGED.js"] ) - end - - it "uploads only the hashed files in the hashed phase, and marks nothing" do - calls = phase_calls("hashed") - - assert_equal 1, calls.size - assert_includes calls.first, "s3://the-bucket/packs --recursive --exclude application.js" - end - it "uploads only the entry files in the publish phase, then marks the dropped files" do - calls = phase_calls("publish") - - uploads = calls.select { |call| call.start_with?("s3 cp") } - assert_equal 2, uploads.size - assert(uploads.all? { |call| call.include?("--cache-control no-cache") }) - assert(calls.any? { |call| call.start_with?("s3api copy-object") && call.include?("--key packs/gone-BBB.js") }) - end + marked = calls.select { |call| call.start_with?("s3api copy-object") }.map { |call| call[/--key (\S+)/, 1] } + assert_equal %w[packs/a-AAA.js packs/b-BBB.js], marked.sort end end end