feat: update a live platform and recreate a single node

.github/workflows/e2e-tests.yaml

@@ -56,6 +56,23 @@ jobs:
       checks: ${{ matrix.checks }}
     secrets: inherit
 
+  update:
+    name: ${{ matrix.name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: update-aws
+            provider: aws
+          - name: update-openstack
+            provider: openstack
+    uses: ./.github/workflows/update-test.yaml
+    with:
+      name: ${{ matrix.name }}
+      stack: "${{ github.workflow }}-${{ github.run_number }}-${{ matrix.name }}"
+      provider: ${{ matrix.provider }}
+    secrets: inherit
+
   snapshot:
     uses: ./.github/workflows/snapshot-test.yaml
     with:

.github/workflows/update-test.yaml

@@ -0,0 +1,236 @@
+name: "Update Test"
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        description: "What the platform is called, in the job names"
+        required: true
+        type: string
+      stack:
+        description: "Name of the stack to spawn and update"
+        required: true
+        type: string
+      provider:
+        description: "Cloud provider to spawn the platform on"
+        required: true
+        type: string
+
+permissions:
+  id-token: write
+  actions: read
+  contents: read
+
+# NOTE: Read by the OpenStack provider straight from the environment, the way
+# the AWS one reads what the credentials step leaves behind. They are harmless
+# to a platform spawned on AWS, which never looks. The password cannot be set
+# here: a workflow level environment cannot read secrets.
+env:
+  OS_AUTH_URL: ${{ vars.OS_AUTH_URL }}
+  OS_USERNAME: ${{ vars.OS_USERNAME }}
+  OS_PROJECT_ID: ${{ vars.OS_PROJECT_ID }}
+  OS_PROJECT_NAME: ${{ vars.OS_PROJECT_NAME }}
+  # The variable holds the region, the tooling reads it under another name
+  OS_REGION_NAME: ${{ vars.OS_REGION }}
+  # Not configured anywhere, being the same for every OVH project
+  OS_USER_DOMAIN_NAME: "Default"
+  OS_PROJECT_DOMAIN_ID: "default"
+  OS_IDENTITY_API_VERSION: "3"
+
+jobs:
+  # Change a live platform rather than spawn one with the change already
+  # made. It updates the same platform several times in a row, which is why
+  # it is not a platform test like the others.
+  test:
+    name: "${{ inputs.name }} test"
+    runs-on: ubuntu-24.04
+    env:
+      OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      # NOTE: Needed whatever the provider, the stacks living in an S3 bucket
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_MAIN_ACCOUNT_ID }}:role/scality-github-e2e-infrastructure
+          role-session-name: GitHubActions-${{ github.run_id }}
+          aws-region: eu-north-1
+
+      - name: Spawn the platform
+        id: spawn
+        uses: ./
+        with:
+          action: spawn
+          artifacts_user: ${{ secrets.ARTIFACTS_USER }}
+          artifacts_password: ${{ secrets.ARTIFACTS_PASSWORD }}
+          stack_name: ${{ inputs.stack }}
+          configuration: |-
+            provider: ${{ inputs.provider }}
+            instance_image: rocky-9
+            instance_count: 2
+            authorized_cidrs: ${{ format('[{0},"__my_ip__"]', join(fromJSON(vars.AUTHORIZED_CIDRS), ',')) }}
+            extra_volumes:
+              - size: 10
+
+      # NOTE: Run before node-2 goes away, not only once it is back. Reaching
+      # the nodes records their host keys, here and on the bastion. Without
+      # them, node-2 would come back with no old key to clash with, and the
+      # host key check further down would prove nothing.
+      - name: Check the platform
+        run: ./tests/check_platform.sh
+
+      - name: Check the platform starts online
+        run: ssh -F ssh_config node-1 'timeout 15 curl -sS -o /dev/null https://1.1.1.1'
+
+      - name: Take the platform offline
+        id: offline
+        uses: ./
+        with:
+          action: update
+          stack_name: ${{ inputs.stack }}
+          configuration: |-
+            offline: true
+
+      - name: Check the platform is offline
+        run: ./tests/check_offline.sh
+
+      # NOTE: The whole point of switching a live platform, rather than
+      # spawning another one offline, is to keep what runs on it
+      - name: Check no machine was replaced
+        env:
+          BASTION_BEFORE: ${{ steps.spawn.outputs.bastion }}
+          BASTION_AFTER: ${{ steps.offline.outputs.bastion }}
+          NODES_BEFORE: ${{ steps.spawn.outputs.nodes }}
+          NODES_AFTER: ${{ steps.offline.outputs.nodes }}
+        run: |-
+          before=$(jq -cS '{bastion: .id}' <<< "$BASTION_BEFORE")
+          before+=$(jq -cS 'map_values(.id)' <<< "$NODES_BEFORE")
+          after=$(jq -cS '{bastion: .id}' <<< "$BASTION_AFTER")
+          after+=$(jq -cS 'map_values(.id)' <<< "$NODES_AFTER")
+          echo "before: $before"
+          echo "after:  $after"
+          [ "$before" == "$after" ]
+
+      - name: Leave something behind on node-2
+        env:
+          MARKER: ${{ github.run_id }}
+        run: ssh -F ssh_config node-2 "echo '$MARKER' > ~/update-marker"
+
+      - name: Take node-2 away
+        id: away
+        uses: ./
+        with:
+          action: update
+          stack_name: ${{ inputs.stack }}
+          configuration: |-
+            absent_nodes: [node-2]
+
+      - name: Check node-2 is gone, and only node-2
+        env:
+          NODES_BEFORE: ${{ steps.offline.outputs.nodes }}
+          NODES_AFTER: ${{ steps.away.outputs.nodes }}
+        run: |-
+          expected=$(jq -cS 'del(."node-2") | map_values(.id)' <<< "$NODES_BEFORE")
+          found=$(jq -cS 'map_values(.id)' <<< "$NODES_AFTER")
+          echo "expected: $expected"
+          echo "found:    $found"
+          if [ "$found" != "$expected" ]; then
+              echo "The nodes are not those of the platform minus node-2" >&2
+              exit 1
+          fi
+          if grep -q '^Host node-2$' ssh_config; then
+              echo "node-2 is still in ssh_config" >&2
+              exit 1
+          fi
+
+      - name: Bring node-2 back
+        id: back
+        uses: ./
+        with:
+          action: update
+          stack_name: ${{ inputs.stack }}
+          configuration: |-
+            absent_nodes: []
+
+      - name: Check node-2 came back as a new machine, where it was
+        env:
+          NODES_BEFORE: ${{ steps.offline.outputs.nodes }}
+          NODES_AFTER: ${{ steps.back.outputs.nodes }}
+        run: |-
+          others_before=$(jq -cS 'del(."node-2") | map_values(.id)' <<< "$NODES_BEFORE")
+          others_after=$(jq -cS 'del(."node-2") | map_values(.id)' <<< "$NODES_AFTER")
+          if [ "$others_before" != "$others_after" ]; then
+              echo "Another node was replaced: $others_before became $others_after" >&2
+              exit 1
+          fi
+
+          id_before=$(jq -r '."node-2".id' <<< "$NODES_BEFORE")
+          id_after=$(jq -r '."node-2".id' <<< "$NODES_AFTER")
+          echo "node-2 was $id_before, is $id_after"
+          if [ "$id_before" == "$id_after" ]; then
+              echo "node-2 is the machine it was" >&2
+              exit 1
+          fi
+
+          ips_before=$(jq -cS '."node-2".private_ips' <<< "$NODES_BEFORE")
+          ips_after=$(jq -cS '."node-2".private_ips' <<< "$NODES_AFTER")
+          echo "node-2 sits at $ips_after"
+          if [ "$ips_before" != "$ips_after" ]; then
+              echo "node-2 moved from $ips_before" >&2
+              exit 1
+          fi
+
+          state=$(ssh -F ssh_config node-2 '[ -e ~/update-marker ] && echo kept || echo blank')
+          echo "node-2 came back $state"
+          if [ "$state" != "blank" ]; then
+              echo "node-2 kept what was left on it" >&2
+              exit 1
+          fi
+
+          disks=$(ssh -F ssh_config node-2 'lsblk --nodeps --noheadings --output TYPE | grep -c disk')
+          echo "node-2 carries $disks disk(s)"
+          # the one it boots on, plus the one asked for
+          [ "$disks" -eq 2 ]
+
+      # NOTE: StrictHostKeyChecking lets a changed host key through, but turns
+      # forwarding off for the connection. Reaching the node proves nothing,
+      # only the warning tells.
+      - name: Check the host key of node-2 is not taken for a changed one
+        run: |-
+          here=$(ssh -F ssh_config node-2 true 2>&1)
+          bastion=$(ssh -F ssh_config bastion 'ssh -F ssh_config node-2 true' 2>&1)
+          for out in "$here" "$bastion"; do
+              if grep -q "HOST IDENTIFICATION HAS CHANGED" <<< "$out"; then
+                  echo "$out" >&2
+                  exit 1
+              fi
+          done
+
+      - name: Check the platform again
+        run: ./tests/check_platform.sh
+
+  destroy:
+    name: "${{ inputs.name }} destroy"
+    needs: test
+    if: always()
+    runs-on: ubuntu-24.04
+    env:
+      OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_MAIN_ACCOUNT_ID }}:role/scality-github-e2e-infrastructure
+          role-session-name: GitHubActions-${{ github.run_id }}
+          aws-region: eu-north-1
+
+      - name: Destroy platform
+        uses: ./
+        with:
+          action: destroy
+          stack_name: ${{ inputs.stack }}

CONTRIBUTING.md

@@ -33,7 +33,11 @@ destroys them. Two more platforms go up beside them on OpenStack: one asking
 for everything at once, offline with extra networks and extra volumes, and one
 that is snapshotted and spawned back from its images, carrying a file written
 before the snapshot to prove it came back whole. That one takes its images
-away afterwards, nothing else ever would.
+away afterwards, nothing else ever would. One last platform goes up on each
+provider and is changed while it runs, through the `update` action. It is
+switched offline, and none of its machines may be replaced on the way. Then
+one of its nodes is taken away and brought back, blank and at the addresses
+it had, while every other machine is left alone.
 
 Each of them is a workflow of its own carrying its spawn, its checks and its
 destroy, so a platform goes away as soon as its own test is done rather than

DESIGN.md

@@ -17,6 +17,17 @@ The provider is picked by the `provider` configuration key, which keeps a stack
 configuration portable from one cloud to another as long as the values it uses
 exist on both.
 
+### A machine keeps its image
+
+The image a machine boots on is read once, when the machine is created. An
+image name is looked up again on every update, and it answers with the latest
+image of that name. Following it would replace every machine of a live
+platform as soon as the cloud publishes a newer one, whatever the update was
+about. Changing `instance_image`, `bastion_image`, `restore_snapshot` or
+`instance_image_file` on a live platform therefore leaves its machines alone.
+Only the machines created afterwards boot on the new image, and destroying the
+platform is the way to have all of them on it.
+
 ### Provider differences
 
 The interface hides most of the differences, a few of them remain visible from

Pulumi.yaml

@@ -76,6 +76,12 @@ config:
     type: integer
     description: "Number of instances to create"
     default: 3
+  absent_nodes:
+    type: array
+    items:
+      type: string
+    description: "Nodes to leave out, keeping their addresses for when they come back (e.g. node-2)"
+    default: []
   instance_flavor:
     type: string
     description: "Flavor of the instance, a portable size or a name from the cloud catalog"

README.md

@@ -52,6 +52,7 @@ A platform is described by the configuration of its stack. Set a key with
 | instance_image_file | Image file to upload and boot the instances on, instead of instance_image | string | `` | no |
 | instance_image_file_format | Format of the uploaded image file | string | `qcow2` | no |
 | instance_count | Number of instances to create | integer | `3` | no |
+| absent_nodes | Nodes to leave out, keeping their addresses for when they come back (e.g. node-2) | array | `[]` | no |
 | instance_flavor | Flavor of the instance, a portable size or a name from the cloud catalog | string | `medium` | no |
 | instance_ssh_user | SSH user for the instances (defaults to the one the image is known to use) | string | `` | no |
 | instance_root_disk_size | Root disk size for the instance in GiB (AWS only, set by the flavor on OpenStack) | integer | `50` | no |