ci(cd): opt-in re-run-safe freshness guard on the apply steps

.github/workflows/cd.yml

@@ -59,6 +59,20 @@ on:
         description: 'The run label to use for the actions'
         type: string
         default: 'ubuntu-latest'
+      enforce-fresh:
+        description: >
+          Re-check live app and infra commits before applying; skip when either is
+          newer or divergent. Defaults to false.
+        type: boolean
+        default: false
+      freshness-module-id:
+        description: 'Terraform module ID used to read the infra deploy marker.'
+        type: string
+        default: ''
+      freshness-allow-stale:
+        description: 'Bypass freshness checks for a deliberate rollback.'
+        type: boolean
+        default: false
 
 concurrency: cd-${{ inputs.stage }}
 
@@ -83,6 +97,11 @@ jobs:
       tf-directory: ${{ inputs.tf-directory }}
       tf-variables: ${{ inputs.tf-variables }}
       run-label: ${{ inputs.run-label }}
+      enforce-fresh: ${{ inputs.enforce-fresh }}
+      freshness-target-commit: ${{ github.sha }}
+      freshness-module-id: ${{ inputs.freshness-module-id }}
+      freshness-aws-role-arn: ${{ inputs.aws-role-arn }}
+      freshness-allow-stale: ${{ inputs.freshness-allow-stale }}
 
   deploy-app:
     name: Deploy App
@@ -99,3 +118,6 @@ jobs:
       aws-role-arn: ${{ inputs.aws-role-arn }}
       aws-region: ${{ inputs.aws-region }}
       run-label: ${{ inputs.run-label }}
+      enforce-fresh: ${{ inputs.enforce-fresh }}
+      freshness-target-commit: ${{ inputs.version }}
+      freshness-allow-stale: ${{ inputs.freshness-allow-stale }}

.github/workflows/deploy-app.yml

@@ -35,6 +35,18 @@ on:
         description: 'The run label to use for the actions'
         type: string
         default: 'ubuntu-latest'
+      enforce-fresh:
+        description: 'Skip when the live app commit is newer or divergent.'
+        type: boolean
+        default: false
+      freshness-target-commit:
+        description: 'Commit this run intends to deploy.'
+        type: string
+        default: ''
+      freshness-allow-stale:
+        description: 'Bypass freshness checks for a deliberate rollback.'
+        type: boolean
+        default: false
 
 concurrency: deploy-${{ inputs.stage }}
 
@@ -74,8 +86,62 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
+      # Re-check at the apply boundary so every re-run is guarded. ECS read
+      # failures block the deploy; allow-stale skips this step.
+      - name: Assert fresh
+        id: freshness
+        if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale }}
+        shell: bash
+        env:
+          TARGET: ${{ inputs.freshness-target-commit }}
+          NAME: ${{ steps.build_task_name.outputs.name }}
+          AWS_REGION: ${{ inputs.aws-region }}
+        run: |
+          set -euo pipefail
+          git fetch --no-tags --prune --unshallow origin 2>/dev/null || git fetch --no-tags origin 2>/dev/null || true
+          if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$TARGET" 2>/dev/null || true; fi
+          if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then
+            # Non-commit image tags cannot be ancestry-checked.
+            echo "::warning::assert-fresh(app): target '${TARGET}' is not a git commit — cannot evaluate freshness; proceeding UNGUARDED"
+            echo "skip=false" >> "$GITHUB_OUTPUT"; exit 0
+          fi
+          set +e
+          td="$(aws ecs describe-services --cluster "${NAME}-cluster" --services "${NAME}-service" --query "services[0].deployments[?status=='PRIMARY'].taskDefinition | [0]" --output text --region "$AWS_REGION" 2>/tmp/ecs_err)"
+          ecs_rc=$?
+          set -e
+          if [ "$ecs_rc" -ne 0 ] || [ -z "$td" ] || [ "$td" = "None" ]; then
+            echo "::error::assert-fresh(app): could not read the PRIMARY ECS task definition ($(tr -d '\n' </tmp/ecs_err)) — refusing to deploy blind (fail-closed). Rerun once ECS is reachable, or set freshness-allow-stale."
+            exit 1
+          fi
+          set +e
+          image="$(aws ecs describe-task-definition --task-definition "$td" --query "taskDefinition.containerDefinitions[?name=='${NAME}'].image | [0]" --output text --region "$AWS_REGION" 2>/tmp/ecs_err)"
+          image_rc=$?
+          set -e
+          if [ "$image_rc" -ne 0 ] || [ -z "$image" ] || [ "$image" = "None" ]; then
+            set +e
+            image="$(aws ecs describe-task-definition --task-definition "$td" --query "taskDefinition.containerDefinitions[0].image" --output text --region "$AWS_REGION" 2>/tmp/ecs_err)"
+            image_rc=$?
+            set -e
+          fi
+          if [ "$image_rc" -ne 0 ] || [ -z "$image" ] || [ "$image" = "None" ]; then
+            echo "::error::assert-fresh(app): could not read the PRIMARY ECS task definition ($(tr -d '\n' </tmp/ecs_err)) — refusing to deploy blind (fail-closed). Rerun once ECS is reachable, or set freshness-allow-stale."
+            exit 1
+          fi
+          running="${image##*:}"
+          skip=false
+          if ! git cat-file -e "${running}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$running" 2>/dev/null || true; fi
+          if ! git cat-file -e "${running}^{commit}" 2>/dev/null; then
+            echo "::warning::assert-fresh(app): live ${running:0:7} unresolvable in history — treating as stale"; skip=true
+          elif git merge-base --is-ancestor "$running" "$TARGET"; then
+            echo "assert-fresh(app): live ${running:0:7} is an ancestor of ${TARGET:0:7} — fresh"
+          else
+            echo "::warning::assert-fresh(app): live ${running:0:7} is NOT an ancestor of ${TARGET:0:7} — newer/divergent already live; skipping ECS deploy"; skip=true
+          fi
+          echo "skip=$skip" >> "$GITHUB_OUTPUT"
+
       - name: Deploy image to ECS
         id: deploy
+        if: ${{ steps.freshness.outputs.skip != 'true' }}
         uses: WalletConnect/actions/aws/ecs/deploy-image/@2.5.4
         with:
           aws-role-arn: ${{ inputs.aws-role-arn }}

.github/workflows/deploy-infra.yml

@@ -39,6 +39,26 @@ on:
         description: 'The run label to use for the actions'
         type: string
         default: 'ubuntu-latest'
+      enforce-fresh:
+        description: 'Skip when the live infra commit is newer or divergent.'
+        type: boolean
+        default: false
+      freshness-target-commit:
+        description: 'Commit this run intends to apply.'
+        type: string
+        default: ''
+      freshness-module-id:
+        description: 'Terraform module ID used to read the infra deploy marker.'
+        type: string
+        default: ''
+      freshness-aws-role-arn:
+        description: 'App-account AWS role used to read the infra deploy marker.'
+        type: string
+        default: ''
+      freshness-allow-stale:
+        description: 'Bypass freshness checks for a deliberate rollback.'
+        type: boolean
+        default: false
     secrets:
       TF_API_TOKEN:
         required: true
@@ -67,6 +87,67 @@ jobs:
           submodules: recursive
           token: ${{ secrets.PRIVATE_SUBMODULE_ACCESS_TOKEN || github.token }}
 
+      # Use app-account credentials for the marker read, then restore monitoring
+      # credentials below. Credential and marker read failures block the apply.
+      - name: Configure AWS Credentials for freshness read
+        if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ inputs.freshness-aws-role-arn }}
+          aws-region: ${{ inputs.aws-region }}
+
+      - name: Assert fresh
+        id: freshness
+        if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale }}
+        shell: bash
+        env:
+          TARGET: ${{ inputs.freshness-target-commit }}
+          MODULE_ID: ${{ inputs.freshness-module-id }}
+          AWS_REGION: ${{ inputs.aws-region }}
+        run: |
+          set -euo pipefail
+          # Missing marker configuration would disable the guard.
+          if [ -z "$MODULE_ID" ]; then
+            echo "::error::assert-fresh(infra): enforce-fresh is set but freshness-module-id is empty — refusing to apply unguarded"; exit 1
+          fi
+          git fetch --no-tags --prune --unshallow origin 2>/dev/null || git fetch --no-tags origin 2>/dev/null || true
+          if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$TARGET" 2>/dev/null || true; fi
+          if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then
+            echo "::error::assert-fresh(infra): target ${TARGET} is not a resolvable commit"; exit 1
+          fi
+          # Read the marker distinguishing "not created yet" (legit first apply →
+          # fail OPEN) from a genuine read ERROR — access denied, throttle, timeout
+          # (anomalous → fail CLOSED). SSM is a reliable read, so an error here is a
+          # real red flag and we will NOT apply blind and risk a rollback. The app
+          # guard likewise uses the reliable ECS control plane and fails closed.
+          # `freshness-allow-stale` overrides for a deliberate run.
+          set +e
+          running="$(aws ssm get-parameter --name "/${MODULE_ID}/deployed_infra_commit" --query 'Parameter.Value' --output text --region "$AWS_REGION" 2>/tmp/ssm_err)"
+          ssm_rc=$?
+          set -e
+          skip=false
+          if [ "$ssm_rc" -ne 0 ]; then
+            if grep -q "ParameterNotFound" /tmp/ssm_err; then
+              running=""   # marker not created yet (first apply) — the open check below logs it
+            else
+              echo "::error::assert-fresh(infra): could not read the deployed_infra_commit marker ($(tr -d '\n' </tmp/ssm_err)) — refusing to apply blind (fail-closed). Rerun once SSM is reachable, or set freshness-allow-stale for a deliberate override."
+              exit 1
+            fi
+          fi
+          if [ -z "$running" ] || [ "$running" = "None" ] || [ "$running" = "unknown" ]; then
+            echo "::warning::assert-fresh(infra): no usable live marker (empty/None/unknown, e.g. first apply or a never-completed prior apply) — proceeding UNGUARDED (nothing to compare against)"
+          else
+            if ! git cat-file -e "${running}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$running" 2>/dev/null || true; fi
+            if ! git cat-file -e "${running}^{commit}" 2>/dev/null; then
+              echo "::warning::assert-fresh(infra): live ${running:0:7} unresolvable in history — treating as stale"; skip=true
+            elif git merge-base --is-ancestor "$running" "$TARGET"; then
+              echo "assert-fresh(infra): live ${running:0:7} is an ancestor of ${TARGET:0:7} — fresh"
+            else
+              echo "::warning::assert-fresh(infra): live ${running:0:7} is NOT an ancestor of ${TARGET:0:7} — newer/divergent already applied; skipping terraform apply"; skip=true
+            fi
+          fi
+          echo "skip=$skip" >> "$GITHUB_OUTPUT"
+
       - name: Configure AWS Credentials for Monitoring account
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -105,6 +186,7 @@ jobs:
             ${{ inputs.tf-variables }}
 
       - name: Apply on ${{ inputs.stage }}
+        if: ${{ steps.freshness.outputs.skip != 'true' }}
         working-directory: ${{ inputs.tf-directory }}
         run: terraform apply -auto-approve -no-color
 
@@ -114,3 +196,52 @@ jobs:
         with:
           workspace-id: ${{ steps.grafana-get-key.outputs.workspace-id }}
           key-name: ${{ steps.grafana-get-key.outputs.key-name }}
+
+      - name: Configure AWS Credentials for marker write
+        if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale && steps.freshness.outputs.skip != 'true' }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ inputs.freshness-aws-role-arn }}
+          aws-region: ${{ inputs.aws-region }}
+
+      - name: Advance infra marker
+        if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale && steps.freshness.outputs.skip != 'true' }}
+        shell: bash
+        env:
+          SHA: ${{ inputs.freshness-target-commit }}
+          MODULE_ID: ${{ inputs.freshness-module-id }}
+          AWS_REGION: ${{ inputs.aws-region }}
+        run: |
+          set -euo pipefail
+          set +e
+          current="$(aws ssm get-parameter --name "/${MODULE_ID}/deployed_infra_commit" --query 'Parameter.Value' --output text --region "$AWS_REGION" 2>/tmp/ssm_err)"
+          ssm_rc=$?
+          set -e
+          if [ "$ssm_rc" -ne 0 ]; then
+            if grep -q "ParameterNotFound" /tmp/ssm_err; then
+              aws ssm put-parameter --name "/${MODULE_ID}/deployed_infra_commit" --type String --overwrite --value "$SHA" --region "$AWS_REGION" >/dev/null
+              exit 0
+            fi
+            echo "::warning::advance-infra-marker: could not read the deployed_infra_commit marker ($(tr -d '\n' </tmp/ssm_err)) — refusing to risk regressing it"
+            exit 0
+          fi
+          if [ -z "$current" ] || [ "$current" = "None" ] || [ "$current" = "unknown" ]; then
+            aws ssm put-parameter --name "/${MODULE_ID}/deployed_infra_commit" --type String --overwrite --value "$SHA" --region "$AWS_REGION" >/dev/null
+            exit 0
+          fi
+          if ! git cat-file -e "${SHA}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$SHA" 2>/dev/null || true; fi
+          if ! git cat-file -e "${current}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$current" 2>/dev/null || true; fi
+          if ! git cat-file -e "${SHA}^{commit}" 2>/dev/null; then
+            echo "::warning::advance-infra-marker: target ${SHA:0:7} is unresolvable in history — marker not advanced"
+            exit 0
+          fi
+          if ! git cat-file -e "${current}^{commit}" 2>/dev/null; then
+            echo "::warning::advance-infra-marker: current marker ${current:0:7} is unresolvable in history — marker not advanced"
+            exit 0
+          fi
+          if git merge-base --is-ancestor "$current" "$SHA"; then
+            aws ssm put-parameter --name "/${MODULE_ID}/deployed_infra_commit" --type String --overwrite --value "$SHA" --region "$AWS_REGION" >/dev/null
+          else
+            echo "::warning::advance-infra-marker: marker ${current:0:7} newer/divergent than ${SHA:0:7} — refusing to regress"
+            exit 0
+          fi