recovery-drill.yml

name: Recovery Drill

# Required GitHub configuration:
#   Repository variables:
#     BACKUP_AWS_REGION
#     BACKUP_BUCKET
#     BACKUP_PREFIX
#     BACKUP_ENVIRONMENT
#     DRILL_STELLAR_NETWORK              e.g. testnet
#     DRILL_STELLAR_NETWORK_PASSPHRASE   network passphrase for the drill target
#     DRILL_SOROBAN_RPC_URL              Soroban RPC endpoint used for replay
#     DRILL_CONTRACT_ID                  Contract ID whose events must be replayed
#     DRILL_REINDEX_FROM_LEDGER          Quarterly replay anchor ledger
#   Repository secrets:
#     RESTORE_AWS_ROLE_ARN               Read-only or read-mostly restore role
#     OPS_ALERT_WEBHOOK_URL              Optional alert destination

on:
  schedule:
    - cron: '0 10 1-7 1,4,7,10 1'
  workflow_dispatch:
    inputs:
      backup_object_key:
        description: 'Optional exact S3 object key to restore; defaults to latest dump'
        required: false
        type: string
      from_ledger:
        description: 'Optional replay anchor ledger; defaults to DRILL_REINDEX_FROM_LEDGER'
        required: false
        type: string
      create_ticket:
        description: 'Create a GitHub issue with drill evidence and follow-ups'
        required: false
        default: true
        type: boolean

permissions:
  contents: read
  id-token: write
  issues: write

concurrency:
  group: recovery-drill-${{ vars.BACKUP_ENVIRONMENT || 'production' }}
  cancel-in-progress: false

jobs:
  restore-and-replay:
    name: Restore latest backup and replay indexer
    runs-on: ubuntu-latest
    timeout-minutes: 90
    services:
      postgres:
        image: postgres:16-alpine
        env:
          POSTGRES_DB: recovery_drill
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d recovery_drill"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis:7-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    env:
      AWS_REGION: ${{ vars.BACKUP_AWS_REGION }}
      BACKUP_BUCKET: ${{ vars.BACKUP_BUCKET }}
      BACKUP_PREFIX: ${{ vars.BACKUP_PREFIX || 'postgres-backups' }}
      RESTORE_ENVIRONMENT: ${{ vars.BACKUP_ENVIRONMENT || 'production' }}
      BACKUP_OBJECT_KEY: ${{ github.event.inputs.backup_object_key }}
      RESTORE_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/recovery_drill
      DRILL_OUTPUT_DIR: ${{ github.workspace }}/drill-evidence
      DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/recovery_drill
      REDIS_URL: redis://127.0.0.1:6379/0
      STELLAR_NETWORK: ${{ vars.DRILL_STELLAR_NETWORK || 'testnet' }}
      STELLAR_NETWORK_PASSPHRASE: ${{ vars.DRILL_STELLAR_NETWORK_PASSPHRASE || 'Test SDF Network ; September 2015' }}
      SOROBAN_RPC_URL: ${{ vars.DRILL_SOROBAN_RPC_URL || 'https://soroban-testnet.stellar.org' }}
      CONTRACT_ID: ${{ vars.DRILL_CONTRACT_ID }}
      DRILL_REINDEX_FROM_LEDGER: ${{ github.event.inputs.from_ledger || vars.DRILL_REINDEX_FROM_LEDGER }}
      CREATE_DRILL_TICKET: ${{ github.event_name == 'schedule' || github.event.inputs.create_ticket == 'true' }}
    steps:
      - uses: actions/checkout@v4

      - name: Validate drill configuration
        run: |
          for name in AWS_REGION BACKUP_BUCKET BACKUP_PREFIX RESTORE_DATABASE_URL CONTRACT_ID DRILL_REINDEX_FROM_LEDGER SOROBAN_RPC_URL STELLAR_NETWORK_PASSPHRASE; do
            if [ -z "${!name}" ]; then
              echo "::error::Missing required value: ${name}"
              exit 1
            fi
          done

      - name: Install PostgreSQL client and jq
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y postgresql-client jq

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.RESTORE_AWS_ROLE_ARN }}
          aws-region: ${{ vars.BACKUP_AWS_REGION }}

      - name: Restore latest backup into fresh Postgres
        run: ./scripts/ops/postgres-restore-drill.sh

      - name: Set up Node.js for replay
        uses: actions/setup-node@v4
        with:
          node-version: '22'
          cache: npm
          cache-dependency-path: backend/package-lock.json

      - name: Install backend dependencies
        working-directory: backend
        run: npm ci

      - name: Generate Prisma client
        working-directory: backend
        run: npx prisma generate

      - name: Replay indexer from ledger anchor
        working-directory: backend
        run: |
          npm run ops:replay-indexer -- \
            --from-ledger "${DRILL_REINDEX_FROM_LEDGER}" \
            --network "${STELLAR_NETWORK}" \
            --output "${DRILL_OUTPUT_DIR}/indexer-replay.json"

      - name: Write recovery drill ticket
        if: always()
        run: |
          mkdir -p "$DRILL_OUTPUT_DIR"
          cat > "$DRILL_OUTPUT_DIR/recovery-drill-ticket.md" <<EOF
          # Recovery Drill Ticket

          ## Metadata

          - Environment: ${RESTORE_ENVIRONMENT}
          - Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)
          - Workflow run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}
          - Backup object key: ${BACKUP_OBJECT_KEY:-latest under ${BACKUP_PREFIX}/${RESTORE_ENVIRONMENT}}
          - Restore database target: ${RESTORE_DATABASE_URL}
          - Replay anchor ledger N: ${DRILL_REINDEX_FROM_LEDGER}

          ## Validation

          - Restore job status: ${{ job.status }}
          - Evidence artifact: recovery-drill-${GITHUB_RUN_ID}
          - Replay output: indexer-replay.json

          ## Follow-up actions

          - File any failed validation, IAM, replay, or timing gaps as follow-up issues within 48 hours.
          - Update docs/ops/recovery-drill-log.md with observed RTO/RPO after reviewing this artifact.
          EOF

          cat "$DRILL_OUTPUT_DIR/recovery-drill-ticket.md" >> "$GITHUB_STEP_SUMMARY"

      - name: Upload drill evidence
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: recovery-drill-${{ github.run_id }}
          path: drill-evidence/
          retention-days: 365

      - name: File recovery drill ticket
        if: always() && env.CREATE_DRILL_TICKET == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          title="Recovery drill ${RESTORE_ENVIRONMENT} ${GITHUB_RUN_ID} - ${{ job.status }}"
          gh issue create \
            --title "$title" \
            --label "Stellar Wave" \
            --body-file "$DRILL_OUTPUT_DIR/recovery-drill-ticket.md"

      - name: Notify ops on drill failure
        if: failure() && secrets.OPS_ALERT_WEBHOOK_URL != ''
        env:
          OPS_ALERT_WEBHOOK_URL: ${{ secrets.OPS_ALERT_WEBHOOK_URL }}
        run: |
          payload="$(jq -n \
            --arg workflow "$GITHUB_WORKFLOW" \
            --arg runId "$GITHUB_RUN_ID" \
            --arg repository "$GITHUB_REPOSITORY" \
            --arg environment "$RESTORE_ENVIRONMENT" \
            --arg url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
            '{
              text: ("Recovery drill failed for " + $repository + " (" + $environment + ")."),
              workflow: $workflow,
              runId: $runId,
              repository: $repository,
              environment: $environment,
              runUrl: $url
            }')"
          curl -fsSL -X POST "$OPS_ALERT_WEBHOOK_URL" \
            -H 'Content-Type: application/json' \
            --data "$payload"