Skip to content

Commit f35593d

Browse files
jhayniffyjhayniffy
authored
Merge pull request #295 from JTKaduma/feat/backup-restore
feat: add backup and restore
2 parents aa1a372 + 0533e57 commit f35593d

10 files changed

Lines changed: 969 additions & 3 deletions

‎.github/workflows/postgres-backup.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: Postgres Backups
2+
3+
# Required GitHub configuration:
4+
# Repository variables:
5+
# BACKUP_AWS_REGION e.g. us-east-1
6+
# BACKUP_BUCKET S3 bucket name for encrypted backups
7+
# BACKUP_PREFIX Prefix within the bucket, e.g. postgres-backups
8+
# BACKUP_ENVIRONMENT production | staging
9+
# BACKUP_RETENTION_DAYS e.g. 35
10+
# Repository secrets:
11+
# BACKUP_AWS_ROLE_ARN OIDC-assumable IAM role with scoped S3/KMS access
12+
# BACKUP_DATABASE_URL Postgres connection string for the protected environment
13+
# BACKUP_KMS_KEY_ID KMS key ARN or alias used for SSE-KMS
14+
# OPS_ALERT_WEBHOOK_URL Optional Slack/PagerDuty/webhook endpoint for failures
15+
16+
on:
17+
schedule:
18+
- cron: '15 */6 * * *'
19+
workflow_dispatch:
20+
inputs:
21+
retention_days:
22+
description: 'Retention window in days (defaults to BACKUP_RETENTION_DAYS)'
23+
required: false
24+
type: string
25+
skip_prune:
26+
description: 'Skip old-object pruning for this run'
27+
required: false
28+
default: false
29+
type: boolean
30+
31+
permissions:
32+
contents: read
33+
id-token: write
34+
35+
concurrency:
36+
group: postgres-backup-${{ vars.BACKUP_ENVIRONMENT || 'production' }}
37+
cancel-in-progress: false
38+
39+
jobs:
40+
backup:
41+
name: Encrypted pg_dump to S3
42+
runs-on: ubuntu-latest
43+
timeout-minutes: 30
44+
env:
45+
AWS_REGION: ${{ vars.BACKUP_AWS_REGION }}
46+
BACKUP_BUCKET: ${{ vars.BACKUP_BUCKET }}
47+
BACKUP_PREFIX: ${{ vars.BACKUP_PREFIX || 'postgres-backups' }}
48+
BACKUP_ENVIRONMENT: ${{ vars.BACKUP_ENVIRONMENT || 'production' }}
49+
BACKUP_RETENTION_DAYS: ${{ github.event.inputs.retention_days || vars.BACKUP_RETENTION_DAYS || '35' }}
50+
BACKUP_SKIP_PRUNE: ${{ github.event.inputs.skip_prune == 'true' && '1' || '0' }}
51+
BACKUP_OUTPUT_DIR: ${{ github.workspace }}/backup-artifacts
52+
DATABASE_URL: ${{ secrets.BACKUP_DATABASE_URL }}
53+
BACKUP_KMS_KEY_ID: ${{ secrets.BACKUP_KMS_KEY_ID }}
54+
steps:
55+
- uses: actions/checkout@v4
56+
57+
- name: Validate backup configuration
58+
run: |
59+
for name in AWS_REGION BACKUP_BUCKET BACKUP_PREFIX DATABASE_URL BACKUP_KMS_KEY_ID; do
60+
if [ -z "${!name}" ]; then
61+
echo "::error::Missing required value: ${name}"
62+
exit 1
63+
fi
64+
done
65+
66+
- name: Install PostgreSQL client and jq
67+
run: |
68+
sudo apt-get update -qq
69+
sudo apt-get install -y postgresql-client jq
70+
71+
- name: Configure AWS credentials
72+
uses: aws-actions/configure-aws-credentials@v4
73+
with:
74+
role-to-assume: ${{ secrets.BACKUP_AWS_ROLE_ARN }}
75+
aws-region: ${{ vars.BACKUP_AWS_REGION }}
76+
77+
- name: Run backup
78+
run: ./scripts/ops/postgres-backup.sh
79+
80+
- name: Upload backup metadata artifact
81+
if: always()
82+
uses: actions/upload-artifact@v4
83+
with:
84+
name: postgres-backup-${{ github.run_id }}
85+
path: backup-artifacts/
86+
retention-days: 180
87+
88+
- name: Notify ops on backup failure
89+
if: failure() && secrets.OPS_ALERT_WEBHOOK_URL != ''
90+
env:
91+
OPS_ALERT_WEBHOOK_URL: ${{ secrets.OPS_ALERT_WEBHOOK_URL }}
92+
run: |
93+
payload="$(jq -n \
94+
--arg workflow "$GITHUB_WORKFLOW" \
95+
--arg runId "$GITHUB_RUN_ID" \
96+
--arg repository "$GITHUB_REPOSITORY" \
97+
--arg environment "$BACKUP_ENVIRONMENT" \
98+
--arg url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
99+
'{
100+
text: ("Postgres backup failed for " + $repository + " (" + $environment + ")."),
101+
workflow: $workflow,
102+
runId: $runId,
103+
repository: $repository,
104+
environment: $environment,
105+
runUrl: $url
106+
}')"
107+
curl -fsSL -X POST "$OPS_ALERT_WEBHOOK_URL" \
108+
-H 'Content-Type: application/json' \
109+
--data "$payload"

‎.github/workflows/recovery-drill.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
name: Recovery Drill
2+
3+
# Required GitHub configuration:
4+
# Repository variables:
5+
# BACKUP_AWS_REGION
6+
# BACKUP_BUCKET
7+
# BACKUP_PREFIX
8+
# BACKUP_ENVIRONMENT
9+
# DRILL_STELLAR_NETWORK e.g. testnet
10+
# DRILL_STELLAR_NETWORK_PASSPHRASE network passphrase for the drill target
11+
# DRILL_SOROBAN_RPC_URL Soroban RPC endpoint used for replay
12+
# DRILL_CONTRACT_ID Contract ID whose events must be replayed
13+
# DRILL_REINDEX_FROM_LEDGER Quarterly replay anchor ledger
14+
# Repository secrets:
15+
# RESTORE_AWS_ROLE_ARN Read-only or read-mostly restore role
16+
# OPS_ALERT_WEBHOOK_URL Optional alert destination
17+
18+
on:
19+
schedule:
20+
- cron: '0 10 1-7 1,4,7,10 1'
21+
workflow_dispatch:
22+
inputs:
23+
backup_object_key:
24+
description: 'Optional exact S3 object key to restore; defaults to latest dump'
25+
required: false
26+
type: string
27+
from_ledger:
28+
description: 'Optional replay anchor ledger; defaults to DRILL_REINDEX_FROM_LEDGER'
29+
required: false
30+
type: string
31+
32+
permissions:
33+
contents: read
34+
id-token: write
35+
36+
concurrency:
37+
group: recovery-drill-${{ vars.BACKUP_ENVIRONMENT || 'production' }}
38+
cancel-in-progress: false
39+
40+
jobs:
41+
restore-and-replay:
42+
name: Restore latest backup and replay indexer
43+
runs-on: ubuntu-latest
44+
timeout-minutes: 90
45+
services:
46+
postgres:
47+
image: postgres:16-alpine
48+
env:
49+
POSTGRES_DB: recovery_drill
50+
POSTGRES_USER: postgres
51+
POSTGRES_PASSWORD: postgres
52+
ports:
53+
- 5432:5432
54+
options: >-
55+
--health-cmd "pg_isready -U postgres -d recovery_drill"
56+
--health-interval 10s
57+
--health-timeout 5s
58+
--health-retries 5
59+
redis:
60+
image: redis:7-alpine
61+
ports:
62+
- 6379:6379
63+
options: >-
64+
--health-cmd "redis-cli ping"
65+
--health-interval 10s
66+
--health-timeout 5s
67+
--health-retries 5
68+
env:
69+
AWS_REGION: ${{ vars.BACKUP_AWS_REGION }}
70+
BACKUP_BUCKET: ${{ vars.BACKUP_BUCKET }}
71+
BACKUP_PREFIX: ${{ vars.BACKUP_PREFIX || 'postgres-backups' }}
72+
RESTORE_ENVIRONMENT: ${{ vars.BACKUP_ENVIRONMENT || 'production' }}
73+
BACKUP_OBJECT_KEY: ${{ github.event.inputs.backup_object_key }}
74+
RESTORE_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/recovery_drill
75+
DRILL_OUTPUT_DIR: ${{ github.workspace }}/drill-evidence
76+
DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/recovery_drill
77+
REDIS_URL: redis://127.0.0.1:6379/0
78+
STELLAR_NETWORK: ${{ vars.DRILL_STELLAR_NETWORK || 'testnet' }}
79+
STELLAR_NETWORK_PASSPHRASE: ${{ vars.DRILL_STELLAR_NETWORK_PASSPHRASE || 'Test SDF Network ; September 2015' }}
80+
SOROBAN_RPC_URL: ${{ vars.DRILL_SOROBAN_RPC_URL || 'https://soroban-testnet.stellar.org' }}
81+
CONTRACT_ID: ${{ vars.DRILL_CONTRACT_ID }}
82+
DRILL_REINDEX_FROM_LEDGER: ${{ github.event.inputs.from_ledger || vars.DRILL_REINDEX_FROM_LEDGER }}
83+
steps:
84+
- uses: actions/checkout@v4
85+
86+
- name: Validate drill configuration
87+
run: |
88+
for name in AWS_REGION BACKUP_BUCKET BACKUP_PREFIX RESTORE_DATABASE_URL CONTRACT_ID DRILL_REINDEX_FROM_LEDGER SOROBAN_RPC_URL STELLAR_NETWORK_PASSPHRASE; do
89+
if [ -z "${!name}" ]; then
90+
echo "::error::Missing required value: ${name}"
91+
exit 1
92+
fi
93+
done
94+
95+
- name: Install PostgreSQL client and jq
96+
run: |
97+
sudo apt-get update -qq
98+
sudo apt-get install -y postgresql-client jq
99+
100+
- name: Configure AWS credentials
101+
uses: aws-actions/configure-aws-credentials@v4
102+
with:
103+
role-to-assume: ${{ secrets.RESTORE_AWS_ROLE_ARN }}
104+
aws-region: ${{ vars.BACKUP_AWS_REGION }}
105+
106+
- name: Restore latest backup into fresh Postgres
107+
run: ./scripts/ops/postgres-restore-drill.sh
108+
109+
- name: Set up Node.js for replay
110+
uses: actions/setup-node@v4
111+
with:
112+
node-version: '22'
113+
cache: npm
114+
cache-dependency-path: backend/package-lock.json
115+
116+
- name: Install backend dependencies
117+
working-directory: backend
118+
run: npm ci
119+
120+
- name: Generate Prisma client
121+
working-directory: backend
122+
run: npx prisma generate
123+
124+
- name: Replay indexer from ledger anchor
125+
working-directory: backend
126+
run: |
127+
npm run ops:replay-indexer -- \
128+
--from-ledger "${DRILL_REINDEX_FROM_LEDGER}" \
129+
--network "${STELLAR_NETWORK}" \
130+
--output "${DRILL_OUTPUT_DIR}/indexer-replay.json"
131+
132+
- name: Upload drill evidence
133+
if: always()
134+
uses: actions/upload-artifact@v4
135+
with:
136+
name: recovery-drill-${{ github.run_id }}
137+
path: drill-evidence/
138+
retention-days: 365
139+
140+
- name: Notify ops on drill failure
141+
if: failure() && secrets.OPS_ALERT_WEBHOOK_URL != ''
142+
env:
143+
OPS_ALERT_WEBHOOK_URL: ${{ secrets.OPS_ALERT_WEBHOOK_URL }}
144+
run: |
145+
payload="$(jq -n \
146+
--arg workflow "$GITHUB_WORKFLOW" \
147+
--arg runId "$GITHUB_RUN_ID" \
148+
--arg repository "$GITHUB_REPOSITORY" \
149+
--arg environment "$RESTORE_ENVIRONMENT" \
150+
--arg url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
151+
'{
152+
text: ("Recovery drill failed for " + $repository + " (" + $environment + ")."),
153+
workflow: $workflow,
154+
runId: $runId,
155+
repository: $repository,
156+
environment: $environment,
157+
runUrl: $url
158+
}')"
159+
curl -fsSL -X POST "$OPS_ALERT_WEBHOOK_URL" \
160+
-H 'Content-Type: application/json' \
161+
--data "$payload"

‎backend/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@
3434
"error-catalog:export": "ts-node scripts/export-error-catalog.ts",
3535
"env:example:generate": "ts-node scripts/generate-env-example.ts",
3636
"env:example:check": "ts-node scripts/check-env-example.ts",
37-
"secrets:generate:jwt": "ts-node scripts/generate-jwt-key.ts"
37+
"secrets:generate:jwt": "ts-node scripts/generate-jwt-key.ts",
38+
"ops:replay-indexer": "ts-node scripts/replay-indexer.ts"
3839
},
3940
"dependencies": {
4041
"@apollo/server": "^4.13.0",

0 commit comments

Comments
 (0)