Merge branch 'main' into feat/issue-152-matching-engine #30

Workflow file for this run

.github/workflows/secret-rotation.yml at a5f968e

	# .github/workflows/secret-rotation.yml
	#
	# Automated External Secret Rotation with Zero-Downtime Rollout
	#
	# Rotates the following secrets on a quarterly schedule (or manually via
	# workflow_dispatch), validates health post-rotation, and automatically
	# rolls back if health checks fail:
	#
	# - DATABASE_URL
	# - JWT_SECRET
	# - WEBHOOK_SIGNING_SECRET
	# - ADMIN_API_KEY
	# - RECURRING_SIGNER_SECRET
	#
	# Workflow phases:
	# 1. Pre-rotation validation — verify all secrets exist in the manager
	# 2. Generate new values — cryptographically random replacements
	# 3. Update secrets manager — write new values to AWS Secrets Manager
	# 4. Trigger ESO force-sync — annotate ExternalSecret for immediate refresh
	# 5. Rolling restart — kubectl rollout restart deployment/backend
	# 6. Health validation — poll /health/ready until healthy or timeout
	# 7. Auto-rollback (on fail) — restore previous secret values
	# 8. Audit logging — write rotation result to secret_rotations table
	#
	# Schedule: 1st day of every quarter (Jan 1, Apr 1, Jul 1, Oct 1) at 02:00 UTC.

	name: Automated Secret Rotation

	on:
	schedule:
	- cron: "0 2 1 1,4,7,10 *"
	workflow_dispatch:
	inputs:
	secrets_to_rotate:
	description: >-
	Comma-separated list of secrets to rotate.
	Defaults to all five: DATABASE_URL,JWT_SECRET,WEBHOOK_SIGNING_SECRET,ADMIN_API_KEY,RECURRING_SIGNER_SECRET
	required: false
	type: string
	default: "DATABASE_URL,JWT_SECRET,WEBHOOK_SIGNING_SECRET,ADMIN_API_KEY,RECURRING_SIGNER_SECRET"
	skip_health_check:
	description: "Skip the post-rotation health check (use with caution)"
	required: false
	type: boolean
	default: false
	skip_rollback:
	description: "Do not auto-rollback on health check failure (debugging use only)"
	required: false
	type: boolean
	default: false

	env:
	AWS_REGION: us-east-1
	SECRETS_MANAGER_PATH: stellar-indigopay/prod
	K8S_NAMESPACE: stellar-indigopay
	BACKEND_DEPLOYMENT: backend
	HEALTH_CHECK_URL: "https://api.stellar-indigopay.com/health/ready"
	HEALTH_CHECK_TIMEOUT_SECONDS: 300
	HEALTH_CHECK_INTERVAL_SECONDS: 10
	ROTATION_ID: ""

	jobs:
	secret-rotation:
	name: "Rotate secrets and validate health"
	runs-on: ubuntu-latest
	timeout-minutes: 30
	permissions:
	id-token: write
	contents: write

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	# ── Phase 1: Pre-rotation validation ───────────────────────────────
	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	aws-region: ${{ env.AWS_REGION }}

	- name: Parse secrets to rotate
	id: parse-secrets
	env:
	INPUT_SECRETS: ${{ github.event.inputs.secrets_to_rotate \|\| 'DATABASE_URL,JWT_SECRET,WEBHOOK_SIGNING_SECRET,ADMIN_API_KEY,RECURRING_SIGNER_SECRET' }}
	run: \|
	# Convert comma-separated list to JSON array for the audit log
	IFS=',' read -ra SECRETS <<< "$INPUT_SECRETS"
	JSON_ARRAY="["
	FIRST=true
	for S in "${SECRETS[@]}"; do
	S=$(echo "$S" \| xargs) # trim whitespace
	if [ "$FIRST" = true ]; then
	JSON_ARRAY="[\"$S\""
	FIRST=false
	else
	JSON_ARRAY="$JSON_ARRAY,\"$S\""
	fi
	done
	JSON_ARRAY="$JSON_ARRAY]"
	echo "secrets_json=$JSON_ARRAY" >> $GITHUB_OUTPUT
	echo "secrets_list=${INPUT_SECRETS//,/ }" >> $GITHUB_OUTPUT

	- name: Pre-rotation validation — verify secrets exist
	id: pre-rotation
	env:
	SECRETS_LIST: ${{ steps.parse-secrets.outputs.secrets_list }}
	run: \|
	set -euo pipefail
	echo "::group::Pre-rotation validation"
	for SECRET in $SECRETS_LIST; do
	echo "Checking that $SECRET exists in AWS Secrets Manager..."
	# Fetch the current secret and store for potential rollback
	CURRENT_VALUE=$(aws secretsmanager get-secret-value \
	--secret-id "${SECRETS_MANAGER_PATH}" \
	--query "SecretString" \
	--output text 2>/dev/null \| python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('${SECRET}', 'MISSING'))" 2>/dev/null \|\| echo "MISSING")

	if [ "$CURRENT_VALUE" = "MISSING" ]; then
	echo "::error::Secret ${SECRET} not found in ${SECRETS_MANAGER_PATH}"
	exit 1
	fi

	# Save old value for rollback
	echo "OLD_${SECRET}=${CURRENT_VALUE}" >> $GITHUB_ENV
	echo "✅ ${SECRET} exists"
	done
	echo "::endgroup::"

	# ── Phase 2: Generate new secret values ────────────────────────────
	- name: Generate new secret values
	id: generate-secrets
	env:
	SECRETS_LIST: ${{ steps.parse-secrets.outputs.secrets_list }}
	run: \|
	set -euo pipefail
	echo "::group::Generating new secret values"
	for SECRET in $SECRETS_LIST; do
	case "$SECRET" in
	DATABASE_URL)
	# Preserve the connection structure; only rotate the password
	# component. The actual DB password rotation must be done
	# separately via the Postgres ALTER ROLE flow. Here we rotate
	# the connection string to point to the new password.
	# In practice, DATABASE_URL rotation should be paired with a
	# prior DB credential rotation step.
	CURRENT_DB_URL="${OLD_DATABASE_URL}"
	# Generate a strong 32-char alphanumeric password
	NEW_PASS=$(openssl rand -base64 24 \| tr -dc 'a-zA-Z0-9' \| head -c 32)
	# Replace the password in the URL. Assumes format: postgres://user:PASS@host:port/db
	NEW_DB_URL=$(echo "$CURRENT_DB_URL" \| sed -E "s\|://([^:]+):([^@]+)@\|://\1:${NEW_PASS}@\|")
	echo "NEW_${SECRET}=${NEW_DB_URL}" >> $GITHUB_ENV
	;;
	JWT_SECRET)
	NEW_VALUE=$(openssl rand -base64 48)
	echo "NEW_${SECRET}=${NEW_VALUE}" >> $GITHUB_ENV
	;;
	WEBHOOK_SIGNING_SECRET)
	NEW_VALUE=$(openssl rand -hex 32)
	echo "NEW_${SECRET}=${NEW_VALUE}" >> $GITHUB_ENV
	;;
	ADMIN_API_KEY)
	NEW_VALUE="ip_admin_$(openssl rand -hex 24)"
	echo "NEW_${SECRET}=${NEW_VALUE}" >> $GITHUB_ENV
	;;
	RECURRING_SIGNER_SECRET)
	NEW_VALUE=$(openssl rand -base64 32)
	echo "NEW_${SECRET}=${NEW_VALUE}" >> $GITHUB_ENV
	;;
	*)
	echo "::warning::Unknown secret ${SECRET}, generating generic 48-byte value"
	NEW_VALUE=$(openssl rand -base64 48)
	echo "NEW_${SECRET}=${NEW_VALUE}" >> $GITHUB_ENV
	;;
	esac
	echo "✅ Generated new value for ${SECRET}"
	done
	echo "::endgroup::"

	# ── Phase 3: Update secrets in AWS Secrets Manager ─────────────────
	- name: Update secrets in AWS Secrets Manager
	id: update-secrets
	env:
	SECRETS_LIST: ${{ steps.parse-secrets.outputs.secrets_list }}
	run: \|
	set -euo pipefail
	echo "::group::Updating secrets in AWS Secrets Manager"

	# Fetch the current secret JSON
	CURRENT_SECRET_JSON=$(aws secretsmanager get-secret-value \
	--secret-id "${SECRETS_MANAGER_PATH}" \
	--query "SecretString" \
	--output text)

	# Build the updated JSON
	UPDATED_JSON=$(echo "$CURRENT_SECRET_JSON" \| python3 -c "
	import json, os, sys
Check failure on line 198 in .github/workflows/secret-rotation.yml View workflow run for this annotation GitHub Actions / .github/workflows/secret-rotation.yml Invalid workflow file `You have an error in your yaml syntax on line 198`
	secrets = json.load(sys.stdin)
	secrets_list = os.environ.get('SECRETS_LIST', '').split()

	for secret in secrets_list:
	new_val = os.environ.get(f'NEW_{secret}')
	if new_val:
	print(f' Rotating {secret}...', file=sys.stderr)
	secrets[secret.lower() if secret != secret.upper() else secret] = new_val
	else:
	print(f' WARNING: No new value found for {secret}', file=sys.stderr)

	print(json.dumps(secrets, indent=2))
	")

	# Write the updated secret
	aws secretsmanager update-secret \
	--secret-id "${SECRETS_MANAGER_PATH}" \
	--secret-string "$UPDATED_JSON"

	echo "✅ Secrets updated in AWS Secrets Manager"
	echo "::endgroup::"

	# ── Phase 4: Trigger ESO force-sync annotation ─────────────────────
	- name: Configure kubeconfig
	uses: azure/setup-kubectl@v4
	with:
	version: "v1.29"

	- name: Force External Secrets Operator sync
	id: eso-force-sync
	run: \|
	set -euo pipefail
	echo "::group::Triggering ESO force-sync"
	TIMESTAMP=$(date +%s)
	kubectl annotate externalsecret stellar-indigopay-secrets \
	--namespace "${K8S_NAMESPACE}" \
	force-sync="${TIMESTAMP}" \
	--overwrite
	echo "eso_force_sync_triggered_at=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
	echo "✅ ESO force-sync annotated with timestamp ${TIMESTAMP}"
	echo "::endgroup::"

	- name: Wait for ESO to sync the new secrets
	run: \|
	set -euo pipefail
	echo "::group::Waiting for ESO to sync secrets"
	# ESO typically refreshes within seconds; give it 30s to propagate
	sleep 30
	echo "::endgroup::"

	# ── Phase 5: Rolling restart ──────────────────────────────────────
	- name: Rolling restart of backend deployment
	id: rolling-restart
	run: \|
	set -euo pipefail
	echo "::group::Rolling restart"
	kubectl rollout restart "deployment/${BACKEND_DEPLOYMENT}" \
	--namespace "${K8S_NAMESPACE}"
	echo "✅ Rollout restart initiated"
	echo "rolling_restart_started_at=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
	echo "::endgroup::"

	- name: Wait for rollout to complete
	id: rollout-status
	run: \|
	set -euo pipefail
	echo "::group::Waiting for rollout to complete"
	kubectl rollout status "deployment/${BACKEND_DEPLOYMENT}" \
	--namespace "${K8S_NAMESPACE}" \
	--timeout=5m
	echo "✅ Rollout completed successfully"
	echo "rolling_restart_completed_at=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
	echo "::endgroup::"

	# ── Phase 6: Health validation ─────────────────────────────────────
	- name: Health check validation
	id: health-check
	if: ${{ !github.event.inputs.skip_health_check }}
	env:
	SKIP_ROLLBACK: ${{ github.event.inputs.skip_rollback }}
	run: \|
	set -euo pipefail
	echo "::group::Health check validation"

	DEADLINE=$(( $(date +%s) + HEALTH_CHECK_TIMEOUT_SECONDS ))
	HEALTHY=false

	while [ $(date +%s) -lt $DEADLINE ]; do
	HTTP_CODE=$(curl -s -o /tmp/health_response.json -w "%{http_code}" \
	"${HEALTH_CHECK_URL}" 2>/dev/null \|\| echo "000")

	if [ "$HTTP_CODE" = "200" ]; then
	HEALTHY=true
	echo "✅ Health check passed (HTTP 200)"
	echo "health_check_passed=true" >> $GITHUB_OUTPUT
	cat /tmp/health_response.json
	break
	fi

	echo "⏳ Waiting... health endpoint returned HTTP ${HTTP_CODE}"
	sleep "${HEALTH_CHECK_INTERVAL_SECONDS}"
	done

	if [ "$HEALTHY" = false ]; then
	echo "::error::Health check timed out after ${HEALTH_CHECK_TIMEOUT_SECONDS}s"
	echo "health_check_passed=false" >> $GITHUB_OUTPUT
	if [ -f /tmp/health_response.json ]; then
	echo "Last response:"
	cat /tmp/health_response.json
	fi

	# Auto-rollback unless explicitly skipped
	if [ "${SKIP_ROLLBACK}" != "true" ]; then
	echo "rollback_needed=true" >> $GITHUB_OUTPUT
	fi
	fi

	echo "::endgroup::"

	# ── Phase 7: Auto-rollback (on health check failure) ───────────────
	- name: Rollback secrets (auto-rollback on failed health check)
	if: ${{ steps.health-check.outputs.rollback_needed == 'true' }}
	id: rollback
	env:
	SECRETS_LIST: ${{ steps.parse-secrets.outputs.secrets_list }}
	run: \|
	set -euo pipefail
	echo "::group::Rolling back secrets to previous values"

	echo "Health check failed — initiating automatic rollback..."

	# Fetch the current secret JSON
	CURRENT_SECRET_JSON=$(aws secretsmanager get-secret-value \
	--secret-id "${SECRETS_MANAGER_PATH}" \
	--query "SecretString" \
	--output text)

	# Restore old values
	ROLLBACK_JSON=$(echo "$CURRENT_SECRET_JSON" \| python3 -c "
	import json, os, sys
	secrets = json.load(sys.stdin)
	secrets_list = os.environ.get('SECRETS_LIST', '').split()

	for secret in secrets_list:
	old_val = os.environ.get(f'OLD_{secret}')
	if old_val:
	print(f' Restoring {secret}...', file=sys.stderr)
	secrets[secret.lower() if secret != secret.upper() else secret] = old_val
	else:
	print(f' WARNING: No old value found for {secret}', file=sys.stderr)

	print(json.dumps(secrets, indent=2))
	")

	# Write the rollback values
	aws secretsmanager update-secret \
	--secret-id "${SECRETS_MANAGER_PATH}" \
	--secret-string "$ROLLBACK_JSON"

	echo "✅ Secrets rolled back to previous values"

	# Trigger another ESO force-sync to pick up the rollback
	TIMESTAMP=$(date +%s)
	kubectl annotate externalsecret stellar-indigopay-secrets \
	--namespace "${K8S_NAMESPACE}" \
	force-sync="${TIMESTAMP}" \
	--overwrite

	echo "✅ ESO force-sync triggered for rollback"

	# Restart pods again to pick up old secrets
	kubectl rollout restart "deployment/${BACKEND_DEPLOYMENT}" \
	--namespace "${K8S_NAMESPACE}"

	kubectl rollout status "deployment/${BACKEND_DEPLOYMENT}" \
	--namespace "${K8S_NAMESPACE}" \
	--timeout=5m

	echo "✅ Rollback rollout completed"
	echo "::endgroup::"

	# ── Phase 8: Write audit log via backend API ──────────────────────
	- name: Record rotation in audit log
	id: audit-log
	env:
	API_BASE_URL: "https://api.stellar-indigopay.com"
	ADMIN_API_KEY_VAR: ${{ secrets.ADMIN_API_KEY }}
	SECRETS_LIST: ${{ steps.parse-secrets.outputs.secrets_list }}
	SECRETS_JSON: ${{ steps.parse-secrets.outputs.secrets_json }}
	run: \|
	set -euo pipefail
	echo "::group::Recording rotation in audit log"

	HEALTH_PASSED="${{ steps.health-check.outputs.health_check_passed }}"
	ROLLBACK_NEEDED="${{ steps.health-check.outputs.rollback_needed }}"
	OVERALL_STATUS="completed"

	if [ "${ROLLBACK_NEEDED}" = "true" ]; then
	OVERALL_STATUS="rolled_back"
	elif [ "${HEALTH_PASSED}" = "false" ]; then
	OVERALL_STATUS="failed"
	fi

	# Build the payload
	PAYLOAD=$(cat << EOF
	{
	"workflowRunId": "${{ github.run_id }}",
	"triggeredBy": "${{ github.event_name }}",
	"secretsRotated": ${SECRETS_JSON},
	"overallStatus": "${OVERALL_STATUS}",
	"healthCheckPassed": ${HEALTH_PASSED:-null},
	"rollbackTriggered": ${ROLLBACK_NEEDED:-false},
	"rollbackReason": $(if [ "${ROLLBACK_NEEDED}" = "true" ]; then echo '"Health check failed after rotation — auto-rollback triggered"'; else echo 'null'; fi),
	"metadata": {
	"githubRunId": "${{ github.run_id }}",
	"githubRunNumber": "${{ github.run_number }}",
	"githubActor": "${{ github.actor }}",
	"githubRef": "${{ github.ref }}",
	"workflowUrl": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",
	"esoForceSyncTriggeredAt": "${{ steps.eso-force-sync.outputs.eso_force_sync_triggered_at }}",
	"rollingRestartStartedAt": "${{ steps.rolling-restart.outputs.rolling_restart_started_at }}",
	"rollingRestartCompletedAt": "${{ steps.rollout-status.outputs.rolling_restart_completed_at }}"
	}
	}
	EOF
	)

	# Call the admin API to record the rotation.
	# adminRequired middleware supports both JWT (Authorization: Bearer)
	# and API key (X-Admin-Key header) authentication. Use X-Admin-Key
	# since the workflow has the ADMIN_API_KEY secret, not a JWT.
	HTTP_CODE=$(curl -s -o /tmp/audit_response.json -w "%{http_code}" \
	-X POST "${API_BASE_URL}/api/admin/secret-rotations" \
	-H "Content-Type: application/json" \
	-H "X-Admin-Key: ${ADMIN_API_KEY_VAR}" \
	-d "$PAYLOAD" 2>/dev/null \|\| echo "000")

	if [ "$HTTP_CODE" = "201" ]; then
	echo "✅ Rotation audit log recorded successfully"
	ROTATION_ID=$(python3 -c "import json; d=json.load(open('/tmp/audit_response.json')); print(d.get('data', {}).get('id', 'unknown'))")
	echo "rotation_id=${ROTATION_ID}" >> $GITHUB_OUTPUT
	else
	echo "::warning::Failed to record rotation audit log (HTTP ${HTTP_CODE})"
	if [ -f /tmp/audit_response.json ]; then
	echo "Response:"
	cat /tmp/audit_response.json
	fi
	fi

	echo "::endgroup::"

	# ── Notifications ──────────────────────────────────────────────────
	- name: Notify on success
	if: success()
	uses: actions/github-script@v7
	with:
	script: \|
	const secrets = JSON.parse('${{ steps.parse-secrets.outputs.secrets_json }}');
	const rotationId = '${{ steps.audit-log.outputs.rotation_id }}';
	const message = `✅ Secret rotation completed successfully\n\n` +
	`Secrets rotated: ${secrets.join(', ')}\n` +
	`Rotation ID: ${rotationId \|\| 'N/A'}\n` +
	`Run: ${context.serverUrl}/${context.repo}/actions/runs/${context.runId}`;
	core.notice(message);

	- name: Open issue on failure
	if: failure()
	uses: actions/github-script@v7
	with:
	script: \|
	const secrets = JSON.parse('${{ steps.parse-secrets.outputs.secrets_json }}');
	const rollbackTriggered = '${{ steps.health-check.outputs.rollback_needed }}' === 'true';
	const title = rollbackTriggered
	? '🚨 Secret Rotation Failed — Auto-Rollback Triggered'
	: '🚨 Secret Rotation Failed';
	const body = `Secrets: ${secrets.join(', ')}\n` +
	`Rollback triggered: ${rollbackTriggered}\n` +
	`Run: ${context.serverUrl}/${context.repo}/actions/runs/${context.runId}\n` +
	`Time: ${new Date().toISOString()}\n\n` +
	`Investigate immediately — this means the secret rotation was interrupted.`;
	await github.rest.issues.create({
	owner: context.repo.owner,
	repo: context.repo.repo,
	title,
	body,
	labels: ['bug', 'devops', 'security', 'oncall'],
	});

Navigation Menu

Merge branch 'main' into feat/issue-152-matching-engine #30

Workflow file

Merge branch 'main' into feat/issue-152-matching-engine #30

Uh oh!

Workflow file for this run

GitHub Actions / .github/workflows/secret-rotation.yml