ACA Off-Hours Scale (Mon-Fri 07-20 Europe/Berlin) #68
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ACA Off-Hours Scale (Mon-Fri 07-20 Europe/Berlin) | |
| # ADR-022/023: re-enable the off-hours schedule for cost optimisation. | |
| # ADR-027: extend it to the full EDC stack. ADR-024 provisioned the 8 EDC | |
| # apps at min=1 (24/7) but this schedule still only covered the 7 "healthy" | |
| # apps — so the EDC stack billed nights + weekends, the cost surge flagged by | |
| # FinOps in May 2026. The schedule now scales all 15 Container Apps. | |
| # | |
| # INF-STG-EU_EHDS no longer requires 24/7 — the public ehds.mabu.red demo is | |
| # documented as a weekday-hours service in the README, so off-hours | |
| # scale-down is non-disruptive. Stop saves ~€250-300/mo by halting every app | |
| # overnight + weekends (apps run ~65h/week instead of 168h). | |
| # | |
| # The schedule was originally introduced by ADR-016 and disabled by ADR-018 | |
| # when the subscription changed; ADR-022/023 reinstated it under the | |
| # Workaround-B (Postgres-on-ACA) model: there's no Microsoft.DBforPostgreSQL | |
| # Flexible Server to start/stop, so postgres scale-to-0 is a Container App | |
| # update like everything else. | |
| on: | |
| schedule: | |
| # Cron is UTC; CEST (summer) = UTC+2, CET (winter) = UTC+1. | |
| # Operating window is "07:00–20:00 Europe/Berlin local"; we set the cron | |
| # at the CEST boundaries (05:00/18:00 UTC) and accept ±1 h drift in | |
| # winter, same as the original ADR-016 schedule. | |
| - cron: "0 5 * * 1-5" # start: Mon–Fri 07:00 Europe/Berlin (CEST) | |
| - cron: "0 18 * * *" # stop: daily 20:00 Europe/Berlin (CEST) | |
| workflow_dispatch: | |
| inputs: | |
| action: | |
| description: "Action to run" | |
| type: choice | |
| options: [start, stop] | |
| required: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| concurrency: | |
| group: "azure-schedule" | |
| cancel-in-progress: false | |
| env: | |
| RESOURCE_GROUP: rg-mvhd-dev | |
| # Workaround B (ADR-018): Postgres runs as an ACA Container App | |
| # (mvhd-postgres), not a Flexible Server. Stop = scale to 0/0; start = | |
| # scale to 1/1 like every other app. | |
| PG_APP: mvhd-postgres | |
| jobs: | |
| determine-action: | |
| name: Determine Action | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| outputs: | |
| action: ${{ steps.decide.outputs.action }} | |
| steps: | |
| - id: decide | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "action=${{ inputs.action }}" >> "$GITHUB_OUTPUT" | |
| elif [ "${{ github.event.schedule }}" = "0 5 * * 1-5" ]; then | |
| # Skip the morning scale-up on Berlin public holidays — otherwise a | |
| # weekday holiday (e.g. Ostermontag, Christi Himmelfahrt) would run | |
| # the full stack 07:00–20:00 for nobody. subdiv='BE' = Land Berlin | |
| # (includes Internationaler Frauentag, which most states lack). | |
| # The evening stop cron still runs, so the stack stays at 0. | |
| TODAY=$(TZ=Europe/Berlin date +%F) | |
| python3 -m pip install --quiet holidays | |
| HOL=$(python3 -c "import holidays, datetime as t; print(1 if t.date.fromisoformat('$TODAY') in holidays.Germany(subdiv='BE') else 0)") | |
| if [ "$HOL" = "1" ]; then | |
| echo "Berlin public holiday ($TODAY) — skipping scale-up" | |
| echo "action=noop" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "action=start" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| echo "action=stop" >> "$GITHUB_OUTPUT" | |
| fi | |
| stop: | |
| name: Scale Down (Stop) | |
| needs: determine-action | |
| if: needs.determine-action.outputs.action == 'stop' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Azure Login (OIDC) | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Scale all Container Apps to 0 | |
| run: | | |
| # All 15 Container Apps are scaled to 0 off-hours (ADR-027). | |
| # - 7 core apps: postgres, neo4j, keycloak, vault, proxy, ui, | |
| # catalog-enricher. Postgres is included because Workaround B | |
| # (ADR-018) made it an ACA app, not a Flexible Server. | |
| # - 8 EDC apps: ADR-024 provisioned the full EDC stack at min=1. | |
| # ADR-027 adds them here so they no longer bill nights + weekends. | |
| # Stop order is irrelevant; start order is not (see the start job). | |
| APPS=( | |
| mvhd-postgres | |
| mvhd-neo4j | |
| mvhd-keycloak | |
| mvhd-vault | |
| mvhd-neo4j-proxy | |
| mvhd-ui | |
| mvhd-catalog-enricher | |
| mvhd-nats | |
| mvhd-controlplane | |
| mvhd-dp-fhir | |
| mvhd-dp-omop | |
| mvhd-identityhub | |
| mvhd-issuerservice | |
| mvhd-tenant-mgr | |
| mvhd-provision-mgr | |
| ) | |
| echo "=== Scaling ${#APPS[@]} Container Apps to min=0 (scale-to-zero) ===" | |
| # ACA requires maxReplicas >= 1; scale-to-zero is min=0 + max=1, not | |
| # min=0 + max=0. The old --max-replicas 0 was rejected by Azure on | |
| # every run ("--max-replicas must be in the range [1,1000]") and the | |
| # error was swallowed by `|| echo ::warning::`, so the job went green | |
| # while nothing ever scaled down — apps billed 24/7 (issue #92). | |
| # The start job restores each app's real max (proxy 1/2, ui 1/3, rest | |
| # 1/1) on the way up, so capping max=1 here is harmless. | |
| FAILED=0 | |
| for app in "${APPS[@]}"; do | |
| echo " $app -> 0/1" | |
| if ! az containerapp update \ | |
| --name "$app" --resource-group "$RESOURCE_GROUP" \ | |
| --min-replicas 0 --max-replicas 1 \ | |
| -o none; then | |
| echo "::error::Failed to scale $app to min=0" | |
| FAILED=1 | |
| continue | |
| fi | |
| # Post-condition: a green check must mean it actually scaled. | |
| min=$(az containerapp show --name "$app" \ | |
| --resource-group "$RESOURCE_GROUP" \ | |
| --query "properties.template.scale.minReplicas" -o tsv) | |
| if [ "$min" != "0" ]; then | |
| echo "::error::$app minReplicas=$min after update (expected 0)" | |
| FAILED=1 | |
| fi | |
| done | |
| if [ "$FAILED" -ne 0 ]; then | |
| echo "::error::One or more apps failed to scale to zero — see logs above" | |
| exit 1 | |
| fi | |
| echo "All ${#APPS[@]} apps confirmed at min=0." | |
| - name: Stop summary | |
| if: always() | |
| run: | | |
| echo "## Off-Hours Scale-Down" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Time:** $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Status:** 15 Container Apps (7 core + 8 EDC) scaled to 0" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Next start:** Monday-Friday 05:00 UTC (07:00 Europe/Berlin CEST)" >> "$GITHUB_STEP_SUMMARY" | |
| start: | |
| name: Scale Up (Start) | |
| needs: determine-action | |
| if: needs.determine-action.outputs.action == 'start' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Azure Login (OIDC) | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Start Postgres first | |
| run: | | |
| # Postgres must be reachable before Keycloak's pod starts — | |
| # Keycloak crash-loops if its DB is unavailable on boot. | |
| echo " $PG_APP -> 1/1" | |
| az containerapp update \ | |
| --name "$PG_APP" --resource-group "$RESOURCE_GROUP" \ | |
| --min-replicas 1 --max-replicas 1 \ | |
| -o none || echo "::warning::Failed to scale $PG_APP" | |
| echo "Waiting 20s for Postgres to accept connections..." | |
| sleep 20 | |
| - name: Restore Container App scale profiles | |
| run: | | |
| echo "=== Restoring Container App replica profiles ===" | |
| # min=1 max=1 (singleton stateful / small services) | |
| for app in mvhd-neo4j mvhd-keycloak mvhd-vault mvhd-catalog-enricher; do | |
| echo " $app -> 1/1" | |
| az containerapp update \ | |
| --name "$app" --resource-group "$RESOURCE_GROUP" \ | |
| --min-replicas 1 --max-replicas 1 \ | |
| -o none || echo "::warning::Failed to scale $app" | |
| done | |
| # min=1 max=2 (auto-scalable read services) | |
| for app in mvhd-neo4j-proxy; do | |
| echo " $app -> 1/2" | |
| az containerapp update \ | |
| --name "$app" --resource-group "$RESOURCE_GROUP" \ | |
| --min-replicas 1 --max-replicas 2 \ | |
| -o none || echo "::warning::Failed to scale $app" | |
| done | |
| # UI: min=1 max=3 | |
| echo " mvhd-ui -> 1/3" | |
| az containerapp update \ | |
| --name mvhd-ui --resource-group "$RESOURCE_GROUP" \ | |
| --min-replicas 1 --max-replicas 3 \ | |
| -o none || echo "::warning::Failed to scale mvhd-ui" | |
| # NATS — started here with the core apps so JetStream is ready | |
| # before the EDC controlplane comes up in the next step. | |
| echo " mvhd-nats -> 1/1" | |
| az containerapp update \ | |
| --name mvhd-nats --resource-group "$RESOURCE_GROUP" \ | |
| --min-replicas 1 --max-replicas 1 \ | |
| -o none || echo "::warning::Failed to scale mvhd-nats" | |
| # The 7 remaining EDC apps (controlplane, data planes, identity hub, | |
| # issuer service, CFM tenant/provision managers) are restored after | |
| # the stateful wait below — see "Restore EDC stack" (ADR-024/027). | |
| - name: Wait for stateful services to be reachable | |
| run: | | |
| echo "Waiting 75s for Neo4j + Vault pods to start..." | |
| sleep 75 | |
| - name: Restore EDC stack (ADR-024) | |
| run: | | |
| # ADR-024 provisions the full EDC stack at min=1 during work hours; | |
| # ADR-027 scales it to 0 off-hours. Postgres (started first) and | |
| # NATS (started with the core apps) are reachable by now. The | |
| # controlplane needs both; the data planes, identity hub, issuer | |
| # service and CFM managers need Postgres. | |
| echo "=== Restoring 7 EDC Container Apps to 1/1 ===" | |
| for app in mvhd-controlplane mvhd-dp-fhir mvhd-dp-omop \ | |
| mvhd-identityhub mvhd-issuerservice \ | |
| mvhd-tenant-mgr mvhd-provision-mgr; do | |
| echo " $app -> 1/1" | |
| az containerapp update \ | |
| --name "$app" --resource-group "$RESOURCE_GROUP" \ | |
| --min-replicas 1 --max-replicas 1 \ | |
| -o none || echo "::warning::Failed to scale $app" | |
| done | |
| - name: Re-bootstrap Vault (in-memory gotcha #1) | |
| run: | | |
| echo "=== Re-running bootstrap job (Vault secrets + keys) ===" | |
| az containerapp job start \ | |
| --name mvhd-vault-bootstrap --resource-group "$RESOURCE_GROUP" \ | |
| -o none || echo "::warning::Vault bootstrap job start failed" | |
| # Neo4j is now persistent on Azure Files (ADR-017). The seed job is no | |
| # longer required on every start — data survives the 0/0 → 1/1 cycle. | |
| # If the database is empty (fresh storage account) the one-time bootstrap | |
| # workflow `bootstrap-stateful.yml` should be run manually instead. | |
| - name: Verify UI is reachable | |
| id: verify | |
| run: | | |
| UI_FQDN=$(az containerapp show --name mvhd-ui \ | |
| --resource-group "$RESOURCE_GROUP" \ | |
| --query "properties.configuration.ingress.fqdn" -o tsv) | |
| echo "ui_url=https://${UI_FQDN}" >> "$GITHUB_OUTPUT" | |
| echo "Checking https://${UI_FQDN}..." | |
| for i in $(seq 1 30); do | |
| code=$(curl -sf -o /dev/null -w "%{http_code}" "https://${UI_FQDN}" 2>/dev/null || echo "000") | |
| if [ "$code" = "200" ]; then | |
| echo "UI is healthy (HTTP 200) after $((i * 10))s" | |
| exit 0 | |
| fi | |
| echo " attempt $i: HTTP $code" | |
| sleep 10 | |
| done | |
| echo "::warning::UI did not return 200 within 300s — may need manual check" | |
| - name: Start summary | |
| if: always() | |
| run: | | |
| echo "## On-Hours Scale-Up" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Time:** $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Target:** ${{ steps.verify.outputs.ui_url }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Status:** PostgreSQL started, 15 Container Apps scaled up (7 core + 8 EDC), Vault re-bootstrapped (Neo4j data persists via Azure Files — see ADR-017)" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Next stop:** Today 18:00 UTC (20:00 Europe/Berlin CEST)" >> "$GITHUB_STEP_SUMMARY" |