staging-deploy.yml

name: Staging Deploy

# Runs on every push to staging so the staging deployment tracks staging HEAD.
# The concurrency group is shared with staging-db-bootstrap.yml, so a deploy
# never runs migrations against a branch a bootstrap is mid-mirror on. The
# group serializes the two but does not order them, so the bootstrap ends by
# dispatching a deploy of its own rather than trusting that a queued one lands
# after the mirror.
#
# GitHub keeps at most one run pending per group, so a newer push supersedes a
# still-queued deploy. That is intended -- the newest commit is what staging
# should run -- but it means only the tip is guaranteed a deploy run. An
# intermediate commit can be left with a superseded run and is then not
# promotable by sha, which is the promotion preflight refusing a commit that
# never reached staging.
on:
  push:
    branches: [staging]
  workflow_dispatch: {}

concurrency:
  group: staging-environment
  cancel-in-progress: false

jobs:
  deploy:
    runs-on: ubuntu-latest
    environment:
      name: staging
      url: https://abundant-ai-staging--oddish-staging-api.modal.run
    timeout-minutes: 45
    permissions:
      contents: read
      packages: read
    # Python 3.13, uv, and a warm uv cache live in the base image --
    # refreshed weekly by .github/workflows/ci-base-image.yml.
    container:
      image: ghcr.io/abundant-ai/oddish-ci-base:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    env:
      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
      MODAL_ENVIRONMENT: staging
      MODAL_APP_NAME: oddish-staging
      MODAL_SECRET_ENVIRONMENT: main
      # Hold LLM trials until a task's nop/oracle baselines validate it (mirrors
      # modal-deploy.yml's prod setting).
      ODDISH_GATE_LLM_ON_BASELINES: "1"
      # GKE (TPU) execution backend, mirroring modal-deploy.yml's prod block.
      # backend/modal_app.py only bakes env vars prefixed ODDISH_GKE_ into the
      # deployed image (its ENV_VARS dict filters on that prefix); an unprefixed
      # name here would be silently dropped rather than erroring, so the prefix
      # on every one of these is load-bearing, not decorative.
      ODDISH_GKE_ENABLED: "true"
      ODDISH_GKE_PROJECT_ID: "abundant-default"
      ODDISH_GKE_REGION: "us-east5"
      ODDISH_GKE_REGISTRY_LOCATION: "us-east5"
      ODDISH_GKE_REGISTRY_NAME: "oddish-envs"
      ODDISH_GKE_AUTO_PROVISION_CLUSTER: "true"
      ODDISH_GKE_AUTO_BUILD_MISSING_IMAGE: "true"
      ODDISH_GKE_IDLE_CLUSTER_TTL_HOURS: "1"
      # Ephemeral Harbor CPU sandboxes. Credentials and the SSH private key are
      # attached from dedicated Modal secrets; only platform-owned launch
      # coordinates are declared in the deployment environment.
      ODDISH_EC2_ENABLED: "true"
      ODDISH_EC2_CONTROL_SECRET_NAME: oddish-ec2-control
      ODDISH_EC2_SSH_SECRET_NAME: oddish-ec2-ssh
      ODDISH_EC2_REGION: us-west-2
      ODDISH_EC2_AMI_ID: ami-0ac74609c6396bed3
      ODDISH_EC2_INSTANCE_TYPE: m7i-flex.2xlarge
      ODDISH_EC2_SUBNET_ID: subnet-0da0349bc81aa34d8
      ODDISH_EC2_SECURITY_GROUP_IDS: '["sg-04d0011ae0f6b6c00"]'
      ODDISH_EC2_KEY_NAME: oddish-harbor
      ODDISH_EC2_SSH_USER: ubuntu
      ODDISH_EC2_ROOT_VOLUME_SIZE_GB: "80"
      ODDISH_EC2_USE_PUBLIC_IP: "true"
      ODDISH_EC2_BOOTSTRAP_DOCKER: "true"
      UV_PROJECT_ENVIRONMENT: /opt/venvs/backend
      # Low caps: staging carries no production traffic.
      ODDISH_MODAL_API_MIN_CONTAINERS: "0"
      ODDISH_MODAL_API_BUFFER_CONTAINERS: "0"
      ODDISH_MODAL_API_MAX_CONTAINERS: "2"
      ODDISH_MODAL_WORKER_MIN_CONTAINERS: "0"
      ODDISH_MODAL_WORKER_BUFFER_CONTAINERS: "0"
      ODDISH_MODAL_WORKER_MAX_CONTAINERS: "8"
      ODDISH_MODAL_MAX_WORKERS_PER_POLL: "16"
      # Explicit documentation of the default (MODAL_APP_NAME != "oddish").
      ODDISH_ENABLE_SLACK_EXPENSE_NOTIFICATIONS: "false"
    steps:
      - uses: actions/checkout@v5

      # A credential that no longer authenticates almost always means Staging
      # DB Bootstrap rotated the branch password: it cannot write a GitHub
      # secret (403), so STAGING_DATABASE_URL stays behind until a human
      # refreshes it, and a deploy queued in the shared group runs into that.
      # Name the cause here instead of leaving a bare Alembic auth error.
      # This fails rather than skips on purpose -- a green run that deployed
      # nothing would satisfy the promotion preflight.
      - name: Check the staging database credential
        shell: bash
        env:
          ODDISH_DATABASE_URL: ${{ secrets.STAGING_DATABASE_URL }}
        run: |
          set -uo pipefail
          if [ -z "${ODDISH_DATABASE_URL:-}" ]; then
            echo "::error::STAGING_DATABASE_URL is not set on the staging environment. Set it with the recipe in the Staging DB Bootstrap job summary, then re-run this workflow."
            exit 1
          fi
          psql_url="${ODDISH_DATABASE_URL/postgresql+asyncpg:\/\//postgresql://}"
          echo "::add-mask::$psql_url"
          # Retry rather than probe once. A branch compute change restarts the
          # instance, and this project has seen connections refused for seconds
          # after health already reports green, so a single shot would blame
          # the credential for a database that is merely coming back.
          deadline=$(($(date +%s) + 300))
          until PGCONNECT_TIMEOUT=15 psql "$psql_url" -c 'select 1' >/dev/null 2>/tmp/psql.err; do
            # A rejected password is deterministic, so retrying it only holds
            # the shared staging-environment lock for five minutes and delays
            # the recovery deploy the operator is trying to run. psql exits 2
            # for the whole connection class including auth, so classify on
            # the message. Only a positively identified rejection fails fast;
            # refusals, timeouts and "the database system is starting up" are
            # a restart and keep retrying.
            if grep -qiE 'password authentication failed|no password supplied|role .* does not exist' /tmp/psql.err; then
              cat /tmp/psql.err >&2
              echo "::error::The staging database rejected STAGING_DATABASE_URL. Staging DB Bootstrap rotates the branch password and cannot write this secret, so refresh it with the recipe in that run's job summary, then re-run this workflow."
              exit 1
            fi
            if [ "$(date +%s)" -ge "$deadline" ]; then
              cat /tmp/psql.err >&2 || true
              echo "::error::The staging database did not accept STAGING_DATABASE_URL within 300s. Staging DB Bootstrap rotates the branch password and cannot write this secret, so refresh it with the recipe in that run's job summary; if instead the branch compute was changed, wait for the branch to report healthy and re-run this workflow."
              exit 1
            fi
            echo "staging database not reachable yet; retrying"
            sleep 10
          done
          echo "staging database credential authenticates"

      # No rotation, no compute pin: this is a PERSISTENT branch, unlike the
      # per-PR preview branches this flow was first modelled on. The database
      # URL is stored once as a secret on the `staging` environment, so a
      # deploy never interrupts the running app's connections and never
      # touches the branch's compute tier. The bootstrap workflow still
      # rotates (it may create the branch from scratch) and still restores
      # the tier after a bulk load.
      - name: Core migrations
        working-directory: oddish
        env:
          UV_PROJECT_ENVIRONMENT: /opt/venvs/oddish
          ODDISH_DATABASE_URL: ${{ secrets.STAGING_DATABASE_URL }}
        run: |
          uv sync --frozen --extra server
          uv run alembic upgrade head

      - name: Backend migrations
        working-directory: backend
        env:
          ODDISH_DATABASE_URL: ${{ secrets.STAGING_DATABASE_URL }}
        run: |
          uv sync --frozen
          uv run alembic upgrade head

      - name: Deploy
        working-directory: backend
        run: |
          uv sync --frozen
          uv run modal deploy deploy.py