deploy.yaml

name: Deploy SAM Hub

on:
  push:
    branches:
      - main
    tags:
      - 'v*.*.*'
  workflow_dispatch:

env:
  REGISTRY: ghcr.io

jobs:
  publish:
    name: Publish Images
    runs-on: ubuntu-latest
    permissions:
      packages: write
      contents: read
    timeout-minutes: 100
    
    steps:
    - name: Check out code
      uses: actions/checkout@v7

    - name: Set up QEMU
      uses: docker/setup-qemu-action@v4

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v4

    - name: Log in to the Container registry
      uses: docker/login-action@v4
      with:
        registry: ${{ env.REGISTRY }}
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Extract metadata for Control Plane
      id: meta-cp
      uses: docker/metadata-action@v6
      with:
        images: ${{ env.REGISTRY }}/google/sam-control-plane
        tags: |
          type=ref,event=branch
          type=ref,event=tag
          type=raw,value=${{ github.sha }}
          type=raw,value=latest,enable={{is_default_branch}}
          type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}

    - name: Build and push Control Plane image
      uses: docker/build-push-action@v7
      with:
        context: .
        file: Dockerfile.sam-control-plane
        platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
        push: true
        tags: ${{ steps.meta-cp.outputs.tags }}
        labels: ${{ steps.meta-cp.outputs.labels }}

    - name: Extract metadata for Router
      id: meta-router
      uses: docker/metadata-action@v6
      with:
        images: ${{ env.REGISTRY }}/google/sam-router
        tags: |
          type=ref,event=branch
          type=ref,event=tag
          type=raw,value=${{ github.sha }}
          type=raw,value=latest,enable={{is_default_branch}}
          type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}

    - name: Build and push Router image
      uses: docker/build-push-action@v7
      with:
        context: .
        file: Dockerfile.sam-router
        platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
        push: true
        tags: ${{ steps.meta-router.outputs.tags }}
        labels: ${{ steps.meta-router.outputs.labels }}

    - name: Extract metadata for Node
      id: meta-node
      uses: docker/metadata-action@v6
      with:
        images: ${{ env.REGISTRY }}/google/sam-node
        tags: |
          type=ref,event=branch
          type=ref,event=tag
          type=raw,value=${{ github.sha }}
          type=raw,value=latest,enable={{is_default_branch}}
          type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}

    - name: Build and push Node image
      uses: docker/build-push-action@v7
      with:
        context: .
        file: Dockerfile.sam-node
        platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
        push: true
        tags: ${{ steps.meta-node.outputs.tags }}
        labels: ${{ steps.meta-node.outputs.labels }}

    - name: Extract metadata for nano-init
      id: meta-nano-init
      uses: docker/metadata-action@v6
      with:
        images: ${{ env.REGISTRY }}/google/sam-nano-init
        tags: |
          type=ref,event=branch
          type=ref,event=tag
          type=raw,value=${{ github.sha }}
          type=raw,value=latest,enable={{is_default_branch}}
          type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}

    - name: Build and push nano-init image
      uses: docker/build-push-action@v7
      with:
        context: .
        file: Dockerfile.nano-init
        platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
        push: true
        tags: ${{ steps.meta-nano-init.outputs.tags }}
        labels: ${{ steps.meta-nano-init.outputs.labels }}

    - name: Extract metadata for sam-box
      id: meta-sambox
      uses: docker/metadata-action@v6
      with:
        images: ${{ env.REGISTRY }}/google/sam-box
        tags: |
          type=ref,event=branch
          type=ref,event=tag
          type=raw,value=${{ github.sha }}
          type=raw,value=latest,enable={{is_default_branch}}
          type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}

    - name: Build and push sam-box image
      uses: docker/build-push-action@v7
      with:
        context: .
        file: Dockerfile.sam-box
        platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
        push: true
        tags: ${{ steps.meta-sambox.outputs.tags }}
        labels: ${{ steps.meta-sambox.outputs.labels }}

  deploy:
    name: Deploy to GKE
    runs-on: ubuntu-latest
    needs: publish
    # Dynamically select the GitHub Environment based on the trigger
    environment: ${{ github.ref_type == 'tag' && 'hub' || 'bananas' }}
    permissions:
      contents: 'read'
      id-token: 'write'

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Determine Image Tag
        run: |
          if [[ "${{ github.ref_type }}" == "tag" ]]; then
            echo "IMAGE_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
          else
            echo "IMAGE_TAG=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Google Auth
        uses: google-github-actions/auth@v3
        with:
          workload_identity_provider: ${{ vars.WIF_PROVIDER_NAME }}
          service_account: ${{ vars.SERVICE_ACCOUNT_EMAIL }}

      - name: Set up GKE credentials
        uses: google-github-actions/get-gke-credentials@v3
        with:
          cluster_name: ${{ vars.CLUSTER_NAME }}
          location: ${{ vars.CLUSTER_REGION }}

      - name: Prepare Namespaces
        run: |
          export ENV_NAME="${{ vars.ENV_NAME }}"
          export NAMESPACE="sam-${ENV_NAME}"
          export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
          
          kubectl create namespace dex --dry-run=client -o yaml | kubectl apply -f -
          kubectl create namespace ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f -
          kubectl create namespace ${CANARY_NAMESPACE} --dry-run=client -o yaml | kubectl apply -f -

      - name: Provision Dex Secrets
        env:
          GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
          GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
          GITHUB_CLIENT_ID: ${{ secrets.GHUB_CLIENT_ID }}
          GITHUB_CLIENT_SECRET: ${{ secrets.GHUB_CLIENT_SECRET }}
          CLI_OAUTH_SECRET: ${{ secrets.CLI_OAUTH_SECRET }}
        run: |
          # Declaratively apply secrets without writing them to disk
          kubectl create secret generic dex-secrets \
            --namespace=dex \
            --from-literal=google-client-id="${GOOGLE_CLIENT_ID}" \
            --from-literal=google-client-secret="${GOOGLE_CLIENT_SECRET}" \
            --from-literal=github-client-id="${GITHUB_CLIENT_ID}" \
            --from-literal=github-client-secret="${GITHUB_CLIENT_SECRET}" \
            --from-literal=cli-oauth-secret="${CLI_OAUTH_SECRET}" \
            --dry-run=client -o yaml | kubectl apply -f -

      - name: Deploy Dex Resources
        run: |
          print_rollout_diagnostics() {
            local namespace="$1"
            local deployment="$2"
            local selector="$3"

            echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..."

            kubectl describe deployment/${deployment} -n ${namespace} || true
            kubectl get pods -n ${namespace} -l "${selector}" -o wide || true
            kubectl describe pods -n ${namespace} -l "${selector}" || true
            kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true

            for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do
              echo "==== Describe ${pod} ===="
              kubectl describe -n ${namespace} "${pod}" || true

              for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do
                echo "==== Logs for ${pod} container ${container} ===="
                kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true
              done
            done
          }

          # Apply the Dex manifests
          kubectl apply -f .github/k8s/dex-config.yaml
          kubectl apply -f .github/k8s/dex-deployment.yaml
          
          # Force a restart to pick up any changes to the secrets, and wait for readiness
          kubectl rollout restart deployment/dex -n dex
          kubectl rollout status deployment/dex -n dex --timeout=120s || {
            echo "Dex Deployment failed!"
            print_rollout_diagnostics "dex" "dex" "app=dex"

            echo "Rolling back Dex deployment..."
            kubectl rollout undo deployment/dex -n dex
            exit 1
          }

      - name: Provision Hub Secrets
        run: |
          export ENV_NAME="${{ vars.ENV_NAME }}"
          export NAMESPACE="sam-${ENV_NAME}"
          # Idempotently provision the sam-hub-secret if it doesn't already exist
          if ! kubectl get secret sam-hub-secret -n ${NAMESPACE} >/dev/null 2>&1; then
            echo "Generating a new random 32-byte hex key for SAM_HUB_KEY..."
            HUB_KEY=$(openssl rand -hex 32)
            kubectl create secret generic sam-hub-secret \
              --namespace=${NAMESPACE} \
              --from-literal=sam-hub-key="${HUB_KEY}"
          else
            echo "sam-hub-secret already exists in namespace ${NAMESPACE}. Skipping generation."
          fi



      - name: Provision Control Plane Secrets
        run: |
          export ENV_NAME="${{ vars.ENV_NAME }}"
          export NAMESPACE="sam-${ENV_NAME}"
          if ! kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then
            echo "Generating a new random 32-byte hex key for admin-token..."
            ADMIN_TOKEN=$(openssl rand -hex 32)
            kubectl create secret generic sam-control-plane-secret-${ENV_NAME} \
              --namespace=${NAMESPACE} \
              --from-literal=admin-token="${ADMIN_TOKEN}"
          else
            echo "sam-control-plane-secret-${ENV_NAME} already exists in namespace ${NAMESPACE}. Skipping generation."
          fi

      - name: Deploy Control Plane and Routers
        run: |
          print_rollout_diagnostics() {
            local namespace="$1"
            local resource="$2"
            local selector="$3"

            echo "Collecting diagnostics for ${resource} in namespace ${namespace}..."

            kubectl describe ${resource} -n ${namespace} || true
            kubectl get pods -n ${namespace} -l "${selector}" -o wide || true
            kubectl describe pods -n ${namespace} -l "${selector}" || true
            kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true

            for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do
              echo "==== Describe ${pod} ===="
              kubectl describe -n ${namespace} "${pod}" || true

              for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do
                echo "==== Logs for ${pod} container ${container} ===="
                kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true
              done
            done
          }

          sudo apt-get install -y gettext-base
          
          export ENV_NAME="${{ vars.ENV_NAME }}"
          export NAMESPACE="sam-${ENV_NAME}"
          export IMAGE_TAG="${{ env.IMAGE_TAG }}"
          export GCP_PROJECT_ID="${{ vars.GCP_PROJECT_ID }}"
          export CLUSTER_REGION="${{ vars.CLUSTER_REGION }}"
          export CLUSTER_NAME="${{ vars.CLUSTER_NAME }}"

          envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-control-plane-template.yaml | kubectl apply -f -
          envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-router-template.yaml | kubectl apply -f -
          envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f -
          
          kubectl rollout status deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || {
            echo "Control Plane Deployment failed!"
            print_rollout_diagnostics "${NAMESPACE}" "deployment/sam-control-plane-${ENV_NAME}" "app=sam-control-plane-${ENV_NAME}"
            exit 1
          }

          kubectl rollout status statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || {
            echo "Router Deployment failed!"
            print_rollout_diagnostics "${NAMESPACE}" "statefulset/sam-router-${ENV_NAME}" "app=sam-router-${ENV_NAME}"

            echo "Rolling back Router deployment..."
            kubectl rollout undo statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE}
            kubectl rollout status statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE}
            exit 1
          }

          echo "Triggering immediate DNS sync after rollout..."
          kubectl delete job dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} --ignore-not-found || true
          kubectl create job --from=cronjob/dns-sync-cronjob dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE}
          echo "Waiting for DNS sync job to complete..."
          kubectl wait --for=condition=complete job/dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} --timeout=180s || {
            echo "DNS sync job failed or timed out!"
            kubectl describe job/dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE}
            kubectl logs -n ${NAMESPACE} -l job-name=dns-sync-rollout-${ENV_NAME} --tail=-1 || true
            exit 1
          }


      - name: Deploy GCE VM Router
        run: |
          export ENV_NAME="${{ vars.ENV_NAME }}"
          export NAMESPACE="sam-${ENV_NAME}"
          export IMAGE_TAG="${{ env.IMAGE_TAG }}"
          export GCP_PROJECT_ID="${{ vars.GCP_PROJECT_ID }}"
          export CLUSTER_REGION="${{ vars.CLUSTER_REGION }}"
          export ZONE="${CLUSTER_REGION}-a"

          echo "Retrieving Admin Token to generate Bootstrap Token..."
          ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d)

          echo "Generating Bootstrap Token via internal GCP/GKE Control Plane service..."
          
          # Clean up any leftover token generator pod
          kubectl delete pod curl-token-gen-vm -n ${NAMESPACE} --ignore-not-found || true
          
          # Run temporary curl pod in control plane namespace
          kubectl run curl-token-gen-vm \
            -n ${NAMESPACE} \
            --image=curlimages/curl:8.6.0 \
            --restart=Never \
            --overrides='{"spec": {"activeDeadlineSeconds": 30}}' \
            -- \
            curl -s -X POST \
              -H "Content-Type: application/json" \
              -H "Authorization: Bearer ${ADMIN_TOKEN}" \
              -d '{"role": "sam:role:router", "max_usages": 1}' \
              http://sam-control-plane-${ENV_NAME}:8080/admin/bootstrap-tokens

          if ! kubectl wait -n ${NAMESPACE} --for=jsonpath='{.status.phase}'=Succeeded pod/curl-token-gen-vm --timeout=15s; then
            echo "ERROR: Token generation pod failed!"
            kubectl describe pod curl-token-gen-vm -n ${NAMESPACE} || true
            kubectl logs pod/curl-token-gen-vm -n ${NAMESPACE} || true
            kubectl delete pod curl-token-gen-vm -n ${NAMESPACE} --ignore-not-found || true
            exit 1
          fi

          TOKEN_JSON=$(kubectl logs pod/curl-token-gen-vm -n ${NAMESPACE})
          kubectl delete pod curl-token-gen-vm -n ${NAMESPACE} --ignore-not-found

          BOOTSTRAP_TOKEN=$(echo "${TOKEN_JSON}" | jq -r .token)
          if [ -z "${BOOTSTRAP_TOKEN}" ] || [ "${BOOTSTRAP_TOKEN}" = "null" ]; then
            echo "ERROR: Generated bootstrap token is empty or invalid!"
            echo "Response: ${TOKEN_JSON}"
            exit 1
          fi
          
          echo "Successfully generated bootstrap token."

          echo "Provisioning GCE VM Router instance..."
          # Delete existing VM if it exists to ensure fresh deployment with the new token and image
          gcloud compute instances delete sam-router-${ENV_NAME}-vm --zone=${ZONE} --project=${GCP_PROJECT_ID} --quiet || true

          # Create a new COS VM instance running the sam-router
          gcloud compute instances create sam-router-${ENV_NAME}-vm \
            --project=${GCP_PROJECT_ID} \
            --zone=${ZONE} \
            --machine-type=e2-micro \
            --image-family=cos-stable \
            --image-project=cos-cloud \
            --tags=sam-router \
            --metadata=startup-script='#!/bin/bash
          set -euo pipefail
          EXTERNAL_IP=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
          docker run -d \
            --name sam-router \
            --network host \
            --restart always \
            ghcr.io/google/sam-router:'"${IMAGE_TAG}"' \
            --control-plane=https://'"${ENV_NAME}"'.sam-mesh.dev \
            --bootstrap-token='"${BOOTSTRAP_TOKEN}"' \
            --listen=/ip4/0.0.0.0/tcp/4501 \
            --listen=/ip4/0.0.0.0/udp/4501/quic-v1 \
            --external-addr=/ip4/${EXTERNAL_IP}/tcp/4501 \
            --external-addr=/ip4/${EXTERNAL_IP}/udp/4501/quic-v1'


      - name: Deploy Canaries
        run: |
          print_rollout_diagnostics() {
            local namespace="$1"
            local deployment="$2"
            local selector="$3"

            echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..."

            kubectl describe deployment/${deployment} -n ${namespace} || true
            kubectl get pods -n ${namespace} -l "${selector}" -o wide || true
            kubectl describe pods -n ${namespace} -l "${selector}" || true
            kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true

            for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do
              echo "==== Describe ${pod} ===="
              kubectl describe -n ${namespace} "${pod}" || true

              for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do
                echo "==== Logs for ${pod} container ${container} ===="
                kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true
              done
            done
          }

          export ENV_NAME="${{ vars.ENV_NAME }}"
          export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
          export NAMESPACE="sam-${ENV_NAME}"
          export IMAGE_TAG="${{ env.IMAGE_TAG }}"
          export GEMINI_API_KEY="${{ secrets.GEMINI_API_KEY }}"
          export HF_TOKEN="${{ secrets.HF_TOKEN }}"
          
          # Read and indent banana bot playground script for the ConfigMap template
          export BANANA_BOT_SCRIPT=$(cat site/content/docs/snippets/banana_bot_playground.py | sed 's/^/    /')
          # Provision the openclaw-secret, preserving the gateway-token if it exists
          GATEWAY_TOKEN=$(kubectl get secret openclaw-secret-${ENV_NAME} -n ${CANARY_NAMESPACE} -o jsonpath="{.data.gateway-token}" 2>/dev/null | base64 -d || openssl rand -hex 16)
          kubectl create secret generic openclaw-secret-${ENV_NAME} \
            --namespace=${CANARY_NAMESPACE} \
            --from-literal=gateway-token="${GATEWAY_TOKEN}" \
            --from-literal=gemini-api-key="${GEMINI_API_KEY}" \
            --dry-run=client -o yaml | kubectl apply -f -

          kubectl create secret generic vllm-secret-${ENV_NAME} \
            --namespace=${CANARY_NAMESPACE} \
            --from-literal=hf-token="${HF_TOKEN}" \
            --dry-run=client -o yaml | kubectl apply -f -

          envsubst < .github/k8s/sam-node-template.yaml | kubectl apply -f -
          envsubst < .github/k8s/sam-node-openclaw-template.yaml | kubectl apply -f -
          envsubst < .github/k8s/sam-node-vllm-template.yaml | kubectl apply -f -
          envsubst < .github/k8s/sam-node-everything-template.yaml | kubectl apply -f -
          envsubst < .github/k8s/sam-node-cop-template.yaml | kubectl apply -f -
          envsubst < .github/k8s/sam-box-canary-template.yaml | kubectl apply -f -

          kubectl rollout status deployment/cop-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || {
            echo "Cop Canary Deployment failed!"
            print_rollout_diagnostics "${CANARY_NAMESPACE}" "cop-canary-${ENV_NAME}" "app=cop-canary-${ENV_NAME}"
            exit 1
          }

          kubectl rollout status deployment/box-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || {
            echo "sam-box Canary Deployment failed!"
            print_rollout_diagnostics "${CANARY_NAMESPACE}" "box-canary-${ENV_NAME}" "app=box-canary-${ENV_NAME}"
            exit 1
          }

          
          kubectl rollout status deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || {
            echo "Canary Deployment failed!"
            print_rollout_diagnostics "${CANARY_NAMESPACE}" "sam-canary-${ENV_NAME}" "app=sam-canary-${ENV_NAME}"

            echo "Rolling back Canary deployment..."
            kubectl rollout undo deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE}
            kubectl rollout status deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
            exit 1
          }

          kubectl rollout status deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || {
            echo "OpenClaw Canary Deployment failed!"
            print_rollout_diagnostics "${CANARY_NAMESPACE}" "openclaw-canary-${ENV_NAME}" "app=openclaw-canary-${ENV_NAME}"

            echo "Rolling back OpenClaw Canary deployment..."
            kubectl rollout undo deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE}
            kubectl rollout status deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
            exit 1
          }

          kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || {
            echo "vLLM Canary Deployment failed!"
            print_rollout_diagnostics "${CANARY_NAMESPACE}" "vllm-canary-${ENV_NAME}" "app=vllm-canary-${ENV_NAME}"

            echo "Rolling back vLLM Canary deployment..."
            kubectl rollout undo deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE}
            kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
            exit 1
          }

          kubectl rollout status deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || {
            echo "Everything Canary Deployment failed!"
            print_rollout_diagnostics "${CANARY_NAMESPACE}" "everything-canary-${ENV_NAME}" "app=everything-canary-${ENV_NAME}"

            echo "Rolling back Everything Canary deployment..."
            kubectl rollout undo deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE}
            kubectl rollout status deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
            exit 1
          }

      - name: Deploy OpenRouter Node to Testnet
        env:
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
        run: |
          print_rollout_diagnostics() {
            local namespace="$1"
            local deployment="$2"
            local selector="$3"

            echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..."

            kubectl describe deployment/${deployment} -n ${namespace} || true
            kubectl get pods -n ${namespace} -l "${selector}" -o wide || true
            kubectl describe pods -n ${namespace} -l "${selector}" || true
            kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true

            for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do
              echo "==== Describe ${pod} ===="
              kubectl describe -n ${namespace} "${pod}" || true

              for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do
                echo "==== Logs for ${pod} container ${container} ===="
                kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true
              done
            done
          }

          export ENV_NAME="${{ vars.ENV_NAME }}"
          export NAMESPACE="sam-${ENV_NAME}"
          export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
          export IMAGE_TAG="${{ env.IMAGE_TAG }}"
          
          # Substitute the API key and other variables into the template and apply it to the cluster
          envsubst < .github/k8s/sam-node-openrouter-template.yaml | kubectl apply -f -
          
          kubectl rollout status deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || {
            echo "OpenRouter Canary Deployment failed!"
            print_rollout_diagnostics "${CANARY_NAMESPACE}" "openrouter-canary-${ENV_NAME}" "app=openrouter-canary-${ENV_NAME}"

            echo "Rolling back OpenRouter Canary deployment..."
            kubectl rollout undo deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE}
            kubectl rollout status deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true
            exit 1
          }