integration-responses-conversations-auth-tests.yml

name: Integration Responses & Conversations Auth Tests

run-name: Run responses and conversations auth tests with Kubernetes authentication

on:
  push:
    branches:
      - main
      - 'release-[0-9]+.[0-9]+.x'
  pull_request:
    branches:
      - main
      - 'release-[0-9]+.[0-9]+.x'
    paths:
      - 'distributions/**'
      - 'src/llama_stack/**'
      - '!src/llama_stack_ui/**'
      - 'tests/integration/**'
      - 'uv.lock'
      - 'pyproject.toml'
      - 'requirements.txt'
      - '.github/workflows/integration-responses-conversations-auth-tests.yml' # This workflow
      - 'scripts/integration-responses-conversations-auth-tests.sh'
  merge_group:
    branches:
      - main
      - 'release-[0-9]+.[0-9]+.x'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  test-matrix:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        auth-provider: [oauth2_token]
      fail-fast: false # we want to run all tests regardless of failure

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install dependencies
        uses: ./.github/actions/setup-runner

      - name: Install minikube
        if: ${{ matrix.auth-provider == 'kubernetes' }}
        uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21

      - name: Start minikube
        if: ${{ matrix.auth-provider == 'oauth2_token' }}
        run: |
          minikube start
          kubectl get pods -A

      - name: Configure Kube Auth
        if: ${{ matrix.auth-provider == 'oauth2_token' }}
        run: |
          kubectl create namespace llama-stack
          for account in llama-stack-auth llama-stack-user1 llama-stack-user2; do
            kubectl create serviceaccount $account -n llama-stack
            kubectl create token $account -n llama-stack > $account-token
          done

      - name: Set Kubernetes Config
        if: ${{ matrix.auth-provider == 'oauth2_token' }}
        run: |
          echo "KUBERNETES_API_SERVER_URL=$(kubectl get --raw /.well-known/openid-configuration| jq -r .jwks_uri)" >> $GITHUB_ENV
          echo "KUBERNETES_CA_CERT_PATH=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.certificate-authority}')" >> $GITHUB_ENV
          echo "KUBERNETES_ISSUER=$(kubectl get --raw /.well-known/openid-configuration| jq -r .issuer)" >> $GITHUB_ENV
          echo "KUBERNETES_AUDIENCE=$(kubectl create token llama-stack-auth -n llama-stack --duration=1h | cut -d. -f2 | base64 -d | jq -r '.aud[0]')" >> $GITHUB_ENV
          echo "TOKEN=$(cat llama-stack-auth-token)" >> $GITHUB_ENV

      - name: List response recordings (replay)
        if: ${{ matrix.auth-provider == 'oauth2_token' }}
        run: |
          echo "Response recordings in checkout:"
          ls -la tests/integration/responses/recordings/ || true

      - name: Set Kube Auth Config and run server
        env:
          OPENAI_API_KEY: "dummy-key-for-replay-mode"
          # Enable replay so the server uses checked-in recordings instead of calling the API
          LLAMA_STACK_TEST_INFERENCE_MODE: "replay"
          # Pin recording dir to workspace so the server finds tests/integration/*/recordings/
          LLAMA_STACK_TEST_RECORDING_DIR: ${{ github.workspace }}/tests/integration/common
          # Tell the server to sync test context from client headers so recordings
          # are looked up in the correct per-test-directory (e.g. responses/recordings/)
          LLAMA_STACK_TEST_STACK_CONFIG_TYPE: "server"
        if: ${{ matrix.auth-provider == 'oauth2_token' }}
        run: |
          run_dir=$(mktemp -d)
          cat <<EOF > $run_dir/config.yaml
          version: '2'
          image_name: kube
          distro_name: kube
          apis:
            - files
            - inference
            - vector_io
            - tool_runtime
            - agents
          providers:
            files:
              - provider_id: builtin-files
                provider_type: inline::localfs
                config:
                  storage_dir: $run_dir/files
                  metadata_store:
                    table_name: files_metadata
                    backend: sql_default
            inference:
              - provider_id: openai
                provider_type: remote::openai
                config:
                  api_key: \${env.OPENAI_API_KEY}
            vector_io:
              - provider_id: faiss
                provider_type: inline::faiss
                config:
                  persistence:
                    namespace: vector_io::faiss
                    backend: kv_default
            tool_runtime:
              - provider_id: file-search
                provider_type: inline::file-search
                config: {}
            agents:
              - provider_id: builtin
                provider_type: inline::builtin
                config:
                  persistence:
                    agent_state:
                      namespace: agents
                      backend: kv_default
                    responses:
                      table_name: responses
                      backend: sql_default
                      max_write_queue_size: 10000
                      num_writers: 4
          storage:
            backends:
              kv_default:
                type: kv_sqlite
                db_path: $run_dir/kvstore.db
              sql_default:
                type: sql_sqlite
                db_path: $run_dir/sql_store.db
            stores:
              metadata:
                namespace: registry
                backend: kv_default
              inference:
                table_name: inference_store
                backend: sql_default
              conversations:
                table_name: openai_conversations
                backend: sql_default
              prompts:
                namespace: prompts
                backend: kv_default
          models:
          - model_id: openai/gpt-4o
            model_type: llm
            provider_id: openai
          server:
            port: 8321
            auth:
              access_policy:
                - permit:
                    actions: [read]
                    resource: model::*
                  description: Any authenticated user can use configured models (inference, responses)
                - permit:
                    actions: [create]
                    resource: sql_record::openai_files::*
                  description: Any authenticated user can create files
                - permit:
                    actions: [read, delete]
                    resource: sql_record::openai_files::*
                  when:
                    - user is owner
                  description: Users can read and delete their own files
                - permit:
                    actions: [create]
                    resource: sql_record::openai_conversations::*
                  description: Any authenticated user can create conversations
                - permit:
                    actions: [read, update, delete]
                    resource: sql_record::openai_conversations::*
                  when:
                    - user is owner
                  description: Users can access their own conversations
                - permit:
                    actions: [create]
                    resource: sql_record::conversation_items::*
                  description: Any authenticated user can create conversation items
                - permit:
                    actions: [read, update, delete]
                    resource: sql_record::conversation_items::*
                  when:
                    - user is owner
                  description: Users can access items in their own conversations
                - permit:
                    actions: [create]
                    resource: sql_record::responses::*
                  description: Any authenticated user can create responses
                - permit:
                    actions: [read, update, delete]
                    resource: sql_record::responses::*
                  when:
                    - user is owner
                  description: Users can access their own responses
          EOF
          yq eval '.server.auth.provider_config.type = "${{ matrix.auth-provider }}"' -i $run_dir/config.yaml
          yq eval '.server.auth.provider_config.tls_cafile = "${{ env.KUBERNETES_CA_CERT_PATH }}"' -i $run_dir/config.yaml
          yq eval '.server.auth.provider_config.issuer = "${{ env.KUBERNETES_ISSUER }}"' -i $run_dir/config.yaml
          yq eval '.server.auth.provider_config.audience = "${{ env.KUBERNETES_AUDIENCE }}"' -i $run_dir/config.yaml
          yq eval '.server.auth.provider_config.jwks.uri = "${{ env.KUBERNETES_API_SERVER_URL }}"' -i $run_dir/config.yaml
          yq eval '.server.auth.provider_config.jwks.token = "${{ env.TOKEN }}"' -i $run_dir/config.yaml
          cat $run_dir/config.yaml

          # avoid line breaks in the server log, especially because we grep it below.
          export LLAMA_STACK_LOG_WIDTH=200
          # OPENAI_API_KEY is set via env at step level
          nohup uv run llama stack run $run_dir/config.yaml > server.log 2>&1 &

      - name: Wait for Llama Stack server to be ready
        run: |
          echo "Waiting for Llama Stack server..."
          for i in {1..30}; do
            # Note: /v1/health does not require authentication
            if curl -s -L http://localhost:8321/v1/health | grep -q "OK"; then
              echo "Llama Stack server is up!"
              if grep -q "Enabling authentication with provider: ${{ matrix.auth-provider }}" server.log; then
                echo "Llama Stack server is configured to use ${{ matrix.auth-provider }} auth"
                exit 0
              else
                echo "Llama Stack server is not configured to use ${{ matrix.auth-provider }} auth"
                cat server.log
                exit 1
              fi
            fi
            sleep 1
          done
          echo "Llama Stack server failed to start"
          cat server.log
          exit 1

      - name: Test auth
        run: |
          export INFERENCE_MODEL=openai/gpt-4o
          # Run the auth tests
          ./scripts/integration-responses-conversations-auth-tests.sh

      - name: Server log on failure
        if: failure()
        run: |
          echo "=== Llama Stack server log (last 500 lines) ==="
          tail -500 server.log