Run responses and conversations auth tests with Kubernetes authentication #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Responses & Conversations Auth Tests | |
| run-name: Run responses and conversations auth tests with Kubernetes authentication | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| paths: | |
| - 'distributions/**' | |
| - 'src/llama_stack/**' | |
| - '!src/llama_stack_ui/**' | |
| - 'tests/integration/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - 'requirements.txt' | |
| - '.github/workflows/integration-responses-conversations-auth-tests.yml' # This workflow | |
| - 'scripts/integration-responses-conversations-auth-tests.sh' | |
| merge_group: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| test-matrix: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| auth-provider: [oauth2_token] | |
| fail-fast: false # we want to run all tests regardless of failure | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Install dependencies | |
| uses: ./.github/actions/setup-runner | |
| - name: Install minikube | |
| if: ${{ matrix.auth-provider == 'kubernetes' }} | |
| uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21 | |
| - name: Start minikube | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| minikube start | |
| kubectl get pods -A | |
| - name: Configure Kube Auth | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| kubectl create namespace llama-stack | |
| for account in llama-stack-auth llama-stack-user1 llama-stack-user2; do | |
| kubectl create serviceaccount $account -n llama-stack | |
| kubectl create token $account -n llama-stack > $account-token | |
| done | |
| - name: Set Kubernetes Config | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| echo "KUBERNETES_API_SERVER_URL=$(kubectl get --raw /.well-known/openid-configuration| jq -r .jwks_uri)" >> $GITHUB_ENV | |
| echo "KUBERNETES_CA_CERT_PATH=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.certificate-authority}')" >> $GITHUB_ENV | |
| echo "KUBERNETES_ISSUER=$(kubectl get --raw /.well-known/openid-configuration| jq -r .issuer)" >> $GITHUB_ENV | |
| echo "KUBERNETES_AUDIENCE=$(kubectl create token llama-stack-auth -n llama-stack --duration=1h | cut -d. -f2 | base64 -d | jq -r '.aud[0]')" >> $GITHUB_ENV | |
| echo "TOKEN=$(cat llama-stack-auth-token)" >> $GITHUB_ENV | |
| - name: List response recordings (replay) | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| echo "Response recordings in checkout:" | |
| ls -la tests/integration/responses/recordings/ || true | |
| - name: Set Kube Auth Config and run server | |
| env: | |
| OPENAI_API_KEY: "dummy-key-for-replay-mode" | |
| # Enable replay so the server uses checked-in recordings instead of calling the API | |
| LLAMA_STACK_TEST_INFERENCE_MODE: "replay" | |
| # Pin recording dir to workspace so the server finds tests/integration/*/recordings/ | |
| LLAMA_STACK_TEST_RECORDING_DIR: ${{ github.workspace }}/tests/integration/common | |
| # Tell the server to sync test context from client headers so recordings | |
| # are looked up in the correct per-test-directory (e.g. responses/recordings/) | |
| LLAMA_STACK_TEST_STACK_CONFIG_TYPE: "server" | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| run_dir=$(mktemp -d) | |
| cat <<EOF > $run_dir/config.yaml | |
| version: '2' | |
| image_name: kube | |
| distro_name: kube | |
| apis: | |
| - files | |
| - inference | |
| - vector_io | |
| - tool_runtime | |
| - agents | |
| providers: | |
| files: | |
| - provider_id: builtin-files | |
| provider_type: inline::localfs | |
| config: | |
| storage_dir: $run_dir/files | |
| metadata_store: | |
| table_name: files_metadata | |
| backend: sql_default | |
| inference: | |
| - provider_id: openai | |
| provider_type: remote::openai | |
| config: | |
| api_key: \${env.OPENAI_API_KEY} | |
| vector_io: | |
| - provider_id: faiss | |
| provider_type: inline::faiss | |
| config: | |
| persistence: | |
| namespace: vector_io::faiss | |
| backend: kv_default | |
| tool_runtime: | |
| - provider_id: file-search | |
| provider_type: inline::file-search | |
| config: {} | |
| agents: | |
| - provider_id: builtin | |
| provider_type: inline::builtin | |
| config: | |
| persistence: | |
| agent_state: | |
| namespace: agents | |
| backend: kv_default | |
| responses: | |
| table_name: responses | |
| backend: sql_default | |
| max_write_queue_size: 10000 | |
| num_writers: 4 | |
| storage: | |
| backends: | |
| kv_default: | |
| type: kv_sqlite | |
| db_path: $run_dir/kvstore.db | |
| sql_default: | |
| type: sql_sqlite | |
| db_path: $run_dir/sql_store.db | |
| stores: | |
| metadata: | |
| namespace: registry | |
| backend: kv_default | |
| inference: | |
| table_name: inference_store | |
| backend: sql_default | |
| conversations: | |
| table_name: openai_conversations | |
| backend: sql_default | |
| prompts: | |
| namespace: prompts | |
| backend: kv_default | |
| models: | |
| - model_id: openai/gpt-4o | |
| model_type: llm | |
| provider_id: openai | |
| server: | |
| port: 8321 | |
| auth: | |
| access_policy: | |
| - permit: | |
| actions: [read] | |
| resource: model::* | |
| description: Any authenticated user can use configured models (inference, responses) | |
| - permit: | |
| actions: [create] | |
| resource: sql_record::openai_files::* | |
| description: Any authenticated user can create files | |
| - permit: | |
| actions: [read, delete] | |
| resource: sql_record::openai_files::* | |
| when: | |
| - user is owner | |
| description: Users can read and delete their own files | |
| - permit: | |
| actions: [create] | |
| resource: sql_record::openai_conversations::* | |
| description: Any authenticated user can create conversations | |
| - permit: | |
| actions: [read, update, delete] | |
| resource: sql_record::openai_conversations::* | |
| when: | |
| - user is owner | |
| description: Users can access their own conversations | |
| - permit: | |
| actions: [create] | |
| resource: sql_record::conversation_items::* | |
| description: Any authenticated user can create conversation items | |
| - permit: | |
| actions: [read, update, delete] | |
| resource: sql_record::conversation_items::* | |
| when: | |
| - user is owner | |
| description: Users can access items in their own conversations | |
| - permit: | |
| actions: [create] | |
| resource: sql_record::responses::* | |
| description: Any authenticated user can create responses | |
| - permit: | |
| actions: [read, update, delete] | |
| resource: sql_record::responses::* | |
| when: | |
| - user is owner | |
| description: Users can access their own responses | |
| EOF | |
| yq eval '.server.auth.provider_config.type = "${{ matrix.auth-provider }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.tls_cafile = "${{ env.KUBERNETES_CA_CERT_PATH }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.issuer = "${{ env.KUBERNETES_ISSUER }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.audience = "${{ env.KUBERNETES_AUDIENCE }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.jwks.uri = "${{ env.KUBERNETES_API_SERVER_URL }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.jwks.token = "${{ env.TOKEN }}"' -i $run_dir/config.yaml | |
| cat $run_dir/config.yaml | |
| # avoid line breaks in the server log, especially because we grep it below. | |
| export LLAMA_STACK_LOG_WIDTH=200 | |
| # OPENAI_API_KEY is set via env at step level | |
| nohup uv run llama stack run $run_dir/config.yaml > server.log 2>&1 & | |
| - name: Wait for Llama Stack server to be ready | |
| run: | | |
| echo "Waiting for Llama Stack server..." | |
| for i in {1..30}; do | |
| # Note: /v1/health does not require authentication | |
| if curl -s -L http://localhost:8321/v1/health | grep -q "OK"; then | |
| echo "Llama Stack server is up!" | |
| if grep -q "Enabling authentication with provider: ${{ matrix.auth-provider }}" server.log; then | |
| echo "Llama Stack server is configured to use ${{ matrix.auth-provider }} auth" | |
| exit 0 | |
| else | |
| echo "Llama Stack server is not configured to use ${{ matrix.auth-provider }} auth" | |
| cat server.log | |
| exit 1 | |
| fi | |
| fi | |
| sleep 1 | |
| done | |
| echo "Llama Stack server failed to start" | |
| cat server.log | |
| exit 1 | |
| - name: Test auth | |
| run: | | |
| export INFERENCE_MODEL=openai/gpt-4o | |
| # Run the auth tests | |
| ./scripts/integration-responses-conversations-auth-tests.sh | |
| - name: Server log on failure | |
| if: failure() | |
| run: | | |
| echo "=== Llama Stack server log (last 500 lines) ===" | |
| tail -500 server.log |