Merge remote-tracking branch 'upstream/main' into responses_prompt_templates

Author: msager27

.github/workflows/README.md

@@ -15,6 +15,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
 | Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suites from tests/integration in replay mode |
 | Vector IO Integration Tests | [integration-vector-io-tests.yml](integration-vector-io-tests.yml) | Run the integration test suite with various VectorIO providers |
 | OpenAPI Generator SDK Validation | [openapi-generator-validation.yml](openapi-generator-validation.yml) | Validate OpenAPI Generator SDK generation |
+| OpenResponses Conformance Tests | [openresponses-conformance.yml](openresponses-conformance.yml) | Run OpenResponses conformance tests against llama-stack Responses API |
 | Pre-commit | [pre-commit.yml](pre-commit.yml) | Run pre-commit checks |
 | Test Llama Stack Build | [providers-build.yml](providers-build.yml) | Test llama stack build |
 | Test llama stack list-deps | [providers-list-deps.yml](providers-list-deps.yml) | Test llama stack list-deps |

.github/workflows/openapi-generator-validation.yml

@@ -7,7 +7,9 @@ on:
     paths:
       # OpenAPI Generator files
       - 'client-sdks/openapi/Makefile'
-      - 'client-sdks/openapi/merge_stainless_to_openapi.py'
+      - 'client-sdks/openapi/merge_stainless_config.py'
+      - 'client-sdks/openapi/build_hierarchy.py'
+      - 'client-sdks/openapi/patch_hierarchy.py'
       - 'client-sdks/openapi/patches.yml'
       - 'client-sdks/openapi/openapi-config.json'
       - 'client-sdks/openapi/openapitools.json'
@@ -56,7 +58,9 @@ jobs:
           filters: |
             critical:
               - 'client-sdks/openapi/Makefile'
-              - 'client-sdks/openapi/merge_stainless_to_openapi.py'
+              - 'client-sdks/openapi/merge_stainless_config.py'
+              - 'client-sdks/openapi/build_hierarchy.py'
+              - 'client-sdks/openapi/patch_hierarchy.py'
               - 'client-sdks/openapi/patches.yml'
               - 'client-sdks/openapi/openapitools.json'
               - 'client-sdks/stainless/openapi.yml'

.github/workflows/openresponses-conformance.yml

@@ -0,0 +1,229 @@
+name: OpenResponses Conformance Tests
+
+run-name: Run OpenResponses conformance tests against llama-stack Responses API
+
+# This job is OPTIONAL and informational — it tracks progress toward full
+# conformance with the OpenResponses spec (https://openresponses.org).
+# Failures are expected while gaps remain in the Responses API implementation.
+# See: https://github.com/llamastack/llama-stack/issues/4818
+#
+# Inference calls are replayed from checked-in recordings under
+# tests/integration/openresponses/recordings/. To add or update recordings,
+# run the server locally with LLAMA_STACK_TEST_INFERENCE_MODE=record-if-missing
+# pointed at that directory, run the compliance tests, and commit the results.
+
+on:
+  push:
+    branches:
+      - main
+      - 'release-[0-9]+.[0-9]+.x'
+    paths:
+      - 'src/llama_stack/providers/inline/agents/**'
+      - 'src/llama_stack/apis/agents/**'
+      - 'tests/integration/openresponses/**'
+      - '.github/workflows/openresponses-conformance.yml'
+  pull_request:
+    branches:
+      - main
+      - 'release-[0-9]+.[0-9]+.x'
+    paths:
+      - 'src/llama_stack/providers/inline/agents/**'
+      - 'src/llama_stack/apis/agents/**'
+      - 'tests/integration/openresponses/**'
+      - '.github/workflows/openresponses-conformance.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  openresponses-conformance:
+    name: OpenResponses Conformance (Informational)
+    runs-on: ubuntu-latest
+    # Failures are expected — this job does NOT block PRs or merges.
+    # It exists solely to track progress toward OpenResponses API conformance.
+    continue-on-error: true
+
+    env:
+      # Dummy key satisfies config parsing; actual inference is replayed from recordings
+      OPENAI_API_KEY: "dummy-key-for-replay-mode"
+      INFERENCE_MODEL: "openai/gpt-4o-mini"
+      LLAMA_STACK_PORT: 8321
+      LLAMA_STACK_TEST_INFERENCE_MODE: "replay"
+      LLAMA_STACK_TEST_RECORDING_DIR: ${{ github.workspace }}/tests/integration/openresponses
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Check for recordings
+        id: check-recordings
+        run: |
+          RECORDINGS_DIR="${{ github.workspace }}/tests/integration/openresponses/recordings"
+          COUNT=$(find "$RECORDINGS_DIR" -name "*.json" 2>/dev/null | wc -l)
+          echo "recordings_count=$COUNT" >> $GITHUB_OUTPUT
+          if [ "$COUNT" -eq 0 ]; then
+            echo "::warning::No recordings found in tests/integration/openresponses/recordings/."
+            echo "::warning::Run the server locally with LLAMA_STACK_TEST_INFERENCE_MODE=record-if-missing"
+            echo "::warning::and commit the resulting recordings to enable conformance testing in CI."
+          else
+            echo "Found $COUNT recording(s)"
+          fi
+
+      - name: Install dependencies
+        uses: ./.github/actions/setup-runner
+        with:
+          python-version: '3.12'
+
+      - name: Install ci-tests distro provider dependencies
+        run: uv run llama stack list-deps ci-tests --format uv | sh
+
+      - name: Start Llama Stack server
+        run: |
+          mkdir -p /tmp/llama-stack-conformance
+          export LLAMA_STACK_LOG_WIDTH=200
+          nohup uv run llama stack run ci-tests --port $LLAMA_STACK_PORT \
+            > /tmp/llama-stack-conformance/server.log 2>&1 &
+          echo "Server PID: $!"
+
+      - name: Wait for Llama Stack server to be ready
+        run: |
+          echo "Waiting for Llama Stack server..."
+          for i in {1..60}; do
+            if curl -s http://localhost:$LLAMA_STACK_PORT/v1/health | grep -q "OK"; then
+              echo "Llama Stack server is ready!"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Llama Stack server failed to start in 120 seconds"
+          cat /tmp/llama-stack-conformance/server.log
+          exit 1
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@3d267786b128fe76c2f16a390aa2448b815359f3 #v2.0.0
+
+      - name: Clone OpenResponses repository
+        run: |
+          git clone --depth=1 https://github.com/openresponses/openresponses.git /tmp/openresponses
+
+      - name: Install OpenResponses dependencies
+        working-directory: /tmp/openresponses
+        run: bun install
+
+      - name: Run OpenResponses conformance tests
+        id: run-tests
+        run: |
+          cd /tmp/openresponses
+          bun run bin/compliance-test.ts \
+            --base-url "http://localhost:$LLAMA_STACK_PORT/v1" \
+            --api-key "llama-stack" \
+            --model "$INFERENCE_MODEL" \
+            --json > /tmp/openresponses-results.json 2>/dev/null || true
+
+          echo "=== OpenResponses Conformance Test Output ==="
+          bun run bin/compliance-test.ts \
+            --base-url "http://localhost:$LLAMA_STACK_PORT/v1" \
+            --api-key "llama-stack" \
+            --model "$INFERENCE_MODEL" \
+            --verbose 2>&1 | tee /tmp/openresponses-output.txt || true
+
+      - name: Generate conformance report
+        if: always()
+        run: |
+          {
+            echo "## OpenResponses Conformance Test Results"
+            echo ""
+            echo "> **Note:** These tests are **informational only** and track progress toward full"
+            echo "> [OpenResponses](https://www.openresponses.org) API conformance."
+            echo "> Failures are expected while gaps remain in the Responses API implementation."
+            echo "> See [#4818](https://github.com/llamastack/llama-stack/issues/4818)."
+            echo ""
+            echo "**Model:** \`$INFERENCE_MODEL\`"
+            echo "**Recordings:** ${{ steps.check-recordings.outputs.recordings_count }} file(s) in \`tests/integration/openresponses/recordings/\`"
+            echo ""
+          } >> $GITHUB_STEP_SUMMARY
+
+          if [ "${{ steps.check-recordings.outputs.recordings_count }}" -eq 0 ]; then
+            {
+              echo "### No Recordings Found"
+              echo ""
+              echo "To enable conformance testing, generate recordings locally:"
+              echo '```bash'
+              echo "OPENAI_API_KEY=\$YOUR_KEY bash scripts/record-openresponses-conformance.sh"
+              echo ""
+              echo "# Then commit the recordings"
+              echo "git add tests/integration/openresponses/recordings/"
+              echo "git commit -m 'chore: add OpenResponses conformance recordings'"
+              echo '```'
+              echo "Commit the resulting \`tests/integration/openresponses/recordings/*.json\` files."
+            } >> $GITHUB_STEP_SUMMARY
+          elif [ -f /tmp/openresponses-results.json ] && jq -e '.summary' /tmp/openresponses-results.json > /dev/null 2>&1; then
+            PASSED=$(jq -r '.summary.passed' /tmp/openresponses-results.json)
+            FAILED=$(jq -r '.summary.failed' /tmp/openresponses-results.json)
+            TOTAL=$(jq -r '.summary.total' /tmp/openresponses-results.json)
+
+            {
+              echo "### Summary"
+              echo ""
+              echo "| Metric | Count |"
+              echo "|--------|-------|"
+              echo "| ✅ Passed | $PASSED |"
+              echo "| ❌ Failed | $FAILED |"
+              echo "| **Total** | **$TOTAL** |"
+              echo ""
+              echo "### Test Details"
+              echo ""
+              echo "| Test | Status | Duration |"
+              echo "|------|--------|----------|"
+              jq -r '.results[] | "| \(.name) | \(if .status == "passed" then "✅ Pass" else "❌ Fail" end) | \(.duration // "—")ms |"' \
+                /tmp/openresponses-results.json
+              echo ""
+            } >> $GITHUB_STEP_SUMMARY
+
+            if [ "$FAILED" -gt 0 ]; then
+              {
+                echo "### Failure Details"
+                echo ""
+                jq -r '.results[] | select(.status == "failed") | "**\(.name)**\n" + ((.errors // []) | map("- \(.)") | join("\n")) + "\n"' \
+                  /tmp/openresponses-results.json
+              } >> $GITHUB_STEP_SUMMARY
+            fi
+          else
+            {
+              echo "### Results"
+              echo ""
+              echo "Could not parse structured results."
+              if [ -f /tmp/openresponses-output.txt ]; then
+                echo ""
+                echo "<details><summary>Raw output</summary>"
+                echo ""
+                echo '```'
+                cat /tmp/openresponses-output.txt
+                echo '```'
+                echo ""
+                echo "</details>"
+              fi
+            } >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Upload conformance test results
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: openresponses-conformance-results-${{ github.run_id }}
+          path: |
+            /tmp/openresponses-results.json
+            /tmp/openresponses-output.txt
+          retention-days: 30
+          if-no-files-found: warn
+
+      - name: Print server log on failure
+        if: failure()
+        run: |
+          echo "=== Llama Stack server log (last 200 lines) ==="
+          tail -200 /tmp/llama-stack-conformance/server.log || true

.gitignore

@@ -44,5 +44,7 @@ tests/integration/client-typescript/node_modules/
 
 # client SDK generated files
 client-sdks/openapi/openapi.yml
+client-sdks/openapi/openapi-hierarchical.yml
+client-sdks/openapi/api-hierarchy.yml
 client-sdks/openapi/sdks/
 client-sdks/openapi/.openapi-generator/

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
     -   id: check-added-large-files
         args: ['--maxkb=1000']
     -   id: end-of-file-fixer
-        exclude: '^(.*\.svg|.*\.md)$'
+        exclude: '^(.*\.svg|.*\.md|.*\.mustache)$'
     -   id: no-commit-to-branch
     -   id: check-yaml
         args: ["--unsafe"]
@@ -27,6 +27,7 @@ repos:
     -   id: check-executables-have-shebangs
     -   id: check-json
     -   id: check-shebang-scripts-are-executable
+        exclude: '^.*\.mustache$'  # these are not executables, only templates of
     -   id: check-symlinks
     -   id: check-toml
 
@@ -187,6 +188,7 @@ repos:
       - id: check-log-usage
         name: Ensure 'llama_stack.log' usage for logging
         entry: bash
+        exclude: '^client-sdks/openapi/.*$'
         language: system
         types: [python]
         pass_filenames: true

client-sdks/openapi/Makefile

@@ -4,29 +4,43 @@ PYTHON := uv run python
 # Auto-detect openapi-generator command (npm installs as openapi-generator-cli, brew as openapi-generator)
 OPENAPI_GENERATOR := $(shell command -v openapi-generator-cli 2>/dev/null || command -v openapi-generator 2>/dev/null || echo openapi-generator-cli)
 
-MERGE_SCRIPT := ./merge_stainless_to_openapi.py
+# Scripts
+MERGE_SCRIPT := ./merge_stainless_config.py
+HIERARCHY_SCRIPT := ./build_hierarchy.py
+PATCH_SCRIPT := ./patch_hierarchy.py
+
+# Inputs
 PATCHES_FILE := ./patches.yml
 STAINLESS_CONFIG := $(STAINLESS_DIR)/config.yml
 STAINLESS_OPENAPI := $(STAINLESS_DIR)/openapi.yml
-OPENAPI_OUTPUT := openapi.yml
 OPENAPI_CONFIG := openapi-config.json
+TEMPLATE_DIR := ./templates/python
+
+# Outputs
+OPENAPI_OUTPUT := openapi.yml
+PROCESSED_SPEC := openapi-hierarchical.yml
+HIERARCHY_FILE := api-hierarchy.yml
 SDK_OUTPUT_DIR := sdks/python
+PACKAGE_NAME := $(shell grep '"packageName"' $(OPENAPI_CONFIG) | cut -d'"' -f4)
 
 # Extract version from root pyproject.toml
 VERSION := $(shell grep 'fallback_version' ../../pyproject.toml | cut -d'"' -f2)
 
-.PHONY: all sdk openapi clean help version check-generator
+.PHONY: all sdk openapi hierarchy clean help version check-generator
 
 all: sdk
 
 help:
 	@echo "Available targets:"
-	@echo "  all       - Generate SDK (default)"
-	@echo "  openapi   - Generate OpenAPI spec from Stainless config"
-	@echo "  sdk       - Generate Python SDK from OpenAPI spec"
-	@echo "  version   - Show version that will be used for SDK"
-	@echo "  clean     - Remove generated files"
-	@echo "  help      - Show this help message"
+	@echo "  all        - Generate SDK (default)"
+	@echo "  openapi    - Generate enriched OpenAPI spec from Stainless config"
+	@echo "  hierarchy  - Process spec for hierarchical SDK (leaf tags, dummy endpoints)"
+	@echo "  sdk        - Generate Python SDK with hierarchical API structure"
+	@echo "  version    - Show version that will be used for SDK"
+	@echo "  clean      - Remove generated files"
+	@echo "  help       - Show this help message"
+	@echo ""
+	@echo "Pipeline: openapi -> hierarchy -> sdk (each target triggers its dependencies)"
 
 version:
 	@echo "SDK version: $(VERSION)"
@@ -62,19 +76,31 @@ check-generator:
 		  echo ""; \
 		  exit 1; }
 
+# Step 1: Merge Stainless config into OpenAPI spec and apply patches
 openapi: $(OPENAPI_OUTPUT)
 
 $(OPENAPI_OUTPUT): $(MERGE_SCRIPT) $(PATCHES_FILE) $(STAINLESS_CONFIG) $(STAINLESS_OPENAPI)
 	@echo "Generating OpenAPI spec..."
 	$(PYTHON) $(MERGE_SCRIPT) --patch $(PATCHES_FILE) --stainless $(STAINLESS_CONFIG) --openapi $(STAINLESS_OPENAPI) --output $(OPENAPI_OUTPUT)
 
-sdk: check-generator $(OPENAPI_OUTPUT)
+# Step 2: Extract tag hierarchy, reduce to leaf tags, create dummy endpoints, fix schemas
+hierarchy: $(PROCESSED_SPEC)
+
+$(PROCESSED_SPEC): $(HIERARCHY_SCRIPT) $(OPENAPI_OUTPUT)
+	@echo "Processing hierarchy..."
+	$(PYTHON) $(HIERARCHY_SCRIPT) --source $(OPENAPI_OUTPUT) --output $(PROCESSED_SPEC) --hierarchy $(HIERARCHY_FILE)
+
+# Step 3: Generate Python SDK from processed spec, then patch in hierarchy
+sdk: check-generator $(PROCESSED_SPEC)
 	@echo "Generating Python SDK (version: $(VERSION))..."
 	@mkdir -p $(SDK_OUTPUT_DIR)
-	$(OPENAPI_GENERATOR) generate -i $(OPENAPI_OUTPUT) -g python -c $(OPENAPI_CONFIG) -o $(SDK_OUTPUT_DIR) \
-		--additional-properties=packageVersion=$(VERSION)
+	$(OPENAPI_GENERATOR) generate -i $(PROCESSED_SPEC) -g python -c $(OPENAPI_CONFIG) -o $(SDK_OUTPUT_DIR) \
+		--additional-properties=packageVersion=$(VERSION) -t $(TEMPLATE_DIR)
+	@cp -R $(TEMPLATE_DIR)/lib $(SDK_OUTPUT_DIR)/$(PACKAGE_NAME)
+	@echo "Patching SDK with hierarchical API structure..."
+	$(PYTHON) $(PATCH_SCRIPT) --hierarchy $(HIERARCHY_FILE) --sdk-dir $(SDK_OUTPUT_DIR) --package $(PACKAGE_NAME)
 
 clean:
 	@echo "Cleaning generated files..."
-	@rm -f $(OPENAPI_OUTPUT)
+	@rm -f $(OPENAPI_OUTPUT) $(PROCESSED_SPEC) $(HIERARCHY_FILE)
 	@rm -rf sdks