Merge pull request #13 from Semih702/ci/helm-regression-contract-tests

ci: add OpenAI-compatible contract tests to helm regression pipeline

Author: Semih702

.github/workflows/helm-regression.yaml

@@ -78,11 +78,13 @@ jobs:
         run: |
           docker build -t llm-proxy:ci ./proxy
           docker build -t llm-collector:ci ./collector
+          docker build -t mock-openai:ci ./tests/contract/mock-openai
 
       - name: Load images into kind
         run: |
           kind load docker-image llm-proxy:ci --name chart-testing
           kind load docker-image llm-collector:ci --name chart-testing
+          kind load docker-image mock-openai:ci --name chart-testing
 
       - name: Create dummy secret for CI
         run: |
@@ -92,6 +94,12 @@ jobs:
             --from-literal=UPSTREAM_OPENAI_API_KEY=dummy-key \
             --dry-run=client -o yaml | kubectl apply -f -
       
+      - name: Deploy mock OpenAI upstream
+        run: |
+          set -euo pipefail
+          kubectl apply -n "${NAMESPACE}" -f tests/contract/k8s/mock-openai.yaml
+          kubectl rollout status deployment/mock-openai -n "${NAMESPACE}" --timeout=180s
+
       - name: Install chart into kind (default)
         run: |
           set -euo pipefail
@@ -104,6 +112,7 @@ jobs:
             --set collector.image.repository=llm-collector \
             --set collector.image.tag=ci \
             --set collector.image.pullPolicy=IfNotPresent \
+            --set proxy.env.UPSTREAM_OPENAI_BASE_URL="http://mock-openai:8080" \
             -n "${NAMESPACE}" --create-namespace
 
           kubectl get all -n "${NAMESPACE}"
@@ -133,3 +142,10 @@ jobs:
 
           kubectl wait --for=condition=Available deployment/llm-proxy -n "${NAMESPACE}" --timeout=180s
           kubectl wait --for=condition=Available deployment/llm-collector -n "${NAMESPACE}" --timeout=180s
+
+      - name: Contract tests (OpenAI-compatible contract)
+        shell: bash
+        run: |
+          chmod +x tests/contract/ci-contract.sh
+          NAMESPACE="${NAMESPACE}" ./tests/contract/ci-contract.sh
+

README.md

@@ -230,64 +230,84 @@ Collector delivery uses a short HTTP timeout and never blocks the request path;
 
 ---
 
-CI Regression & Chart Testing (for Contributors)
+## CI Regression & Chart Testing (for Contributors)
 
 This repository includes a mandatory CI regression pipeline to ensure Helm chart correctness,
 Kubernetes compatibility, and safe upgrade behavior.
 
 All pull requests that modify Helm charts or CI workflows must pass this pipeline before being merged.
 
-What is validated in CI?
+---
+
+### What is validated in CI?
 
 The helm-regression workflow performs the following checks, in order:
 
 1. Helm chart linting
-- Runs helm lint on the chart
-- Catches common issues:
-  - Invalid chart metadata
-  - Obvious template mistakes
-  - Missing required values
+   - Runs `helm lint` on the chart
+   - Catches common issues:
+     - Invalid chart metadata
+     - Obvious template mistakes
+     - Missing required values
 
 2. Template rendering (dry-run)
-- Renders manifests using:
-  - values-ci.yaml (required)
-  - values-test.yaml (optional, if present)
-- Ensures templates render successfully without a live cluster
+   - Renders manifests using:
+     - `values-ci.yaml` (required)
+     - `values-test.yaml` (optional, if present)
+   - Ensures templates render successfully without a live cluster
 
 3. Kubernetes schema validation (kubeconform)
-- Validates rendered YAML against official Kubernetes schemas
-- Uses strict mode
-- Catches:
-  - Invalid API versions
-  - Invalid fields
-  - Structural mismatches that Helm itself does not detect
+   - Validates rendered YAML against official Kubernetes schemas
+   - Uses strict mode
+   - Catches:
+     - Invalid API versions
+     - Invalid fields
+     - Structural mismatches that Helm itself does not detect
 
 4. Helm unit tests
-- Runs helm unittest against the chart
-- Validates:
-  - Expected resources are created
-  - Correct values are applied
-  - Conditional logic behaves as intended
+   - Runs `helm unittest` against the chart
+   - Validates:
+     - Expected resources are created
+     - Correct values are applied
+     - Conditional logic behaves as intended
 
 5. In-cluster install & upgrade test (kind)
-- Spins up a real Kubernetes cluster using kind
-- Builds proxy and collector images locally
-- Loads images into the cluster
-- Installs the chart using helm upgrade --install
-- Verifies:
-  - Pods start successfully
-  - Deployments become ready
-- Performs a real Helm upgrade
-  - Uses values-test.yaml if present
-  - Ensures upgrades do not break running workloads
+   - Spins up a real Kubernetes cluster using kind
+   - Builds proxy and collector images locally
+   - Loads images into the cluster
+   - Installs the chart using `helm upgrade --install`
+   - Verifies:
+     - Pods start successfully
+     - Deployments become ready
+   - Performs a real Helm upgrade:
+     - Uses `values-test.yaml` if present
+     - Ensures upgrades do not break running workloads
 
 6. Smoke checks
-- Confirms:
-  - Pods are running
-  - Services are created
-  - Deployments reach Available condition
+   - Confirms:
+     - Pods are running
+     - Services are created
+     - Deployments reach Available condition
+
+7. OpenAI-compatible contract tests
+   - Deploys a lightweight in-cluster mock OpenAI upstream
+   - Configures the gateway to point to the mock via `UPSTREAM_OPENAI_BASE_URL`
+   - Sends real HTTP requests to the gateway service
+   - Validates the public OpenAI-compatible contract, including:
+     - Chat completions (non-streaming)
+     - VLM-style payloads (`image_url` content)
+     - Streaming (SSE) passthrough with `[DONE]`
+     - Presence of `usage` fields in responses
+
+   These tests intentionally validate the API contract and behavior,
+   not model semantics or response quality.
+
+   They ensure that changes to the proxy, chart, or deployment logic
+   do not silently break OpenAI compatibility.
+
+---
 
-Files contributors should be aware of:
+### Files contributors should be aware of
 
 - charts/llm-gateway/values-ci.yaml
   Required for CI. Used for deterministic, non-secret test installs.
@@ -298,7 +318,13 @@ Files contributors should be aware of:
 - .github/workflows/helm-regression.yml
   CI definition. Any change here is also gated by this workflow.
 
-Running key checks locally (recommended):
+- tests/contract/
+  Contains OpenAI-compatible contract tests executed in CI and runnable locally.
+  See tests/contract/README.md for details.
+
+---
+
+### Running key checks locally (recommended)
 
 helm lint charts/llm-gateway
 
@@ -311,7 +337,9 @@ For full parity with CI (optional but ideal):
 - Build images locally
 - Install the chart with helm upgrade --install
 
-CI expectations:
+---
+
+### CI expectations
 
 - CI failures must be fixed, not bypassed
 - Do not disable schema validation or tests to make CI green
@@ -325,13 +353,15 @@ This pipeline exists to ensure the Helm chart remains:
 - Kubernetes-version compatible
 - Predictable across environments
 
+---
+
 ## Planned next steps
 
 ### Regression & compatibility test suite
 - Introduce a minimal regression test suite to validate the gateway’s public contract ✅
 - Focus on high-risk areas:
-  - OpenAI-compatible request/response schemas
-  - Streaming (SSE) pass-through behavior
+  - OpenAI-compatible request/response schemas ✅
+  - Streaming (SSE) pass-through behavior ✅
   - Error and status code mapping
   - Metering event emission (usage present vs missing)
 - Use a mock upstream and lightweight in-cluster setup (e.g., kind or docker-compose) ✅

tests/contract/README.md

@@ -0,0 +1,56 @@
+# Contract tests
+
+These tests validate the LLM Gateway's public OpenAI-compatible contract in a deterministic way.
+
+## Scope
+- Chat completions (non-stream)
+- VLM-style payloads (messages content with `image_url`)
+- Streaming passthrough (SSE) with `[DONE]`
+
+## Design
+- A lightweight in-cluster `mock-openai` acts as upstream.
+- The gateway is deployed to a kind cluster via the Helm chart.
+- Tests send HTTP requests to the gateway service and assert the response shape and streaming behavior (not model semantics).
+
+## Files
+- `mock-openai/`: mock upstream server implementation + Dockerfile
+- `k8s/`: Kubernetes manifests for the mock upstream
+- `fixtures/`: request payload fixtures
+
+## Adding a new test
+1. Add a new request fixture under `fixtures/`.
+2. Extend `mock-openai/main.go` if a new upstream behavior is needed.
+3. Extend the contract test runner (ci-contract.sh) with new assertions.
+
+## CI usage
+
+In CI, the helm-regression workflow:
+- Creates a kind cluster
+- Builds and loads images (llm-proxy, llm-collector, mock-openai)
+- Deploys mock-openai
+- Installs the Helm chart with UPSTREAM_OPENAI_BASE_URL pointing to mock-openai
+- Runs tests/contract/ci-contract.sh
+
+The CI runner assumes:
+- A reachable Kubernetes cluster
+- llm-gateway already installed
+- mock-openai running
+- Proxy service reachable inside the cluster
+
+It does not perform any setup.
+
+## Local usage
+
+Make scripts executable:
+chmod +x tests/contract/local-run.sh
+chmod +x tests/contract/ci-contract.sh
+
+Run:
+./tests/contract/local-run.sh
+
+This mirrors the CI workflow:
+- Creates or reuses a kind cluster
+- Builds and loads images
+- Deploys mock-openai
+- Installs the chart pointing to the mock upstream
+- Runs the same CI contract tests

tests/contract/ci-contract.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+NAMESPACE="${NAMESPACE:-llm-system}"
+PROXY_SVC="${PROXY_SVC:-llm-proxy}"
+PROXY_PORT="${PROXY_PORT:-8080}"
+
+need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing: $1" >&2; exit 1; }; }
+need kubectl
+
+echo "[contract] namespace=${NAMESPACE} target=http://${PROXY_SVC}:${PROXY_PORT}"
+
+# run inside cluster (no port-forward flake)
+kubectl run -n "${NAMESPACE}" contract-curl --rm -i --restart=Never \
+  --image=curlimages/curl:8.6.0 \
+  -- sh -lc "
+    set -e
+
+    echo '--- normal ---'
+    cat <<'JSON' | curl -sS http://${PROXY_SVC}:${PROXY_PORT}/v1/chat/completions \
+      -H 'Content-Type: application/json' \
+      -H 'Authorization: Bearer dummy' \
+      -d @- | tee /tmp/normal.json >/dev/null
+{
+  \"model\": \"gpt-4o-mini\",
+  \"messages\": [{\"role\":\"user\",\"content\":\"hi\"}]
+}
+JSON
+    grep -q 'ok-from-mock' /tmp/normal.json
+    grep -q '\"usage\"' /tmp/normal.json
+
+    echo '--- vlm ---'
+    cat <<'JSON' | curl -sS http://${PROXY_SVC}:${PROXY_PORT}/v1/chat/completions \
+      -H 'Content-Type: application/json' \
+      -H 'Authorization: Bearer dummy' \
+      -d @- | tee /tmp/vlm.json >/dev/null
+{
+  \"model\": \"gpt-4o-mini\",
+  \"messages\": [
+    {
+      \"role\": \"user\",
+      \"content\": [
+        {\"type\":\"text\",\"text\":\"what is in this image?\"},
+        {\"type\":\"image_url\",\"image_url\":{\"url\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO2Xc+UAAAAASUVORK5CYII=\"}}
+      ]
+    }
+  ]
+}
+JSON
+    grep -q 'ok-from-mock' /tmp/vlm.json
+    grep -q '\"usage\"' /tmp/vlm.json
+
+    echo '--- stream ---'
+    cat <<'JSON' | curl -sS -N http://${PROXY_SVC}:${PROXY_PORT}/v1/chat/completions \
+      -H 'Content-Type: application/json' \
+      -H 'Authorization: Bearer dummy' \
+      -d @- | tee /tmp/stream.txt >/dev/null
+{
+  \"model\": \"gpt-4o-mini\",
+  \"stream\": true,
+  \"messages\": [{\"role\":\"user\",\"content\":\"stream pls\"}]
+}
+JSON
+    grep -q '\\[DONE\\]' /tmp/stream.txt
+
+    echo 'OK: contract tests passed.'
+  "

tests/contract/fixtures/normal.json

@@ -0,0 +1,9 @@
+{
+  "model": "gpt-4o-mini",
+  "messages": [
+    {
+      "role": "user",
+      "content": "hi from llm-gateway!"
+    }
+  ]
+}

tests/contract/fixtures/stream.json

@@ -0,0 +1,10 @@
+{
+  "model": "gpt-4o-mini",
+  "stream": true,
+  "messages": [
+    {
+      "role": "user",
+      "content": "may the stream be with you!"
+    }
+  ]
+}

tests/contract/fixtures/vlm.json

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mock-openai
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mock-openai
+  template:
+    metadata:
+      labels:
+        app: mock-openai
+    spec:
+      containers:
+        - name: mock-openai
+          image: mock-openai:ci
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-openai
+spec:
+  selector:
+    app: mock-openai
+  ports:
+    - name: http
+      port: 8080
+      targetPort: 8080