fix: scrub secret values from CI log artifacts before upload (#488)

## Summary
- Add a defense-in-depth scrub step to the CI workflow that replaces actual secret env-var values with `***REDACTED***` in all log files before the `upload-artifact` step
- Add a `check-secret-scrub` pre-commit hook that ensures the scrub list stays in sync with `smoke.sh` — greps for secret-looking env vars (`KEY`, `TOKEN`, `PASSWORD`, `SECRET`, `CREDENTIAL`) passed to the container and fails if any are missing from the workflow's scrub list

## Test plan
- [x] `pre-commit run check-secret-scrub --all-files` passes
- [x] Temporarily removing a var from the scrub list causes the hook to fail with a clear error message
- [ ] CI run uploads log artifacts with redacted values

🤖 Generated with [Claude Code](https://claude.com/claude-code)


## Summary by CodeRabbit

* **New Features**
  * Added automatic redaction of sensitive values from generated log and test-result artifacts before they’re uploaded.
  * Introduced CI secret-scrubbing steps across multiple response-test workflows.
  * Added a pre-commit hook to validate consistency between local smoke-test secret inputs and CI scrubbing configuration.

* **Bug Fixes**
  * Reduced the risk of accidentally exposing tokens, passwords, credentials, and related secret values in uploaded artifacts.



Approved-by: Artemon-line

Approved-by: cdoern

Author: mergify[bot]

.github/workflows/redhat-distro-container.yml

@@ -302,6 +302,13 @@ jobs:
             done
           fi
 
+          # Scrub secret values from log files before upload (CWE-532)
+          ./tests/scrub_secrets.sh 'logs/*.log' \
+            VLLM_API_TOKEN VLLM_EMBEDDING_API_TOKEN \
+            OPENAI_API_KEY GEMINI_API_KEY ANTHROPIC_API_KEY \
+            POSTGRES_PASSWORD PGVECTOR_PASSWORD \
+            GOOGLE_APPLICATION_CREDENTIALS
+
       - name: Upload logs as artifacts
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/responses-openai.yml

@@ -142,6 +142,15 @@ jobs:
           reporter: java-junit
           fail-on-error: 'false'
 
+      - name: Scrub secrets from test results
+        if: ${{ !cancelled() }}
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          TAVILY_SEARCH_API_KEY: ${{ secrets.TAVILY_SEARCH_API_KEY }}
+        run: |
+          ./tests/scrub_secrets.sh '/tmp/test-results/*' \
+            OPENAI_API_KEY TAVILY_SEARCH_API_KEY
+
       - name: Upload artifacts
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/responses-vertexai.yml

@@ -158,6 +158,15 @@ jobs:
           reporter: java-junit
           fail-on-error: 'false'
 
+      - name: Scrub secrets from test results
+        if: ${{ !cancelled() }}
+        env:
+          VERTEX_AI_PROJECT: ${{ secrets.VERTEX_AI_PROJECT }}
+          TAVILY_SEARCH_API_KEY: ${{ secrets.TAVILY_SEARCH_API_KEY }}
+        run: |
+          ./tests/scrub_secrets.sh '/tmp/test-results/*' \
+            VERTEX_AI_PROJECT TAVILY_SEARCH_API_KEY
+
       - name: Upload artifacts
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/responses-vllm-maas.yml

@@ -146,6 +146,16 @@ jobs:
           reporter: java-junit
           fail-on-error: 'false'
 
+      - name: Scrub secrets from test results
+        if: ${{ !cancelled() }}
+        env:
+          VLLM_API_TOKEN: ${{ secrets.MAAS_VLLM_API_TOKEN }}
+          VLLM_EMBEDDING_API_TOKEN: ${{ secrets.MAAS_EMBEDDING_API_TOKEN }}
+          TAVILY_SEARCH_API_KEY: ${{ secrets.TAVILY_SEARCH_API_KEY }}
+        run: |
+          ./tests/scrub_secrets.sh '/tmp/test-results/*' \
+            VLLM_API_TOKEN VLLM_EMBEDDING_API_TOKEN TAVILY_SEARCH_API_KEY
+
       - name: Upload artifacts
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.pre-commit-config.yaml

@@ -61,3 +61,10 @@ repos:
         files: ^(build/(build\.yaml|build\.env)|distribution/config\.yaml)$
         additional_dependencies:
           - pyyaml==6.0.2
+
+      - id: check-secret-scrub
+        name: CI log-scrub list matches smoke.sh secrets
+        entry: ./tests/check_secret_scrub_list.sh
+        language: script
+        pass_filenames: false
+        files: ^(tests/smoke\.sh|\.github/workflows/redhat-distro-container\.yml)$

tests/check_secret_scrub_list.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Ensure every secret-looking env var passed to the OGX container in smoke.sh
+# is listed in the CI workflow's log-scrub step. Exits non-zero on drift.
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+SMOKE="$REPO_ROOT/tests/smoke.sh"
+WORKFLOW="$REPO_ROOT/.github/workflows/redhat-distro-container.yml"
+
+# Extract env var names passed via --env to the docker container in smoke.sh
+smoke_vars=$(grep -oP '(?<=--env ")([A-Z_]+)(?==)' "$SMOKE" | sort -u)
+
+# Filter to secret-looking names
+secret_pattern='KEY|TOKEN|PASSWORD|SECRET|CREDENTIAL'
+smoke_secrets=$(echo "$smoke_vars" | grep -E "$secret_pattern" || true)
+
+if [ -z "$smoke_secrets" ]; then
+  echo "No secret-looking env vars found in smoke.sh (unexpected)"
+  exit 1
+fi
+
+# Extract the var names from the workflow's scrub_secrets.sh invocation
+scrub_vars=$(grep -A10 'scrub_secrets\.sh' "$WORKFLOW" \
+  | grep -oP '\b[A-Z][A-Z_]{2,}\b' \
+  | grep -E "$secret_pattern" \
+  | sort -u)
+
+missing=()
+while IFS= read -r var; do
+  if ! echo "$scrub_vars" | grep -qx "$var"; then
+    missing+=("$var")
+  fi
+done <<< "$smoke_secrets"
+
+if [ ${#missing[@]} -gt 0 ]; then
+  echo "ERROR: Secret env var(s) passed to the OGX container in smoke.sh"
+  echo "are missing from the CI log-scrub list in redhat-distro-container.yml:"
+  for v in "${missing[@]}"; do
+    echo "  - $v"
+  done
+  echo ""
+  echo "Add them to the Python scrub snippet in the 'Gather logs' step."
+  exit 1
+fi
+
+echo "All secret env vars in smoke.sh are in the CI log-scrub list."

tests/scrub_secrets.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Scrub secret values from files before artifact upload (CWE-532).
+#
+# Usage: scrub_secrets.sh <glob> <ENV_VAR_NAME> [ENV_VAR_NAME ...]
+#
+# Each ENV_VAR_NAME is read from the environment.  Values shorter than
+# 4 characters are silently skipped to avoid false-positive replacements.
+#
+# Example:
+#   scrub_secrets.sh 'logs/*.log' VLLM_API_TOKEN OPENAI_API_KEY
+
+set -euo pipefail
+
+if [ $# -lt 2 ]; then
+  echo "Usage: $0 <glob> <ENV_VAR_NAME> [ENV_VAR_NAME ...]" >&2
+  exit 1
+fi
+
+glob_pattern="$1"
+shift
+
+python3 -c "
+import os, glob, sys
+
+var_names = sys.argv[1:]
+secrets = [os.environ.get(v, '') for v in var_names]
+secrets = [s for s in secrets if len(s) >= 4]
+
+for f in glob.glob(sys.argv[0]):
+    try:
+        with open(f, 'r', errors='replace') as fh:
+            content = fh.read()
+        for s in secrets:
+            content = content.replace(s, '***REDACTED***')
+        with open(f, 'w') as fh:
+            fh.write(content)
+    except IsADirectoryError:
+        pass
+" "$glob_pattern" "$@"