Main Channel RTT #59
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main Channel RTT | |
| on: | |
| schedule: | |
| - cron: "45 */6 * * *" | |
| pull_request: | |
| paths: | |
| - ".github/actionlint.yaml" | |
| - ".github/workflows/ci.yml" | |
| - ".github/workflows/codeql.yml" | |
| - ".github/workflows/main-channel-rtt.yml" | |
| - "scripts/channel-rtt-config.mjs" | |
| - "scripts/channel-rtt-summary.mjs" | |
| - "scripts/import-live-transport-rtt.mjs" | |
| - "scripts/read-channel-rtt-rows.mjs" | |
| - "scripts/validate.mjs" | |
| workflow_dispatch: | |
| inputs: | |
| channels: | |
| description: Space or comma separated channel ids | |
| required: false | |
| default: "slack whatsapp" | |
| type: string | |
| samples: | |
| description: Number of RTT samples | |
| required: false | |
| default: "20" | |
| type: string | |
| sample_timeout_seconds: | |
| description: Per-sample timeout in seconds | |
| required: false | |
| default: "" | |
| type: string | |
| permissions: | |
| actions: read | |
| contents: write | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" | |
| NODE_VERSION: "24.x" | |
| PNPM_VERSION: "10.33.0" | |
| CHANNEL_RTT_PR_SAMPLES: "3" | |
| CHANNEL_RTT_MAX_ATTEMPTS: "3" | |
| CHANNEL_RTT_RETRY_BASE_SECONDS: "15" | |
| CHANNEL_RTT_RETRY_MAX_SECONDS: "60" | |
| # Must exceed OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS so lease waits can | |
| # finish and the QA CLI can write a structured failure summary. | |
| CHANNEL_RTT_SAMPLE_TIMEOUT_SECONDS: "300" | |
| jobs: | |
| resolve: | |
| name: Resolve main channel RTT matrix | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| matrix: ${{ steps.channels.outputs.matrix }} | |
| should_run: ${{ steps.channels.outputs.should_run }} | |
| steps: | |
| - name: Resolve channels | |
| id: channels | |
| env: | |
| INPUT_CHANNELS: ${{ github.event_name == 'workflow_dispatch' && inputs.channels || 'slack whatsapp' }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| node <<'NODE' >>"$GITHUB_OUTPUT" | |
| const requested = new Set( | |
| (process.env.INPUT_CHANNELS || "slack whatsapp") | |
| .split(/[\s,]+/) | |
| .map((value) => value.trim().toLowerCase()) | |
| .filter(Boolean), | |
| ); | |
| const channels = [ | |
| { | |
| channel: "slack", | |
| label: "Slack", | |
| scenario: "slack-canary", | |
| summary: "slack-qa-summary.json", | |
| observed: "slack-qa-observed-messages.json", | |
| }, | |
| { | |
| channel: "whatsapp", | |
| label: "WhatsApp", | |
| scenario: "whatsapp-canary", | |
| summary: "whatsapp-qa-summary.json", | |
| observed: "whatsapp-qa-observed-messages.json", | |
| }, | |
| ].filter((entry) => requested.has(entry.channel)); | |
| if (channels.length === 0) { | |
| console.log("should_run=false"); | |
| console.log("matrix={\"include\":[]}"); | |
| } else { | |
| console.log("should_run=true"); | |
| console.log(`matrix=${JSON.stringify({ include: channels })}`); | |
| } | |
| NODE | |
| measure: | |
| name: Measure OpenClaw main ${{ matrix.label }} RTT | |
| needs: resolve | |
| if: needs.resolve.outputs.should_run == 'true' | |
| runs-on: blacksmith-32vcpu-ubuntu-2404 | |
| timeout-minutes: 120 | |
| environment: qa-live-shared | |
| concurrency: | |
| group: channel-rtt-measure-${{ matrix.channel }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }} | |
| steps: | |
| - name: Checkout RTT tracker | |
| uses: actions/checkout@v6 | |
| with: | |
| path: openclaw-rtt | |
| fetch-depth: 0 | |
| - name: Checkout OpenClaw main | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: openclaw/openclaw | |
| ref: main | |
| path: openclaw | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| - name: Setup Node | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Setup pnpm | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| corepack enable | |
| corepack prepare "pnpm@${PNPM_VERSION}" --activate | |
| - name: Locate pnpm store | |
| id: pnpm-store | |
| working-directory: openclaw | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| echo "path=$(pnpm store path --silent)" >>"$GITHUB_OUTPUT" | |
| - name: Restore pnpm store | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ steps.pnpm-store.outputs.path }} | |
| key: openclaw-main-channel-rtt-pnpm-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('openclaw/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| openclaw-main-channel-rtt-pnpm-${{ runner.os }}-${{ runner.arch }}- | |
| - name: Install OpenClaw dependencies | |
| working-directory: openclaw | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| time pnpm install --frozen-lockfile --prefer-offline --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true --config.side-effects-cache=true | |
| - name: Resolve OpenClaw ref | |
| id: openclaw_ref | |
| working-directory: openclaw | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| version="$(node -p "require('./package.json').version")" | |
| revision="$(git rev-parse --short=10 HEAD)" | |
| echo "version=${version}+${revision}" >>"$GITHUB_OUTPUT" | |
| - name: Build private QA runtime | |
| working-directory: openclaw | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=8192 | |
| OPENCLAW_BUILD_PRIVATE_QA: "1" | |
| OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1" | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| time pnpm build | |
| - name: Run ${{ matrix.label }} RTT samples | |
| id: channel_rtt | |
| working-directory: openclaw | |
| env: | |
| INPUT_SAMPLES: ${{ github.event_name == 'workflow_dispatch' && inputs.samples || '' }} | |
| INPUT_SAMPLE_TIMEOUT_SECONDS: ${{ github.event_name == 'workflow_dispatch' && inputs.sample_timeout_seconds || '' }} | |
| OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} | |
| OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} | |
| OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "180000" | |
| OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "60000" | |
| OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then | |
| samples="${INPUT_SAMPLES:-$CHANNEL_RTT_PR_SAMPLES}" | |
| else | |
| samples="${INPUT_SAMPLES:-20}" | |
| fi | |
| if ! [[ "$samples" =~ ^[1-9][0-9]*$ ]]; then | |
| echo "samples must be a positive integer, got: $samples" >&2 | |
| exit 1 | |
| fi | |
| max_attempts="${CHANNEL_RTT_MAX_ATTEMPTS:-3}" | |
| retry_base_seconds="${CHANNEL_RTT_RETRY_BASE_SECONDS:-15}" | |
| retry_max_seconds="${CHANNEL_RTT_RETRY_MAX_SECONDS:-60}" | |
| sample_timeout_seconds="${INPUT_SAMPLE_TIMEOUT_SECONDS:-${CHANNEL_RTT_SAMPLE_TIMEOUT_SECONDS:-120}}" | |
| for value_name in max_attempts retry_base_seconds retry_max_seconds sample_timeout_seconds; do | |
| value="${!value_name}" | |
| if ! [[ "$value" =~ ^[1-9][0-9]*$ ]]; then | |
| echo "${value_name} must be a positive integer, got: ${value}" >&2 | |
| exit 1 | |
| fi | |
| done | |
| output_root=".artifacts/qa-e2e/${{ matrix.channel }}-rtt-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| sample_paths="$RUNNER_TEMP/openclaw-${{ matrix.channel }}-rtt-samples.tsv" | |
| : >"$sample_paths" | |
| echo "output_root=${output_root}" >>"$GITHUB_OUTPUT" | |
| echo "sample_paths=${sample_paths}" >>"$GITHUB_OUTPUT" | |
| for sample in $(seq 1 "$samples"); do | |
| output_dir="${output_root}/sample-${sample}" | |
| summary_path="${output_dir}/${{ matrix.summary }}" | |
| observed_path="${output_dir}/${{ matrix.observed }}" | |
| metrics_path="${output_dir}/resource-metrics.env" | |
| sample_started_at="$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ")" | |
| status=1 | |
| for attempt in $(seq 1 "$max_attempts"); do | |
| rm -rf "$output_dir" | |
| mkdir -p "$output_dir" | |
| echo "CHANNEL_RTT_PHASE:${{ matrix.channel }} sample=${sample}/${samples} attempt=${attempt}/${max_attempts}" | |
| set +e | |
| /usr/bin/time \ | |
| -f 'max_rss_kb=%M\nelapsed_seconds=%e' \ | |
| -o "$metrics_path" \ | |
| timeout "${sample_timeout_seconds}s" \ | |
| pnpm openclaw qa "${{ matrix.channel }}" \ | |
| --repo-root . \ | |
| --output-dir "${output_dir}" \ | |
| --provider-mode mock-openai \ | |
| --model mock-openai/gpt-5.5 \ | |
| --alt-model mock-openai/gpt-5.5-alt \ | |
| --fast \ | |
| --credential-source convex \ | |
| --credential-role ci \ | |
| --scenario "${{ matrix.scenario }}" \ | |
| --allow-failures | |
| status="$?" | |
| set -e | |
| { | |
| echo "sample=${sample}" | |
| echo "attempts=${attempt}" | |
| echo "exit_status=${status}" | |
| } >>"$metrics_path" | |
| scenario_status="" | |
| if [[ -f "$summary_path" ]]; then | |
| scenario_status="$(node -e 'const fs = require("node:fs"); const [summaryPath, scenarioId] = process.argv.slice(1); const summary = JSON.parse(fs.readFileSync(summaryPath, "utf8")); const scenario = Array.isArray(summary.scenarios) ? summary.scenarios.find((item) => item && item.id === scenarioId) : undefined; process.stdout.write(typeof scenario?.status === "string" ? scenario.status : "missing");' "$summary_path" "${{ matrix.scenario }}")" | |
| fi | |
| if [[ "$scenario_status" == "pass" ]]; then | |
| break | |
| fi | |
| if [[ "$attempt" -lt "$max_attempts" ]]; then | |
| delay=$((retry_base_seconds * (2 ** (attempt - 1)))) | |
| if [[ "$delay" -gt "$retry_max_seconds" ]]; then | |
| delay="$retry_max_seconds" | |
| fi | |
| echo "${{ matrix.label }} QA sample ${sample} attempt ${attempt} did not pass (status=${scenario_status:-no-summary}); retrying in ${delay}s." >&2 | |
| sleep "$delay" | |
| fi | |
| done | |
| if [[ ! -f "$summary_path" ]]; then | |
| echo "No ${{ matrix.label }} QA summary produced for sample ${sample}; recording failed sample." >&2 | |
| sample_finished_at="$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ")" | |
| node "${GITHUB_WORKSPACE}/openclaw-rtt/scripts/write-failed-channel-summary.mjs" \ | |
| "$summary_path" \ | |
| "${{ matrix.scenario }}" \ | |
| "${{ matrix.label }} canary" \ | |
| "QA command exited with status ${status} before writing a summary." \ | |
| "$sample_started_at" \ | |
| "$sample_finished_at" | |
| scenario_status="fail" | |
| fi | |
| if [[ -f "$observed_path" ]]; then | |
| printf '%s\t%s\t%s\n' "${PWD}/${summary_path}" "${PWD}/${observed_path}" "${PWD}/${metrics_path}" >>"$sample_paths" | |
| else | |
| printf '%s\t\t%s\n' "${PWD}/${summary_path}" "${PWD}/${metrics_path}" >>"$sample_paths" | |
| fi | |
| if [[ "$scenario_status" != "pass" ]]; then | |
| break | |
| fi | |
| done | |
| - name: Prepare ${{ matrix.label }} import artifact | |
| working-directory: openclaw | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| import_dir="$RUNNER_TEMP/main-channel-rtt-import-${{ matrix.channel }}" | |
| rm -rf "$import_dir" | |
| mkdir -p "$import_dir" | |
| cp -R "${{ steps.channel_rtt.outputs.output_root }}" "$import_dir/artifacts" | |
| { | |
| echo "channel=${{ matrix.channel }}" | |
| echo "scenario=${{ matrix.scenario }}" | |
| echo "summary=${{ matrix.summary }}" | |
| echo "observed=${{ matrix.observed }}" | |
| echo "version=${{ steps.openclaw_ref.outputs.version }}" | |
| } >"$import_dir/metadata.env" | |
| - name: Upload ${{ matrix.label }} import artifact | |
| uses: actions/upload-artifact@v7.0.1 | |
| with: | |
| name: main-channel-rtt-import-${{ matrix.channel }} | |
| path: ${{ runner.temp }}/main-channel-rtt-import-${{ matrix.channel }} | |
| retention-days: 14 | |
| if-no-files-found: error | |
| - name: Upload ${{ matrix.label }} RTT artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v7.0.1 | |
| with: | |
| name: main-${{ matrix.channel }}-rtt-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: openclaw/.artifacts/qa-e2e/${{ matrix.channel }}-rtt-${{ github.run_id }}-${{ github.run_attempt }} | |
| retention-days: 14 | |
| if-no-files-found: warn | |
| report: | |
| name: Import channel RTT reports | |
| needs: | |
| - resolve | |
| - measure | |
| if: always() && !cancelled() && needs.resolve.outputs.should_run == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| concurrency: | |
| group: rtt-report-writer-${{ github.event_name == 'pull_request' && github.run_id || github.ref }} | |
| cancel-in-progress: false | |
| steps: | |
| - name: Checkout RTT tracker | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Node | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Download channel RTT import artifacts | |
| uses: actions/download-artifact@v8.0.1 | |
| with: | |
| pattern: main-channel-rtt-import-* | |
| path: ${{ runner.temp }}/channel-rtt-imports | |
| - name: Import channel RTT results | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [[ "${GITHUB_EVENT_NAME}" != "pull_request" ]]; then | |
| git pull --rebase origin main | |
| fi | |
| import_root="$RUNNER_TEMP/channel-rtt-imports" | |
| shopt -s nullglob | |
| import_dirs=("$import_root"/main-channel-rtt-import-*) | |
| if [[ -f "$import_root/metadata.env" ]]; then | |
| import_dirs=("$import_root") | |
| fi | |
| if [[ "${#import_dirs[@]}" -eq 0 ]]; then | |
| echo "No channel RTT import artifacts downloaded." >&2 | |
| exit 1 | |
| fi | |
| get_metadata() { | |
| local metadata_path="$1" | |
| local key="$2" | |
| grep -m 1 "^${key}=" "$metadata_path" | cut -d= -f2- | |
| } | |
| for import_dir in "${import_dirs[@]}"; do | |
| metadata_path="${import_dir}/metadata.env" | |
| channel="$(get_metadata "$metadata_path" channel)" | |
| scenario="$(get_metadata "$metadata_path" scenario)" | |
| summary="$(get_metadata "$metadata_path" summary)" | |
| observed="$(get_metadata "$metadata_path" observed)" | |
| version="$(get_metadata "$metadata_path" version)" | |
| sample_paths="$RUNNER_TEMP/openclaw-${channel}-rtt-samples.tsv" | |
| : >"$sample_paths" | |
| while IFS= read -r sample_dir; do | |
| summary_path="${sample_dir}/${summary}" | |
| observed_path="${sample_dir}/${observed}" | |
| metrics_path="${sample_dir}/resource-metrics.env" | |
| if [[ ! -f "$summary_path" ]]; then | |
| echo "Missing ${channel} summary at ${summary_path}." >&2 | |
| exit 1 | |
| fi | |
| if [[ -f "$observed_path" ]]; then | |
| printf '%s\t%s\t%s\n' "$summary_path" "$observed_path" "$metrics_path" >>"$sample_paths" | |
| else | |
| printf '%s\t\t%s\n' "$summary_path" "$metrics_path" >>"$sample_paths" | |
| fi | |
| done < <(find "${import_dir}/artifacts" -mindepth 1 -maxdepth 1 -type d -name 'sample-*' | sort -V) | |
| node scripts/import-live-transport-rtt.mjs "$sample_paths" \ | |
| --channel "$channel" \ | |
| --spec openclaw@main \ | |
| --version "$version" \ | |
| --provider-mode mock-openai \ | |
| --scenario "$scenario" | |
| done | |
| node scripts/validate.mjs | |
| node scripts/channel-rtt-summary.mjs | |
| - name: Commit result | |
| if: github.event_name != 'pull_request' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| git add data/channels/ runs/ | |
| if git diff --cached --quiet --exit-code; then | |
| echo "No channel RTT changes to commit." | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git commit -m "data: record main channel rtt" | |
| git push || { | |
| git pull --rebase origin main | |
| git push | |
| } |