test-consistency.yaml

name: Test cross-implementation consistency

on:
  push:
    branches: [ main, master ]
  pull_request:
    branches: [ main, master ]
  workflow_call:

env:
  CARGO_TERM_COLOR: always

permissions:
  contents: read
  pull-requests: write

jobs:
  consistency:
    name: Verify all 3 implementations produce identical basic-example output
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        submodules: recursive

    - name: Install Rust
      uses: dtolnay/rust-toolchain@stable

    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y build-essential pkg-config hyperfine

    - name: Build C library
      run: |
        cd crates/libspot/libspot
        make clean
        make

    - name: Build all 3 release binaries into /tmp
      run: |
        set -euo pipefail

        # Raw C: compile the wrapper crate's basic.c against the static lib.
        # The submodule ships a different basic.c (smaller K, different format);
        # the wrapper crate's version is the canonical source kept in sync with
        # the Rust examples.
        cd crates/libspot
        ARCHIVE=$(ls libspot/dist/libspot.a.* | head -n1)
        echo "Using archive: $ARCHIVE"
        cc -O2 -std=c99 -o /tmp/basic_c examples/basic.c -Ilibspot/include/ "$ARCHIVE" -lm

        # Rust FFI (release)
        cargo build --release --example basic
        cp target/release/examples/basic /tmp/basic_ffi

        # Pure Rust (release)
        cd ../libspot-rs
        cargo build --release --example basic
        cp target/release/examples/basic /tmp/basic_pure

    - name: Run each implementation once and capture output
      run: |
        /tmp/basic_c    | tee /tmp/out_c.txt
        /tmp/basic_ffi  | tee /tmp/out_ffi.txt
        /tmp/basic_pure | tee /tmp/out_pure.txt

    - name: Compare ANOMALY/EXCESS/NORMAL/Z/T across the 3 implementations
      run: |
        set -euo pipefail

        extract() {
          # Extracts the two canonical lines into a normalized "key=value" form.
          local file="$1"
          local counts z_t
          counts=$(grep -E '^ANOMALY=[0-9]+ EXCESS=[0-9]+ NORMAL=[0-9]+$' "$file" | tail -n1)
          z_t=$(grep -E '^Z=[-0-9.]+ T=[-0-9.]+$' "$file" | tail -n1)
          if [[ -z "$counts" || -z "$z_t" ]]; then
            echo "ERROR: could not find canonical output lines in $file" >&2
            cat "$file" >&2
            exit 1
          fi
          printf '%s\n%s\n' "$counts" "$z_t"
        }

        C_OUT=$(extract /tmp/out_c.txt)
        FFI_OUT=$(extract /tmp/out_ffi.txt)
        PURE_OUT=$(extract /tmp/out_pure.txt)

        echo "::group::Raw C"
        echo "$C_OUT"
        echo "::endgroup::"
        echo "::group::Rust FFI (libspot)"
        echo "$FFI_OUT"
        echo "::endgroup::"
        echo "::group::Pure Rust (libspot-rs)"
        echo "$PURE_OUT"
        echo "::endgroup::"

        if [[ "$C_OUT" != "$FFI_OUT" ]]; then
          echo "::error::Raw C and Rust FFI outputs differ"
          diff <(echo "$C_OUT") <(echo "$FFI_OUT") || true
          exit 1
        fi

        if [[ "$C_OUT" != "$PURE_OUT" ]]; then
          echo "::error::Raw C and pure-Rust outputs differ"
          diff <(echo "$C_OUT") <(echo "$PURE_OUT") || true
          exit 1
        fi

        echo "All three implementations produced identical output."

        # Build a markdown table for the workflow summary / PR comment.
        c_counts=$(grep -E '^ANOMALY=' /tmp/out_c.txt    | tail -n1)
        c_zt=$(grep    -E '^Z='       /tmp/out_c.txt    | tail -n1)
        f_counts=$(grep -E '^ANOMALY=' /tmp/out_ffi.txt  | tail -n1)
        f_zt=$(grep    -E '^Z='       /tmp/out_ffi.txt  | tail -n1)
        p_counts=$(grep -E '^ANOMALY=' /tmp/out_pure.txt | tail -n1)
        p_zt=$(grep    -E '^Z='       /tmp/out_pure.txt | tail -n1)

        # ANOMALY=N EXCESS=N NORMAL=N  -> three values
        read -r c_a c_e c_n <<<"$(echo "$c_counts" | sed -E 's/[A-Z]+=//g')"
        read -r f_a f_e f_n <<<"$(echo "$f_counts" | sed -E 's/[A-Z]+=//g')"
        read -r p_a p_e p_n <<<"$(echo "$p_counts" | sed -E 's/[A-Z]+=//g')"
        read -r c_z c_t     <<<"$(echo "$c_zt"     | sed -E 's/[A-Z]+=//g')"
        read -r f_z f_t     <<<"$(echo "$f_zt"     | sed -E 's/[A-Z]+=//g')"
        read -r p_z p_t     <<<"$(echo "$p_zt"     | sed -E 's/[A-Z]+=//g')"

        {
          echo '| Metric | C | Rust FFI | Pure Rust |'
          echo '|---|---:|---:|---:|'
          printf '| Anomalies | %s | %s | %s |\n' "$c_a" "$f_a" "$p_a"
          printf '| Excess    | %s | %s | %s |\n' "$c_e" "$f_e" "$p_e"
          printf '| Normal    | %s | %s | %s |\n' "$c_n" "$f_n" "$p_n"
          printf '| Z         | %s | %s | %s |\n' "$c_z" "$f_z" "$p_z"
          printf '| T         | %s | %s | %s |\n' "$c_t" "$f_t" "$p_t"
        } > /tmp/consistency.md

    - name: Benchmark all 3 implementations (release builds, 5 runs)
      run: |
        set -euo pipefail

        hyperfine \
          --warmup 1 \
          --runs 5 \
          --command-name 'raw C' /tmp/basic_c \
          --command-name 'Rust FFI (libspot)' /tmp/basic_ffi \
          --command-name 'Pure Rust (libspot-rs)' /tmp/basic_pure \
          --export-markdown /tmp/bench.md

        # Surface the benchmark table on the workflow run summary page.
        {
          echo '## Consistency (basic example, single run)'
          echo
          cat /tmp/consistency.md
          echo
          echo '## Benchmark (release, hyperfine, warmup=1, runs=5)'
          echo
          cat /tmp/bench.md
        } >> "$GITHUB_STEP_SUMMARY"

    - name: Post results as PR comment
      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
      uses: actions/github-script@v7
      with:
        script: |
          const fs = require('fs');
          const marker = '<!-- hyperfine-bench-comment -->';
          const sha = context.sha.substring(0, 7);
          const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
          const body = [
            marker,
            '### ▸ Cross-implementation results',
            '',
            `_commit \`${sha}\` · \`${process.env.RUNNER_OS}\` · release builds_`,
            '',
            '#### Consistency (basic example)',
            '',
            fs.readFileSync('/tmp/consistency.md', 'utf8').trim(),
            '',
            '#### Benchmark (`hyperfine --warmup 1 --runs 5`)',
            '',
            fs.readFileSync('/tmp/bench.md', 'utf8').trim(),
            '',
            `<sub>updated by [${context.workflow}](${runUrl})</sub>`,
          ].join('\n');

          const { data: comments } = await github.rest.issues.listComments({
            owner: context.repo.owner,
            repo: context.repo.repo,
            issue_number: context.issue.number,
            per_page: 100,
          });
          const existing = comments.find(c => c.body && c.body.startsWith(marker));
          if (existing) {
            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: existing.id,
              body,
            });
          } else {
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body,
            });
          }