tlsn/.github/workflows/regression.yml at 77a8a25f4770277d0744495d55742de63a7dfef9 · tlsnotary/tlsn · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
name: Regression Detection

on:
  workflow_dispatch:
    inputs:
      commit1:
        description: 'First commit/branch/tag (baseline)'
        required: true
        default: 'main'
        type: string
      commit2:
        description: 'Second commit/branch/tag (comparison)'
        required: true
        default: 'main'
        type: string
      mode:
        # quick: representative bench (bench.toml) at 3 samples
        # precise: all benches (bench.toml + sweeps) at 5 samples
        description: 'Benchmark mode'
        required: true
        default: 'quick'
        type: choice
        options:
          - quick
          - precise

jobs:
  compare:
    runs-on: [self-hosted-ghr, size-l-x64]
    timeout-minutes: 300

    steps:
      # Setup phase - install all dependencies once
      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y wget gnupg clang build-essential iproute2 iptables
          # Install Google Chrome (deb package, not snap)
          wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
          sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
          sudo apt-get update
          sudo apt-get install -y google-chrome-stable

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: stable

      - name: Install Rust nightly
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: nightly
          targets: wasm32-unknown-unknown
          components: rust-src

      # Checkout default branch to make bench-commit.sh available
      - name: Checkout scripts
        uses: actions/checkout@v4
        with:
          path: _scripts

      - name: Create results directory
        run: mkdir -p /tmp/benchmark-results

      # ===================================================================
      # COMMIT 1: Build and run all benchmarks
      # ===================================================================

      - name: Checkout tlsn at ${{ github.event.inputs.commit1 }}
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.inputs.commit1 }}
          path: tlsn-commit1

      - name: Benchmark ${{ github.event.inputs.commit1 }}
        working-directory: tlsn-commit1/crates/harness
        run: ${{ github.workspace }}/_scripts/.github/workflows/bench-commit.sh commit1 ${{ github.event.inputs.mode }}

      - name: Cleanup commit1

        run: |
          cd tlsn-commit1/crates/harness && sudo ./bin/runner clean || true

      # ===================================================================
      # COMMIT 2: Build and run all benchmarks
      # ===================================================================

      - name: Checkout tlsn at ${{ github.event.inputs.commit2 }}
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.inputs.commit2 }}
          path: tlsn-commit2

      - name: Benchmark ${{ github.event.inputs.commit2 }}
        working-directory: tlsn-commit2/crates/harness
        run: ${{ github.workspace }}/_scripts/.github/workflows/bench-commit.sh commit2 ${{ github.event.inputs.mode }}

      - name: Cleanup commit2

        run: |
          cd tlsn-commit2/crates/harness && sudo ./bin/runner clean || true

      # ===================================================================
      # UPLOAD RESULTS & ANALYSIS
      # ===================================================================

      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: /tmp/benchmark-results/

      # Analyze results
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install pandas

      - name: Analyze benchmarks
        run: |
          CONFIGS="bench"
          if [ "${{ github.event.inputs.mode }}" = "precise" ]; then
            CONFIGS="bench bench_bandwidth_sweep bench_latency_sweep bench_download_sweep"
          fi
          FAILED=0
          for target in native browser; do
            echo "## ${target^} Benchmark Analysis" >> $GITHUB_STEP_SUMMARY
            for config in $CONFIGS; do
              echo "### ${config}" >> $GITHUB_STEP_SUMMARY
              python3 ${{ github.workspace }}/_scripts/.github/workflows/regression-analyze.py \
                /tmp/benchmark-results/commit1-${target}-${config}.csv \
                /tmp/benchmark-results/commit2-${target}-${config}.csv \
                --commit1 "${{ github.event.inputs.commit1 }} (${target}/${config})" \
                --commit2 "${{ github.event.inputs.commit2 }} (${target}/${config})" || FAILED=1
            done
            echo "---" >> $GITHUB_STEP_SUMMARY
          done
          exit $FAILED