Skip to content
Benchmarks

Standardize benchmark methodology and enforce single-threaded execution #224

Sign in to view logs

Standardize benchmark methodology and enforce single-threaded execution

Standardize benchmark methodology and enforce single-threaded execution #224

Workflow file for this run

.github/workflows/benchmarks.yml at 8dbc7b7
name: Benchmarks
on:
pull_request:
push:
branches:
- master
tags: '*'
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || github.run_number }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
permissions:
actions: write
contents: write
pull-requests: read
statuses: write
# Force single-threaded execution for all benchmarks
env:
OMP_NUM_THREADS: 1
OPENBLAS_NUM_THREADS: 1
MKL_NUM_THREADS: 1
GOMAXPROCS: 1
JULIA_NUM_THREADS: 1
NUMBA_NUM_THREADS: 1
# ---------------------------------------------------------------------------
# Per-language benchmark jobs
# ---------------------------------------------------------------------------
jobs:
bench-c:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up dSFMT"
run: |
mkdir -p dSFMT
cd dSFMT
wget -q https://github.com/MersenneTwister-Lab/dSFMT/archive/refs/tags/v2.2.4.tar.gz
echo "39682961ecfba621a98dbb6610b6ae2b7d6add450d4f08d8d4edd0e10abd8174 v2.2.4.tar.gz" | sha256sum --check --status
tar -xzf v2.2.4.tar.gz
mv dSFMT-*/* ./
cd ..
- name: "Set up OpenBLAS"
run: sudo apt-get install -y libopenblas-dev
- name: "Run benchmark"
run: make benchmarks/c.csv
- name: "Capture version"
run: bash bin/versions.sh c > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-c
path: |
benchmarks/c.csv
version.csv
bench-fortran:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up OpenBLAS"
run: sudo apt-get install -y libopenblas-dev
- name: "Run benchmark"
run: make benchmarks/fortran.csv
- name: "Capture version"
run: bash bin/versions.sh fortran > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-fortran
path: |
benchmarks/fortran.csv
version.csv
bench-go:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: '1.24'
- name: "Set up OpenBLAS"
run: sudo apt-get install -y libopenblas-dev
- name: "Download Go modules"
run: cd go && go mod download
- name: "Run benchmark"
run: make benchmarks/go.csv
- name: "Capture version"
run: bash bin/versions.sh go > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-go
path: |
benchmarks/go.csv
version.csv
bench-java:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Java"
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '24'
cache: 'maven'
- name: "Download dependencies and compile"
run: cd java && mvn -q dependency:resolve compile
- name: "Run benchmark"
run: make benchmarks/java.csv
- name: "Capture version"
run: bash bin/versions.sh java > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-java
path: |
benchmarks/java.csv
version.csv
bench-javascript:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up JavaScript"
uses: actions/setup-node@v4
with:
node-version: '24'
- name: "Run benchmark"
run: make benchmarks/javascript.csv
- name: "Capture version"
run: bash bin/versions.sh javascript > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-javascript
path: |
benchmarks/javascript.csv
version.csv
bench-julia:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Julia"
uses: julia-actions/setup-julia@v2
with:
version: 'v1'
- name: "Run benchmark"
run: make benchmarks/julia.csv
- name: "Capture version"
run: bash bin/versions.sh julia > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-julia
path: |
benchmarks/julia.csv
version.csv
bench-lua:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up LuaJIT"
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq luajit luarocks libopenblas-dev
sudo luarocks install sci
- name: "Run benchmark"
run: make benchmarks/lua.csv
- name: "Capture version"
run: bash bin/versions.sh lua > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-lua
path: |
benchmarks/lua.csv
version.csv
bench-octave:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Octave"
run: sudo apt-get update && sudo apt-get install -y octave libopenblas-dev
- name: "Run benchmark"
run: make benchmarks/octave.csv
- name: "Capture version"
run: bash bin/versions.sh octave > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-octave
path: |
benchmarks/octave.csv
version.csv
bench-numba:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Python"
uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: "Set up Numba"
run: pip install numpy==2.3 numba
- name: "Run benchmark"
run: make benchmarks/numba.csv
- name: "Capture version"
run: bash bin/versions.sh numba > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-numba
path: |
benchmarks/numba.csv
version.csv
bench-python:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Python"
uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: "Set up NumPy"
run: pip install numpy==2.3
- name: "Run benchmark"
run: make benchmarks/python.csv
- name: "Capture version"
run: bash bin/versions.sh python > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-python
path: |
benchmarks/python.csv
version.csv
bench-r:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up R"
uses: r-lib/actions/setup-r@v2
with:
r-version: '4.5.1'
- name: "Run benchmark"
run: make benchmarks/r.csv
- name: "Capture version"
run: bash bin/versions.sh r > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-r
path: |
benchmarks/r.csv
version.csv
bench-rust:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up OpenBLAS"
run: sudo apt-get install -y libopenblas-dev
- name: "Set up Rust"
uses: dtolnay/rust-toolchain@stable
- name: "Download dependencies and build"
run: cd rust && cargo build --release
- name: "Run benchmark"
run: make benchmarks/rust.csv
- name: "Capture version"
run: bash bin/versions.sh rust > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-rust
path: |
benchmarks/rust.csv
version.csv
bench-scala:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Java"
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
- name: "Set up sbt"
uses: sbt/setup-sbt@v1
- name: "Set up OpenBLAS"
run: sudo apt-get install -y libopenblas-dev
- name: "Download dependencies and compile"
run: cd scala && sbt --batch compile
- name: "Run benchmark"
run: make benchmarks/scala.csv
- name: "Capture version"
run: bash bin/versions.sh scala > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-scala
path: |
benchmarks/scala.csv
version.csv
bench-swift:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Swift"
uses: swift-actions/setup-swift@v2
with:
swift-version: '6.1'
- name: "Set up OpenBLAS"
run: sudo apt-get install -y libopenblas-dev
- name: "Build"
run: cd swift && swift build -c release -Xlinker -lopenblas
- name: "Run benchmark"
run: make benchmarks/swift.csv
- name: "Capture version"
run: bash bin/versions.sh swift > version.csv
- name: "Upload results"
uses: actions/upload-artifact@v4
with:
name: bench-swift
path: |
benchmarks/swift.csv
version.csv
# ---------------------------------------------------------------------------
# Aggregation + deploy
# ---------------------------------------------------------------------------
report:
runs-on: ubuntu-latest
needs:
- bench-c
- bench-fortran
- bench-go
- bench-java
- bench-javascript
- bench-julia
- bench-lua
- bench-numba
- bench-octave
- bench-python
- bench-r
- bench-rust
- bench-scala
- bench-swift
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: "Set up Julia"
uses: julia-actions/setup-julia@v2
with:
version: 'v1'
- name: "Download all benchmark artifacts"
uses: actions/download-artifact@v4
with:
path: artifacts
- name: "Collect benchmark CSVs"
run: |
mkdir -p benchmarks
for dir in artifacts/bench-*; do
lang=$(basename "$dir" | sed 's/^bench-//')
cp "$dir/benchmarks/${lang}.csv" benchmarks/
done
- name: "Merge benchmarks"
run: julia bin/collect.jl benchmarks/*.csv > gh_action_benchmarks.csv
- name: "Collect versions"
run: |
for dir in artifacts/bench-*; do
cat "$dir/version.csv"
done > gh_action_versions.csv
- name: "Generate HTML table"
run: julia bin/table.jl gh_action_versions.csv gh_action_benchmarks.csv > gh_action_benchmarks.html
- name: "Print versions"
run: cat gh_action_versions.csv
- name: "Print benchmark data"
run: cat gh_action_benchmarks.csv
- uses: julia-actions/cache@v2
- name: "Build and deploy site"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
julia --color=yes --project=docs -e 'using Pkg; Pkg.instantiate()'
julia --color=yes --project=docs --compiled-modules=existing docs/make.jl