fix(csharp): fix */% wildcard semantics and TABLE_CAT identifier echo diverge between Thrift and SEA (#525) #320
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2025 ADBC Drivers Contributors | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Performance Benchmarks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - '.github/workflows/benchmarks.yml' | |
| - 'csharp/src/**' | |
| - 'csharp/Benchmarks/**' | |
| pull_request: | |
| types: [labeled] # Trigger when a label is added to the PR | |
| paths: | |
| - '.github/workflows/benchmarks.yml' | |
| - 'csharp/src/**' | |
| - 'csharp/Benchmarks/**' | |
| workflow_dispatch: | |
| inputs: | |
| query: | |
| description: 'Custom SQL query (single query mode - backward compatible)' | |
| required: false | |
| type: string | |
| queries: | |
| description: 'Query names from benchmark-queries.json (comma-separated or "all" for multi-query mode)' | |
| required: false | |
| default: '' | |
| type: string | |
| concurrency: | |
| group: benchmarks-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write # Required to push benchmark results to gh-pages branch | |
| pull-requests: write # Required to post comparison comments on PRs | |
| actions: read # Required to download baseline artifacts from main branch runs | |
| jobs: | |
| setup: | |
| name: Determine Benchmark Queries | |
| runs-on: ubuntu-latest | |
| outputs: | |
| queries_json: ${{ steps.set-queries.outputs.queries_json }} | |
| is_multi_query: ${{ steps.set-queries.outputs.is_multi_query }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Determine queries to run | |
| id: set-queries | |
| run: | | |
| QUERIES_INPUT="${{ github.event.inputs.queries }}" | |
| QUERY_INPUT="${{ github.event.inputs.query }}" | |
| IS_MULTI_QUERY="false" | |
| HAS_BENCHMARK_LABEL="${{ contains(github.event.pull_request.labels.*.name, 'benchmark') }}" | |
| if [ -n "$QUERIES_INPUT" ]; then | |
| # Multi-query mode via workflow_dispatch with queries input | |
| echo "Multi-query mode detected - queries input: $QUERIES_INPUT" | |
| if [ "$QUERIES_INPUT" = "all" ]; then | |
| QUERIES=$(jq -c '.' csharp/Benchmarks/benchmark-queries.json) | |
| else | |
| # Filter specific queries by name | |
| IFS=',' read -ra QUERY_NAMES <<< "$QUERIES_INPUT" | |
| FILTER=$(printf '"%s",' "${QUERY_NAMES[@]}") | |
| FILTER="[${FILTER%,}]" | |
| QUERIES=$(jq -c --argjson names "$FILTER" '[.[] | select(.name as $n | $names | index($n))]' csharp/Benchmarks/benchmark-queries.json) | |
| fi | |
| IS_MULTI_QUERY="true" | |
| elif [ "${{ github.event_name }}" = "push" ]; then | |
| # Multi-query mode on push to main | |
| echo "Multi-query mode detected - push to main" | |
| QUERIES=$(jq -c '.' csharp/Benchmarks/benchmark-queries.json) | |
| IS_MULTI_QUERY="true" | |
| elif [ "$HAS_BENCHMARK_LABEL" = "true" ] && [ "${{ github.event_name }}" = "pull_request" ]; then | |
| # Multi-query mode via PR benchmark label | |
| echo "Multi-query mode detected - benchmark label on PR" | |
| QUERIES=$(jq -c '.' csharp/Benchmarks/benchmark-queries.json) | |
| IS_MULTI_QUERY="true" | |
| else | |
| # Single-query mode (manual workflow_dispatch with custom query) | |
| echo "Single-query mode detected" | |
| if [ -n "$QUERY_INPUT" ]; then | |
| QUERY="$QUERY_INPUT" | |
| else | |
| QUERY="select * from main.tpcds_sf1_delta.catalog_sales" | |
| fi | |
| QUERIES=$(jq -nc --arg q "$QUERY" '[{ | |
| "name": "default", | |
| "description": "Custom or default query", | |
| "query": $q, | |
| "category": "custom" | |
| }]') | |
| fi | |
| echo "is_multi_query=$IS_MULTI_QUERY" >> $GITHUB_OUTPUT | |
| echo "queries_json=$QUERIES" >> $GITHUB_OUTPUT | |
| echo "Benchmark queries configuration:" | |
| echo "$QUERIES" | jq '.' | |
| echo "Multi-query mode: $IS_MULTI_QUERY" | |
| echo "Query count: $(echo "$QUERIES" | jq 'length')" | |
| benchmark-net472: | |
| name: Benchmark Suite (.NET Framework 4.7.2) | |
| needs: setup | |
| if: | | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) | |
| runs-on: windows-2022 | |
| timeout-minutes: 180 | |
| environment: azure-prod | |
| env: | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_TEST_CLIENT_ID }} | |
| DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_TEST_CLIENT_SECRET }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 | |
| with: | |
| dotnet-version: '8.0.x' | |
| - name: Build | |
| shell: bash | |
| run: | | |
| ./ci/scripts/csharp_build.sh "${{ github.workspace }}" | |
| - name: Run benchmark suite | |
| shell: pwsh | |
| run: | | |
| $configDir = Join-Path $env:USERPROFILE ".databricks" | |
| New-Item -ItemType Directory -Force -Path $configDir | Out-Null | |
| New-Item -ItemType Directory -Force -Path "benchmark-results" | Out-Null | |
| # Create config file with connection details (query will be read from benchmark-queries.json) | |
| $config = @{ | |
| uri = "https://${{ env.DATABRICKS_SERVER_HOSTNAME }}${{ env.DATABRICKS_HTTP_PATH }}" | |
| auth_type = "oauth" | |
| grant_type = "client_credentials" | |
| client_id = "${{ env.DATABRICKS_CLIENT_ID }}" | |
| client_secret = "${{ env.DATABRICKS_CLIENT_SECRET }}" | |
| type = "databricks" | |
| catalog = "main" | |
| db_schema = "ADBC_Testing" | |
| query = "select 1" | |
| } | ConvertTo-Json | |
| $configPath = Join-Path $configDir "connection.json" | |
| $config | Out-File -FilePath $configPath -Encoding utf8 | |
| $env:DATABRICKS_TEST_CONFIG_FILE = $configPath | |
| $queriesFile = Join-Path "${{ github.workspace }}" "csharp/Benchmarks/benchmark-queries.json" | |
| $env:DATABRICKS_BENCHMARK_QUERIES_FILE = $queriesFile | |
| $queryCount = (Get-Content $queriesFile | ConvertFrom-Json).Count | |
| Write-Host "=== Running benchmark suite with $queryCount queries ===" | |
| Write-Host "Queries will be executed by BenchmarkDotNet in a single session" | |
| # Run benchmark once - BenchmarkDotNet will run all queries via ArgumentsSource | |
| $benchmarkPath = Join-Path "${{ github.workspace }}" "ci/scripts/csharp_benchmark.sh" | |
| bash $benchmarkPath "${{ github.workspace }}" "net472" | |
| if ($LASTEXITCODE -eq 0) { | |
| Write-Host "✅ Benchmark suite completed successfully" | |
| # Save all results | |
| $resultsPath = "csharp/Benchmarks/BenchmarkDotNet.Artifacts/results" | |
| if (Test-Path $resultsPath) { | |
| Copy-Item -Path "$resultsPath/*" -Destination "benchmark-results/" -Recurse -Force | |
| } | |
| } else { | |
| Write-Host "❌ Benchmark suite failed" | |
| exit 1 | |
| } | |
| - name: Upload all benchmark results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: benchmark-results-net472 | |
| path: benchmark-results/ | |
| retention-days: 90 | |
| - name: Generate summary | |
| if: needs.setup.outputs.is_multi_query == 'true' | |
| shell: bash | |
| run: | | |
| echo "# Benchmark Suite Results (.NET Framework 4.7.2)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "All queries executed in a single BenchmarkDotNet session to avoid redundant builds." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "benchmark-results"/*-report-full-compressed.json ]; then | |
| echo "✅ Benchmark suite completed successfully" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Results include all $(jq 'length' csharp/Benchmarks/benchmark-queries.json) queries from benchmark-queries.json" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ Benchmark suite failed" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| benchmark-net8: | |
| name: Benchmark Suite (.NET 8.0) | |
| needs: setup | |
| if: | | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| environment: azure-prod | |
| env: | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_TEST_CLIENT_ID }} | |
| DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_TEST_CLIENT_SECRET }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 | |
| with: | |
| dotnet-version: '8.0.x' | |
| - name: Build | |
| shell: bash | |
| run: | | |
| ./ci/scripts/csharp_build.sh "${{ github.workspace }}" | |
| - name: Run benchmark suite | |
| shell: bash | |
| run: | | |
| mkdir -p ~/.databricks | |
| mkdir -p benchmark-results | |
| # Create config file with connection details (query will be read from benchmark-queries.json) | |
| cat > ~/.databricks/connection.json << EOF | |
| { | |
| "uri": "https://${{ env.DATABRICKS_SERVER_HOSTNAME }}${{ env.DATABRICKS_HTTP_PATH }}", | |
| "auth_type": "oauth", | |
| "grant_type": "client_credentials", | |
| "client_id": "${{ env.DATABRICKS_CLIENT_ID }}", | |
| "client_secret": "${{ env.DATABRICKS_CLIENT_SECRET }}", | |
| "type": "databricks", | |
| "catalog": "main", | |
| "db_schema": "ADBC_Testing", | |
| "query": "select 1" | |
| } | |
| EOF | |
| export DATABRICKS_TEST_CONFIG_FILE="$HOME/.databricks/connection.json" | |
| export DATABRICKS_BENCHMARK_QUERIES_FILE="${{ github.workspace }}/csharp/Benchmarks/benchmark-queries.json" | |
| QUERY_COUNT=$(jq 'length' "$DATABRICKS_BENCHMARK_QUERIES_FILE") | |
| echo "=== Running benchmark suite with $QUERY_COUNT queries ===" | |
| echo "Queries will be executed by BenchmarkDotNet in a single session" | |
| # Run benchmark once - BenchmarkDotNet will run all queries via ArgumentsSource | |
| if ./ci/scripts/csharp_benchmark.sh "${{ github.workspace }}" "net8.0"; then | |
| echo "✅ Benchmark suite completed successfully" | |
| # Save all results | |
| if [ -d "csharp/Benchmarks/BenchmarkDotNet.Artifacts/results" ]; then | |
| cp -r csharp/Benchmarks/BenchmarkDotNet.Artifacts/results/* benchmark-results/ | |
| fi | |
| else | |
| echo "❌ Benchmark suite failed" | |
| exit 1 | |
| fi | |
| - name: Upload all benchmark results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: benchmark-results-net8 | |
| path: benchmark-results/ | |
| retention-days: 90 | |
| - name: Generate summary | |
| if: needs.setup.outputs.is_multi_query == 'true' | |
| run: | | |
| echo "# Benchmark Suite Results (.NET 8.0)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "All queries executed in a single BenchmarkDotNet session to avoid redundant builds." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "benchmark-results"/*-report-full-compressed.json ]; then | |
| echo "✅ Benchmark suite completed successfully" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Results include all $(jq 'length' csharp/Benchmarks/benchmark-queries.json) queries from benchmark-queries.json" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ Benchmark suite failed" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| post-results: | |
| name: Post Benchmark Results to PR | |
| needs: [setup, benchmark-net8, benchmark-net472] | |
| if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Download .NET 8.0 results | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: benchmark-results-net8 | |
| path: results-net8 | |
| - name: Download .NET Framework 4.7.2 results | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: benchmark-results-net472 | |
| path: results-net472 | |
| - name: Download baseline results from main | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Find the most recent successful workflow run on main branch | |
| MAIN_RUN_ID=$(gh run list \ | |
| --repo ${{ github.repository }} \ | |
| --workflow benchmarks.yml \ | |
| --branch main \ | |
| --status success \ | |
| --limit 1 \ | |
| --json databaseId \ | |
| --jq '.[0].databaseId') | |
| if [ -z "$MAIN_RUN_ID" ]; then | |
| echo "No baseline run found on main branch" | |
| exit 0 | |
| fi | |
| echo "Found baseline run: $MAIN_RUN_ID" | |
| mkdir -p baseline-net8 baseline-net472 | |
| # Download baseline artifacts | |
| gh run download $MAIN_RUN_ID \ | |
| --repo ${{ github.repository }} \ | |
| --name benchmark-results-net8 \ | |
| --dir baseline-net8 || echo "No baseline for .NET 8.0" | |
| gh run download $MAIN_RUN_ID \ | |
| --repo ${{ github.repository }} \ | |
| --name benchmark-results-net472 \ | |
| --dir baseline-net472 || echo "No baseline for .NET Framework 4.7.2" | |
| - name: Compare and post results | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Find the result files | |
| NET8_RESULTS=$(find results-net8 -name "*-report-full-compressed.json" | head -1) | |
| NET472_RESULTS=$(find results-net472 -name "*-report-full-compressed.json" | head -1) | |
| NET8_BASELINE=$(find baseline-net8 -name "*-report-full-compressed.json" 2>/dev/null | head -1) | |
| NET472_BASELINE=$(find baseline-net472 -name "*-report-full-compressed.json" 2>/dev/null | head -1) | |
| # Debug: Check if baseline files exist | |
| echo "=== Baseline File Check ===" | |
| echo "NET8_BASELINE=$NET8_BASELINE" | |
| echo "NET472_BASELINE=$NET472_BASELINE" | |
| if [ -n "$NET8_BASELINE" ]; then | |
| echo "✅ NET8 baseline file found" | |
| echo "First benchmark entry:" | |
| jq '.Benchmarks[0] | {Parameters, Mean: .Statistics.Mean, Memory: .Memory.BytesAllocatedPerOperation, Gen2: .Memory.Gen2Collections}' "$NET8_BASELINE" 2>&1 || echo "Failed to parse baseline" | |
| else | |
| echo "❌ NET8 baseline file NOT found" | |
| fi | |
| # Create comment body | |
| cat > comment.md << 'EOF' | |
| ## 📊 CloudFetch Benchmark Results | |
| EOF | |
| # Function to format comparison | |
| format_results() { | |
| local CURRENT=$1 | |
| local BASELINE=$2 | |
| local PLATFORM=$3 | |
| if [ ! -f "$CURRENT" ]; then | |
| echo "⚠️ Results not found" | |
| return | |
| fi | |
| echo "| Query | Mean (ms) | Peak Memory (MB) | Allocated Memory (MB) | Gen2 | Rows | Cols |" | |
| echo "|-------|-----------|------------------|----------------------|------|------|------|" | |
| # Parse current results - handle both string and object Parameters | |
| jq -r ' | |
| .Benchmarks[] | | |
| ( | |
| if (.Parameters | type) == "string" then | |
| (.Parameters | split("&") | map(select(startswith("benchmarkQuery=")) | split("=")[1]) | .[0] // "") as $param_query | | |
| if ($param_query | contains("(...)")) then | |
| (.FullName | match("benchmarkQuery: ([^,)]+)") | .captures[0].string) | |
| else | |
| $param_query | |
| end | |
| elif (.Parameters | type) == "object" then | |
| if (.Parameters.benchmarkQuery | type) == "object" then | |
| .Parameters.benchmarkQuery.name | |
| else | |
| .Parameters.benchmarkQuery | |
| end | |
| else | |
| "" | |
| end | |
| ) as $query | | |
| "\($query)|\(.Statistics.Mean / 1000000)|\(.Memory.TotalBytesAllocated // .Memory.BytesAllocatedPerOperation // 0)|\(.Memory.BytesAllocatedPerOperation // 0)|\(.Memory.Gen2Collections // 0)" | |
| ' "$CURRENT" | while IFS='|' read -r query mean peak_memory alloc_memory gen2; do | |
| if [ -z "$query" ] || [ "$query" = "null" ]; then | |
| continue | |
| fi | |
| # Get rows and columns from the benchmark-queries.json if available | |
| rows="-" | |
| cols="-" | |
| if [ -f "csharp/Benchmarks/benchmark-queries.json" ]; then | |
| rows=$(jq -r --arg q "$query" '.[] | select(.name == $q) | .expected_rows // "-"' csharp/Benchmarks/benchmark-queries.json 2>/dev/null || echo "-") | |
| cols=$(jq -r --arg q "$query" '.[] | select(.name == $q) | .columns // "-"' csharp/Benchmarks/benchmark-queries.json 2>/dev/null || echo "-") | |
| fi | |
| # Format current values | |
| mean_formatted=$(printf "%.2f" "$mean") | |
| peak_memory_mb=$(echo "scale=2; $peak_memory / 1048576" | bc) | |
| peak_memory_formatted=$(printf "%.2f" "$peak_memory_mb") | |
| alloc_memory_mb=$(echo "scale=2; $alloc_memory / 1048576" | bc) | |
| alloc_memory_formatted=$(printf "%.2f" "$alloc_memory_mb") | |
| # Try to find baseline | |
| if [ -f "$BASELINE" ]; then | |
| # Extract all baseline metrics for this query | |
| baseline_data=$(jq -r --arg q "$query" '.Benchmarks[] | | |
| select( | |
| if (.Parameters | type) == "string" then | |
| # Handle truncated Parameters by checking FullName | |
| if (.Parameters | contains("(...)")) then | |
| (.FullName | contains("benchmarkQuery: " + $q)) | |
| else | |
| (.Parameters | contains("benchmarkQuery=" + $q)) | |
| end | |
| elif (.Parameters | type) == "object" then | |
| ((.Parameters.benchmarkQuery.name // .Parameters.benchmarkQuery) == $q) | |
| else | |
| false | |
| end | |
| ) | | |
| "\(.Statistics.Mean / 1000000)|\(.Memory.TotalBytesAllocated // .Memory.BytesAllocatedPerOperation // 0)|\(.Memory.BytesAllocatedPerOperation // 0)|\(.Memory.Gen2Collections // 0)"' "$BASELINE" 2>/dev/null | head -1) | |
| if [ -n "$baseline_data" ] && [ "$baseline_data" != "null" ]; then | |
| baseline_mean=$(echo "$baseline_data" | cut -d'|' -f1) | |
| baseline_peak_memory=$(echo "$baseline_data" | cut -d'|' -f2) | |
| baseline_alloc_memory=$(echo "$baseline_data" | cut -d'|' -f3) | |
| baseline_gen2=$(echo "$baseline_data" | cut -d'|' -f4) | |
| # Format baseline values | |
| baseline_mean_formatted=$(printf "%.2f" "$baseline_mean") | |
| baseline_peak_memory_mb=$(echo "scale=2; $baseline_peak_memory / 1048576" | bc) | |
| baseline_peak_memory_formatted=$(printf "%.2f" "$baseline_peak_memory_mb") | |
| baseline_alloc_memory_mb=$(echo "scale=2; $baseline_alloc_memory / 1048576" | bc) | |
| baseline_alloc_memory_formatted=$(printf "%.2f" "$baseline_alloc_memory_mb") | |
| # Calculate percentage differences with proper decimal handling | |
| time_diff=$(awk -v curr="$mean" -v base="$baseline_mean" 'BEGIN {printf "%.1f", ((curr - base) / base) * 100}') | |
| peak_memory_diff=$(awk -v curr="$peak_memory" -v base="$baseline_peak_memory" 'BEGIN {if (base != 0) printf "%.1f", ((curr - base) / base) * 100; else print "0"}') | |
| alloc_memory_diff=$(awk -v curr="$alloc_memory" -v base="$baseline_alloc_memory" 'BEGIN {if (base != 0) printf "%.1f", ((curr - base) / base) * 100; else print "0"}') | |
| # Format time diff | |
| if (( $(awk -v d="$time_diff" 'BEGIN {print (d > 0.5)}') )); then | |
| time_diff_formatted="🔴 +${time_diff}%" | |
| elif (( $(awk -v d="$time_diff" 'BEGIN {print (d < -0.5)}') )); then | |
| time_diff_formatted="🟢 ${time_diff}%" | |
| else | |
| time_diff_formatted="⚪ ~0%" | |
| fi | |
| # Format peak memory diff | |
| if (( $(awk -v d="$peak_memory_diff" 'BEGIN {print (d > 0.5)}') )); then | |
| peak_memory_diff_formatted="🔴 +${peak_memory_diff}%" | |
| elif (( $(awk -v d="$peak_memory_diff" 'BEGIN {print (d < -0.5)}') )); then | |
| peak_memory_diff_formatted="🟢 ${peak_memory_diff}%" | |
| else | |
| peak_memory_diff_formatted="⚪ ~0%" | |
| fi | |
| # Format allocated memory diff | |
| if (( $(awk -v d="$alloc_memory_diff" 'BEGIN {print (d > 0.5)}') )); then | |
| alloc_memory_diff_formatted="🔴 +${alloc_memory_diff}%" | |
| elif (( $(awk -v d="$alloc_memory_diff" 'BEGIN {print (d < -0.5)}') )); then | |
| alloc_memory_diff_formatted="🟢 ${alloc_memory_diff}%" | |
| else | |
| alloc_memory_diff_formatted="⚪ ~0%" | |
| fi | |
| # Format with baseline in parentheses | |
| mean_with_baseline="${mean_formatted} (${baseline_mean_formatted}) ${time_diff_formatted}" | |
| peak_memory_with_baseline="${peak_memory_formatted} (${baseline_peak_memory_formatted}) ${peak_memory_diff_formatted}" | |
| alloc_memory_with_baseline="${alloc_memory_formatted} (${baseline_alloc_memory_formatted}) ${alloc_memory_diff_formatted}" | |
| echo "| $query | $mean_with_baseline | $peak_memory_with_baseline | $alloc_memory_with_baseline | $gen2 | $rows | $cols |" | |
| else | |
| echo "| $query | $mean_formatted | $peak_memory_formatted | $alloc_memory_formatted | $gen2 | $rows | $cols |" | |
| fi | |
| else | |
| echo "| $query | $mean_formatted | $peak_memory_formatted | $alloc_memory_formatted | $gen2 | $rows | $cols |" | |
| fi | |
| done | |
| } | |
| # .NET 8.0 results | |
| echo "<details open>" >> comment.md | |
| echo "<summary><b>.NET 8.0</b></summary>" >> comment.md | |
| echo "" >> comment.md | |
| format_results "$NET8_RESULTS" "$NET8_BASELINE" ".NET 8.0" >> comment.md | |
| echo "" >> comment.md | |
| echo "</details>" >> comment.md | |
| echo "" >> comment.md | |
| # .NET Framework 4.7.2 results | |
| echo "<details>" >> comment.md | |
| echo "<summary><b>.NET Framework 4.7.2</b></summary>" >> comment.md | |
| echo "" >> comment.md | |
| format_results "$NET472_RESULTS" "$NET472_BASELINE" ".NET Framework 4.7.2" >> comment.md | |
| echo "" >> comment.md | |
| echo "</details>" >> comment.md | |
| # Footer | |
| cat >> comment.md << 'EOF' | |
| --- | |
| *🟢 Improvement | 🔴 Regression | ⚪ No change* | |
| **Format:** `current_value (baseline) diff%` | |
| - **Baseline**: Latest successful run on main branch | |
| **Metrics:** | |
| - **Mean**: Execution time in milliseconds | |
| - **Peak Memory**: Total bytes allocated during operation in MB | |
| - **Allocated Memory**: Bytes allocated per operation in MB | |
| - **Gen2**: Number of Gen2 garbage collections | |
| EOF | |
| # Post comment to PR | |
| gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --body-file comment.md | |
| publish-to-gh-pages: | |
| name: Publish Benchmark Results to GitHub Pages | |
| needs: [benchmark-net8, benchmark-net472] | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Download .NET 8.0 results | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: benchmark-results-net8 | |
| path: results-net8 | |
| - name: Download .NET Framework 4.7.2 results | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: benchmark-results-net472 | |
| path: results-net472 | |
| - name: Process benchmark results | |
| run: | | |
| # Process individual queries | |
| python3 .github/scripts/process-benchmarks.py \ | |
| results-net8/Apache.Arrow.Adbc.Benchmarks.CloudFetchRealE2EBenchmark-report-full-compressed.json \ | |
| results-net8/Apache.Arrow.Adbc.Benchmarks.CloudFetchRealE2EBenchmark-report.csv \ | |
| processed-net8 | |
| python3 .github/scripts/process-benchmarks.py \ | |
| results-net472/Apache.Arrow.Adbc.Benchmarks.CloudFetchRealE2EBenchmark-report-full-compressed.json \ | |
| results-net472/Apache.Arrow.Adbc.Benchmarks.CloudFetchRealE2EBenchmark-report.csv \ | |
| processed-net472 | |
| # Organize by metric type | |
| python3 .github/scripts/organize-by-metric.py processed-net8 by-metric-net8 | |
| python3 .github/scripts/organize-by-metric.py processed-net472 by-metric-net472 | |
| - name: Publish Mean Execution Time (.NET 8.0) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Mean Execution Time (.NET 8.0)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net8/mean-execution-time.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/mean-time/net8 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Publish Peak Memory (.NET 8.0) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Peak Memory (.NET 8.0)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net8/peak-memory.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/peak-memory/net8 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Publish Allocated Memory (.NET 8.0) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Allocated Memory (.NET 8.0)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net8/allocated-memory.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/allocated-memory/net8 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Publish Gen2 Collections (.NET 8.0) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Gen2 Collections (.NET 8.0)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net8/gen2-collections.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/gen2-collections/net8 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Publish Mean Execution Time (.NET Framework 4.7.2) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Mean Execution Time (.NET Framework 4.7.2)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net472/mean-execution-time.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/mean-time/net472 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Publish Peak Memory (.NET Framework 4.7.2) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Peak Memory (.NET Framework 4.7.2)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net472/peak-memory.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/peak-memory/net472 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Publish Allocated Memory (.NET Framework 4.7.2) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Allocated Memory (.NET Framework 4.7.2)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net472/allocated-memory.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/allocated-memory/net472 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Publish Gen2 Collections (.NET Framework 4.7.2) | |
| uses: benchmark-action/github-action-benchmark@a60cea5bc7b49e15c1f58f411161f99e0df48372 # v1.22.0 | |
| with: | |
| name: 'Gen2 Collections (.NET Framework 4.7.2)' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: by-metric-net472/gen2-collections.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: bench/gen2-collections/net472 | |
| alert-threshold: '150%' | |
| fail-on-alert: false | |
| - name: Create and publish index page | |
| run: | | |
| # Generate index page | |
| bash .github/scripts/create-index-page.sh | |
| # Checkout gh-pages and add index | |
| git fetch origin gh-pages:gh-pages | |
| git checkout gh-pages | |
| git add bench/index.html | |
| git commit -m "Update benchmark index page" || echo "No changes to index" | |
| git push origin gh-pages | |
| git checkout - |