feat(task): add OpenTelemetry trace and log export for task executions #22435
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: hyperfine | |
| on: | |
| pull_request: | |
| branches: ["main"] | |
| # paths: | |
| # - ".github/workflows/hyperfine.yml" | |
| # - "Cargo.toml" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| GITHUB_TOKEN: ${{ secrets.MISE_GH_TOKEN || secrets.GITHUB_TOKEN }} | |
| MISE_EXPERIMENTAL: 1 | |
| permissions: | |
| issues: write | |
| pull-requests: write | |
| jobs: | |
| benchmark: | |
| runs-on: namespace-profile-endev-linux-amd64-large;overrides.cache-tag=mise-pr-rust-linux | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 | |
| with: | |
| fetch-depth: 0 | |
| - uses: namespacelabs/nscloud-cache-action@58bf6e08898e88803c098e2b522668541cd3b2e3 # v1 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| ~/.cargo/.global-cache | |
| ~/.cache/sccache | |
| - uses: taiki-e/install-action@51cd0b8c0499559d9a4d75c0f5c67bec3a894ec8 # v2 | |
| with: { tool: sccache } | |
| - name: Configure sccache | |
| run: | | |
| export RUSTC_WRAPPER=sccache | |
| export SCCACHE_DIR="$HOME/.cache/sccache" | |
| export SCCACHE_CACHE_SIZE=20G | |
| { | |
| echo "RUSTC_WRAPPER=$RUSTC_WRAPPER" | |
| echo "SCCACHE_DIR=$SCCACHE_DIR" | |
| echo "SCCACHE_CACHE_SIZE=$SCCACHE_CACHE_SIZE" | |
| } >> "$GITHUB_ENV" | |
| sccache --zero-stats | |
| - run: curl https://mise.run | MISE_INSTALL_PATH="$HOME/bin/mise-release" sh | |
| - run: echo "$HOME/bin" >> "$GITHUB_PATH" | |
| - id: versions | |
| run: | | |
| #echo "main=$(git rev-parse --short origin/main)" >> "$GITHUB_OUTPUT" | |
| echo "release=$(mise-release v | awk '{print $1}')" >> "$GITHUB_OUTPUT" | |
| #- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 | |
| # with: | |
| # path: ~/bin/mise-${{ steps.versions.outputs.main }} | |
| # key: mise-hyperfine-main-${{ steps.versions.outputs.main }}-${{ runner.os }}-${{ runner.arch }} | |
| #- name: build main | |
| # run: | | |
| # if [ ! -f "$HOME/bin/mise-${{ steps.versions.outputs.main }}" ]; then | |
| # git checkout main | |
| # cargo build --profile serious && mv target/serious/mise "$HOME/bin/mise-${{ steps.versions.outputs.main }}" | |
| # git checkout - | |
| # fi | |
| - run: mv "$HOME/bin/mise-release" "$HOME/bin/mise-${{ steps.versions.outputs.release }}" | |
| #- run: cp "$HOME/bin/mise-${{ steps.versions.outputs.main }}" "$HOME/bin/mise-main" | |
| - run: cargo build --profile serious && mv target/serious/mise "$HOME/bin" | |
| - uses: ./.github/actions/mise-tools | |
| - run: | | |
| set -exo pipefail | |
| failed=false | |
| outlier_warning="Statistical outliers were detected" | |
| threshold_scaled=1100 | |
| scale_decimal() { | |
| local number="$1" | |
| local whole fraction | |
| whole=${number%%.*} | |
| fraction=${number#*.} | |
| if [ "$fraction" = "$number" ]; then | |
| fraction=0 | |
| fi | |
| fraction="${fraction}000" | |
| echo $((10#$whole * 1000 + 10#${fraction:0:3})) | |
| } | |
| CMDS=( | |
| "x -- echo" | |
| "env" | |
| "hook-env" | |
| "ls" | |
| ) | |
| echo "## Hyperfine Performance" >> comment.md | |
| for cmd in "${CMDS[@]}"; do | |
| hyperfine_log="$(mktemp)" | |
| reference_cmd="${MISE_ALT:-mise-${{ steps.versions.outputs.release }}} $cmd" | |
| current_cmd="mise $cmd" | |
| mise x hyperfine -- hyperfine -N -w 10 -r 500 --export-markdown out.md --reference "$reference_cmd" "$current_cmd" 2>&1 | tee "$hyperfine_log" | |
| echo "### \`mise $cmd\`" >> comment.md | |
| cat out.md >> comment.md | |
| cat out.md | |
| noisy=false | |
| if grep -q "$outlier_warning" "$hyperfine_log"; then | |
| noisy=true | |
| fi | |
| # Extract relative performance from hyperfine output | |
| relative_row="$(grep "±.*±" out.md || true)" | |
| if [ -z "$relative_row" ]; then | |
| echo "::warning title=Missing hyperfine relative result::Could not find a relative result for mise $cmd." | |
| echo "⚠️ Inconclusive: could not find a relative result for \`$cmd\`." >> comment.md | |
| rm -f "$hyperfine_log" | |
| continue | |
| fi | |
| variance=$(printf '%s\n' "$relative_row" | awk '{print $(NF-3)}' | sed 's/%//') | |
| uncertainty=$(printf '%s\n' "$relative_row" | awk '{print $(NF-1)}' | sed 's/%//') | |
| if [ -z "$variance" ] || [ -z "$uncertainty" ]; then | |
| echo "::warning title=Malformed hyperfine relative result::Could not parse the relative result for mise $cmd." | |
| echo "⚠️ Inconclusive: could not parse the relative result for \`$cmd\`." >> comment.md | |
| rm -f "$hyperfine_log" | |
| continue | |
| fi | |
| variance_scaled=$(scale_decimal "$variance") | |
| uncertainty_scaled=$(scale_decimal "$uncertainty") | |
| variance=$(((variance_scaled - 1000) / 10)) | |
| # Add warning if variance exceeds 10% | |
| if [ "$variance_scaled" -gt "$threshold_scaled" ]; then | |
| if printf '%s\n' "$relative_row" | grep -Fq "| \`$reference_cmd\` |"; then | |
| echo "✅ Performance improvement for \`$cmd\` is ${variance}%" >> comment.md | |
| elif [ "$noisy" = true ]; then | |
| echo "::warning title=Noisy hyperfine result::mise $cmd measured ${variance}% slower, but hyperfine reported statistical outliers. Treating this benchmark as inconclusive." | |
| echo "⚠️ Inconclusive: \`$cmd\` measured ${variance}% slower, but hyperfine reported statistical outliers." >> comment.md | |
| elif [ "$((variance_scaled - uncertainty_scaled))" -le "$threshold_scaled" ]; then | |
| echo "::warning title=Inconclusive hyperfine result::mise $cmd measured ${variance}% slower, but the relative uncertainty overlaps the 10% threshold. Treating this benchmark as inconclusive." | |
| echo "⚠️ Inconclusive: \`$cmd\` measured ${variance}% slower, but the relative uncertainty overlaps the 10% threshold." >> comment.md | |
| else | |
| echo "⚠️ Warning: Performance variance for \`$cmd\` is ${variance}%" >> comment.md | |
| failed=true | |
| fi | |
| fi | |
| rm -f "$hyperfine_log" | |
| done | |
| if [ "$failed" = true ]; then | |
| exit 1 | |
| fi | |
| env: | |
| SHELL: zsh | |
| - run: mise run test:perf | |
| if: ${{ !cancelled() }} | |
| env: | |
| NUM_TOOLS: 200 | |
| NUM_TASKS: 2000 | |
| RUNS: 10 | |
| MISE_ALT: mise-${{ steps.versions.outputs.release }} | |
| - if: ${{ !cancelled() }} | |
| run: sccache --show-stats | |
| - run: cat comment.md >> "$GITHUB_STEP_SUMMARY" | |
| if: ${{ !cancelled() && github.event_name == 'pull_request' }} | |
| - name: Comment on PR | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 | |
| if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} | |
| continue-on-error: true | |
| with: | |
| script: | | |
| const fs = require("fs"); | |
| const marker = "<!-- hyperfine -->"; | |
| const body = `${marker}\n${fs.readFileSync("comment.md", "utf8")}`; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const comment = comments.find((comment) => comment.body?.includes(marker)); | |
| if (comment) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: comment.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body, | |
| }); | |
| } |