Skip to content

Commit 3999880

Browse files
committed
Transformers uplift automation pipeline - v3
1 parent 6587c0d commit 3999880

10 files changed

Lines changed: 301 additions & 159 deletions

.github/scripts/uplift/transformers/detect-new-version.sh

Lines changed: 15 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,3 @@
1-
#!/usr/bin/env bash
2-
#
3-
# Detects whether a newer stable release of `transformers` is available on
4-
# PyPI compared to the pin in venv/requirements-dev.txt.
5-
#
6-
# Emits (to $GITHUB_OUTPUT when set, otherwise stdout only):
7-
# current_version=<X.Y.Z>
8-
# new_version=<X.Y.Z> (empty when no update)
9-
# has_update=<true|false>
10-
#
11-
# Exit code is always 0 unless something goes wrong with the lookup itself
12-
# ("no new version" is not an error).
13-
141
set -euo pipefail
152

163
REPO_ROOT="$(git rev-parse --show-toplevel)"
@@ -38,24 +25,21 @@ RELEASE_TAGS=$(gh release list --repo huggingface/transformers \
3825
--limit 100 --json tagName,isPrerelease \
3926
-q '.[] | select(.isPrerelease == false) | .tagName')
4027

41-
# TEMP: pinned to 5.9.0 for transformers-uplift pipeline testing.
42-
# Restore the python block below before this change merges.
43-
LATEST="5.9.0"
44-
# LATEST=$(python3 - "$CURRENT" <<PY
45-
# import sys
46-
# from packaging.version import Version, InvalidVersion
47-
# cur = Version(sys.argv[1])
48-
# nexts = []
49-
# for t in """$RELEASE_TAGS""".split():
50-
# try:
51-
# v = Version(t.lstrip("v"))
52-
# except InvalidVersion:
53-
# continue
54-
# if v > cur:
55-
# nexts.append(v)
56-
# print(min(nexts) if nexts else "")
57-
# PY
58-
# )
28+
LATEST=$(python3 - "$CURRENT" <<PY
29+
import sys
30+
from packaging.version import Version, InvalidVersion
31+
cur = Version(sys.argv[1])
32+
nexts = []
33+
for t in """$RELEASE_TAGS""".split():
34+
try:
35+
v = Version(t.lstrip("v"))
36+
except InvalidVersion:
37+
continue
38+
if v > cur:
39+
nexts.append(v)
40+
print(min(nexts) if nexts else "")
41+
PY
42+
)
5943

6044
HAS_UPDATE="false"
6145
NEW_VERSION=""

.github/workflows/call-perf-test.yml

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -192,8 +192,6 @@ jobs:
192192
export PYTHONPATH="${{ steps.strings.outputs.work-dir }}:$PYTHONPATH"
193193
194194
source venv/activate
195-
echo "=== Active transformers version ==="
196-
python -c "import transformers; print('transformers ==', transformers.__version__, 'from', transformers.__file__)"
197195
pytest -svv --confcutdir=tests/benchmark ${{ matrix.build.pytest }} ${{ matrix.build.accuracy-testing && '--accuracy-testing' || '' }} ${{ matrix.build.batch-size && format('--batch-size {0}', matrix.build.batch-size) || '' }} ${{ inputs.check_fusions && '--check-fusions' || '' }} --output-file ${{ steps.strings.outputs.perf_report_json_file }}
198196
199197
- name: Dump stablehlo to report
@@ -456,13 +454,10 @@ jobs:
456454
workflow_file="schedule-nightly.yml"
457455
fi
458456
echo "mode=$mode" >> $GITHUB_OUTPUT
459-
# TEMP: hardcoded baseline nightly run for transformers-uplift
460-
# perf testing. Restore the curl query below before merging.
461-
workflow_id="28209379088"
462-
# workflow_id=$(curl -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
463-
# -H "Accept: application/vnd.github.v3+json" \
464-
# "https://api.github.com/repos/${{ github.repository }}/actions/workflows/${workflow_file}/runs?status=completed&branch=main&per_page=100" \
465-
# | jq -r '.workflow_runs[0].id')
457+
workflow_id=$(curl -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
458+
-H "Accept: application/vnd.github.v3+json" \
459+
"https://api.github.com/repos/${{ github.repository }}/actions/workflows/${workflow_file}/runs?status=completed&branch=main&per_page=100" \
460+
| jq -r '.workflow_runs[0].id')
466461
echo "last_nightly_workflow_id=$workflow_id" >> $GITHUB_OUTPUT
467462
468463
echo "device_type=${{ matrix.build.runs-on-original }}" >> $GITHUB_OUTPUT

.github/workflows/call-perf-uplift.yml

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Perf Benchmark + Uplift Fix
1+
name: Perf Benchmark + Transformers Uplift Fix
22

33
permissions:
44
contents: write
@@ -10,9 +10,6 @@ permissions:
1010
on:
1111
workflow_call:
1212
inputs:
13-
branch:
14-
type: string
15-
required: true
1613
iteration:
1714
type: number
1815
required: false
@@ -42,10 +39,7 @@ on:
4239
adv_filter:
4340
type: string
4441
required: false
45-
# TEMP: narrowed to n150-perf only for transformers-uplift perf
46-
# testing. Restore the full-arch entry below before merging.
47-
default: '[ {"accuracy-testing": false}, {"runs-on": "n150-perf", "exclude": "vllm"}]'
48-
# default: '[ {"accuracy-testing": false}, {"runs-on": "n150-perf", "exclude": "vllm"}, {"runs-on": "p150-perf", "exclude": "vllm"}, {"runs-on": "n300-llmbox", "exclude": "vllm"}, {"runs-on": "galaxy-wh-6u", "exclude": "vllm"}, {"runs-on": "qb2-blackhole", "exclude": "vllm"} ]'
42+
default: '[ {"accuracy-testing": false}, {"runs-on": "n150-perf", "exclude": "vllm"}, {"runs-on": "p150-perf", "exclude": "vllm"}, {"runs-on": "n300-llmbox", "exclude": "vllm"}, {"runs-on": "galaxy-wh-6u", "exclude": "vllm"}, {"runs-on": "qb2-blackhole", "exclude": "vllm"} ]'
4943
outputs:
5044
perf_result:
5145
description: 'Result of the underlying perf job (success | failure | skipped | cancelled)'
@@ -85,7 +79,7 @@ jobs:
8579
steps:
8680
- uses: actions/checkout@v6
8781
with:
88-
ref: ${{ inputs.branch }}
82+
ref: ${{ github.ref_name }}
8983

9084
- name: Download failed perf job logs
9185
shell: bash
@@ -155,7 +149,7 @@ jobs:
155149
156150
- uses: actions/checkout@v6
157151
with:
158-
ref: ${{ inputs.branch }}
152+
ref: ${{ github.ref_name }}
159153
token: ${{ secrets.GH_TOKEN }}
160154
submodules: recursive
161155

@@ -216,7 +210,7 @@ jobs:
216210
if: always()
217211
shell: bash
218212
env:
219-
BRANCH: ${{ inputs.branch }}
213+
BRANCH: ${{ github.ref_name }}
220214
ITER: ${{ inputs.iteration }}
221215
TAG: ${{ inputs.tag }}
222216
run: |
@@ -238,11 +232,6 @@ jobs:
238232
git add -A
239233
git commit "${COMMIT_FLAG[@]}"
240234
241-
echo "=== submodule push diagnostic ==="
242-
echo "local HEAD : $(git rev-parse HEAD)"
243-
echo "local origin/$BRANCH : $(git rev-parse "origin/$BRANCH" 2>/dev/null || echo "(missing)")"
244-
echo "server tip $BRANCH : $(git ls-remote origin "$BRANCH" | cut -f1)"
245-
echo "================================="
246235
git push --force origin "$BRANCH"
247236
popd >/dev/null
248237
fi

.github/workflows/call-test-uplift.yml

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Test + Uplift Fix
1+
name: Test + Transformers Uplift Fix
22

33
permissions:
44
contents: write
@@ -27,9 +27,6 @@ on:
2727
required: false
2828
default: "default"
2929
description: 'Extra artifact/commit-message discriminator (e.g. "base" vs "full") when the caller invokes this workflow more than once in the same run with the same scope.'
30-
branch:
31-
type: string
32-
required: true
3330
iteration:
3431
type: number
3532
required: false
@@ -89,7 +86,7 @@ jobs:
8986
steps:
9087
- uses: actions/checkout@v6
9188
with:
92-
ref: ${{ inputs.branch }}
89+
ref: ${{ github.ref_name }}
9390

9491
- uses: actions/download-artifact@v4
9592
with:
@@ -169,7 +166,7 @@ jobs:
169166
170167
- uses: actions/checkout@v6
171168
with:
172-
ref: ${{ inputs.branch }}
169+
ref: ${{ github.ref_name }}
173170
token: ${{ secrets.GH_TOKEN }}
174171
submodules: recursive
175172

@@ -230,7 +227,7 @@ jobs:
230227
if: always()
231228
shell: bash
232229
env:
233-
BRANCH: ${{ inputs.branch }}
230+
BRANCH: ${{ github.ref_name }}
234231
ITER: ${{ inputs.iteration }}
235232
SCOPE: ${{ inputs.scope }}
236233
TAG: ${{ inputs.tag }}
@@ -253,11 +250,6 @@ jobs:
253250
git add -A
254251
git commit "${COMMIT_FLAG[@]}"
255252
256-
echo "=== submodule push diagnostic ==="
257-
echo "local HEAD : $(git rev-parse HEAD)"
258-
echo "local origin/$BRANCH : $(git rev-parse "origin/$BRANCH" 2>/dev/null || echo "(missing)")"
259-
echo "server tip $BRANCH : $(git ls-remote origin "$BRANCH" | cut -f1)"
260-
echo "================================="
261253
git push --force origin "$BRANCH"
262254
popd >/dev/null
263255
fi

.github/workflows/call-test.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -399,8 +399,6 @@ jobs:
399399
run: |
400400
source venv/activate
401401
[[ -f topology_exports.sh ]] && source topology_exports.sh
402-
echo "=== Active transformers version ==="
403-
python -c "import transformers; print('transformers ==', transformers.__version__, 'from', transformers.__file__)"
404402
python -m pytest ${{ env.VERBOSITY }} ${{ env.BASE_PYTEST_CMD }} \
405403
--junitxml=${{ steps.strings.outputs.test_report_path }} \
406404
--perf-report-dir=${{ steps.strings.outputs.perf_report_dir }} \
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
name: Manual - Perf + Transformers Uplift Fix
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
target_version:
7+
description: 'Target transformers version (e.g. 5.9.0). Used for SKILL prompt context.'
8+
required: true
9+
type: string
10+
current_version:
11+
description: 'Previous transformers version (e.g. 5.8.0). Used for SKILL prompt context.'
12+
required: true
13+
type: string
14+
runs-on-filter:
15+
description: "Runner pool to target. 'All' enables every runner the matrix offers."
16+
required: false
17+
type: choice
18+
options:
19+
- All
20+
- n150-perf
21+
- p150-perf
22+
- n300-llmbox
23+
- galaxy-wh-6u
24+
- qb2-blackhole
25+
default: n150-perf
26+
test_filter:
27+
description: "Optional name-substring filter (case-insensitive). e.g. 'resn' or 'resn,mni,yol'."
28+
required: false
29+
type: string
30+
skip-device-perf:
31+
description: 'Skip device performance measurement.'
32+
required: false
33+
type: boolean
34+
default: true
35+
regression_check:
36+
description: 'Enable perf metrics regression testing.'
37+
required: false
38+
type: boolean
39+
default: true
40+
check_fusions:
41+
description: 'Verify expected fusion ops are present in compiled IR.'
42+
required: false
43+
type: boolean
44+
default: true
45+
tag:
46+
description: 'Artifact / commit-message discriminator.'
47+
required: false
48+
type: string
49+
default: 'manual'
50+
mlir_override:
51+
description: 'Optional git SHA of tenstorrent/tt-mlir to override the submodule pointer.'
52+
required: false
53+
type: string
54+
55+
permissions:
56+
contents: write
57+
actions: write
58+
checks: write
59+
packages: write
60+
id-token: write
61+
pull-requests: write
62+
63+
run-name: "Perf + Uplift Fix ( branch: ${{ github.ref_name }} | runner: ${{ inputs.runs-on-filter }} | filter: ${{ inputs.test_filter || '(all)' }} )"
64+
65+
jobs:
66+
build-image:
67+
uses: ./.github/workflows/call-build-docker.yml
68+
secrets: inherit
69+
with:
70+
mlir_override: ${{ inputs.mlir_override }}
71+
dockbuild: all
72+
73+
build-ttxla:
74+
needs: build-image
75+
uses: ./.github/workflows/call-build.yml
76+
secrets: inherit
77+
with:
78+
docker_image: ${{ needs.build-image.outputs.docker-image }}
79+
docker_image_manylinux: ${{ needs.build-image.outputs.docker-image-manylinux }}
80+
mlir_override: ${{ inputs.mlir_override }}
81+
build_types: release
82+
83+
set-filter:
84+
runs-on: ubuntu-latest
85+
outputs:
86+
filter: ${{ steps.gen.outputs.filter }}
87+
steps:
88+
- name: Build adv_filter
89+
id: gen
90+
shell: bash
91+
run: |
92+
echo "[" > adv_filter.json
93+
if [[ "${{ inputs.runs-on-filter }}" == "All" ]]; then
94+
echo '{ "skip": false, "accuracy-testing": false }' >> adv_filter.json
95+
else
96+
echo '{ "runs-on": "${{ inputs.runs-on-filter }}", "accuracy-testing": false }' >> adv_filter.json
97+
fi
98+
if [[ -n "${{ inputs.test_filter }}" ]]; then
99+
echo ',{"filter": "${{ inputs.test_filter }}"}' >> adv_filter.json
100+
fi
101+
echo "]" >> adv_filter.json
102+
filter=$(cat adv_filter.json | jq -r -c '.')
103+
echo "filter=$filter" >> "$GITHUB_OUTPUT"
104+
echo "### Perf uplift inputs" >> "$GITHUB_STEP_SUMMARY"
105+
echo "Branch: ${{ github.ref_name }}" >> "$GITHUB_STEP_SUMMARY"
106+
echo "Target version: ${{ inputs.target_version }}" >> "$GITHUB_STEP_SUMMARY"
107+
echo "Runs-on filter: ${{ inputs.runs-on-filter }}" >> "$GITHUB_STEP_SUMMARY"
108+
echo "Test filter: ${{ inputs.test_filter || '(none)' }}" >> "$GITHUB_STEP_SUMMARY"
109+
110+
perf-and-fix:
111+
needs: [build-image, build-ttxla, set-filter]
112+
if: ${{ !cancelled() && !failure() }}
113+
uses: ./.github/workflows/call-perf-uplift.yml
114+
secrets: inherit
115+
with:
116+
iteration: 1
117+
tag: ${{ inputs.tag }}
118+
target_version: ${{ inputs.target_version }}
119+
current_version: ${{ inputs.current_version }}
120+
artifact_release_run_id: ${{ needs.build-ttxla.outputs.artifacts_run_id }}
121+
artifact_suffix: ${{ needs.build-ttxla.outputs.artifact_suffix }}
122+
docker_image: ${{ needs.build-image.outputs.docker-image-base }}
123+
adv_filter: ${{ needs.set-filter.outputs.filter }}

0 commit comments

Comments
 (0)