Merge branch 'main' into videollama3

Author: lkhl

.circleci/create_circleci_config.py

@@ -177,7 +177,9 @@ def to_dict(self):
                     "command": f"TESTS=$(circleci tests split  --split-by=timings {self.job_name}_test_list.txt) && echo $TESTS > splitted_tests.txt && echo $TESTS | tr ' ' '\n'" if self.parallelism else f"awk '{{printf \"%s \", $0}}' {self.job_name}_test_list.txt > splitted_tests.txt"
                     }
             },
-            {"run": {"name": "fetch hub objects before pytest", "command": "python3 utils/fetch_hub_objects_for_ci.py"}},
+            # During the CircleCI docker images build time, we might already (or not) download the data.
+            # If it's done already, the files are inside the directory `/test_data/`.
+            {"run": {"name": "fetch hub objects before pytest", "command": "cp -r /test_data/* . 2>/dev/null || true; python3 utils/fetch_hub_objects_for_ci.py"}},
             {"run": {
                 "name": "Run tests",
                 "command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n {self.pytest_num_workers} {junit_flags} {repeat_on_failure_flags} {' '.join(pytest_flags)} $(cat splitted_tests.txt) | tee tests_output.txt)"}
@@ -262,7 +264,6 @@ def job_name(self):
     docker_image=[{"image": "huggingface/transformers-custom-tokenizers"}],
 )
 
-
 examples_torch_job = CircleCIJob(
     "examples_torch",
     additional_env={"OMP_NUM_THREADS": 8},
@@ -286,35 +287,20 @@ def job_name(self):
     resource_class="medium",
 )
 
-
-onnx_job = CircleCIJob(
-    "onnx",
-    docker_image=[{"image":"huggingface/transformers-torch-tf-light"}],
-    install_steps=[
-        "uv pip install .[testing,sentencepiece,onnxruntime,vision,rjieba]",
-    ],
-    pytest_options={"k onnx": None},
-    pytest_num_workers=1,
-    resource_class="small",
-)
-
-
 exotic_models_job = CircleCIJob(
     "exotic_models",
     docker_image=[{"image":"huggingface/transformers-exotic-models"}],
     parallelism=4,
     pytest_options={"durations": 100},
 )
 
-
 repo_utils_job = CircleCIJob(
     "repo_utils",
     docker_image=[{"image":"huggingface/transformers-consistency"}],
     pytest_num_workers=4,
     resource_class="large",
 )
 
-
 non_model_job = CircleCIJob(
     "non_model",
     docker_image=[{"image": "huggingface/transformers-torch-light"}],
@@ -350,7 +336,7 @@ def job_name(self):
     pytest_num_workers=1,
 )
 
-REGULAR_TESTS = [torch_job, hub_job, onnx_job, tokenization_job, processor_job, generate_job, non_model_job] # fmt: skip
+REGULAR_TESTS = [torch_job, hub_job, tokenization_job, processor_job, generate_job, non_model_job] # fmt: skip
 EXAMPLES_TESTS = [examples_torch_job]
 PIPELINE_TESTS = [pipelines_torch_job]
 REPO_UTIL_TESTS = [repo_utils_job]

.github/workflows/build-ci-docker-images.yml

@@ -26,7 +26,7 @@ jobs:
 
     strategy:
       matrix:
-        file: ["quality", "consistency", "custom-tokenizers", "torch-light", "tf-light", "exotic-models", "torch-tf-light", "jax-light", "examples-torch",  "examples-tf"]
+        file: ["quality", "consistency", "custom-tokenizers", "torch-light", "exotic-models", "examples-torch"]
     continue-on-error: true
 
     steps:

.github/workflows/build-nightly-ci-docker-images.yml

@@ -2,6 +2,10 @@ name: Build docker images (Nightly CI)
 
 on:
   workflow_call:
+    inputs:
+      job:
+        required: true
+        type: string
   push:
     branches:
       - build_nightly_ci_docker_image*
@@ -12,7 +16,8 @@ concurrency:
 
 jobs:
   latest-with-torch-nightly-docker:
-    name: "Nightly PyTorch + Stable TensorFlow"
+    name: "Nightly PyTorch"
+    if: inputs.job == 'latest-with-torch-nightly-docker' || inputs.job == ''
     runs-on:
       group: aws-general-8-plus
     steps:
@@ -41,6 +46,7 @@ jobs:
 
   nightly-torch-deepspeed-docker:
     name: "Nightly PyTorch + DeepSpeed"
+    if: inputs.job == 'nightly-torch-deepspeed-docker' || inputs.job == ''
     runs-on:
       group: aws-g4dn-2xlarge-cache
     steps:

.github/workflows/model_jobs.yml

@@ -25,6 +25,12 @@ on:
         required: false
         default: run_models_gpu
         type: string
+      runner_type:
+        required: false
+        type: string
+      report_repo_id:
+        required: false
+        type: string
 
 env:
   HF_HOME: /mnt/cache
@@ -143,3 +149,15 @@ jobs:
         with:
           name: ${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
           path: /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports
+
+  collated_reports:
+    name: Collated Reports
+    if: ${{ always() }}
+    needs: run_models_gpu
+    uses: huggingface/transformers/.github/workflows/collated-reports.yml@main
+    with:
+      job: run_models_gpu
+      report_repo_id: ${{ inputs.report_repo_id }}
+      gpu_name: ${{ inputs.runner_type }}
+      machine_type: ${{ inputs.machine_type }}
+    secrets: inherit

.github/workflows/self-nightly-caller.yml

@@ -22,6 +22,8 @@ jobs:
   build_nightly_torch_ci_images:
     name: Build CI Docker Images with nightly torch
     uses: ./.github/workflows/build-nightly-ci-docker-images.yml
+    with:
+      job: latest-with-torch-nightly-docker
     secrets: inherit
 
   setup:

.github/workflows/self-scheduled-amd-mi355-caller.yml

@@ -21,7 +21,7 @@ jobs:
       job: run_models_gpu
       slack_report_channel: "#amd-hf-ci"
       runner_scale_set: amd-mi355-ci
-      docker: huggingface/transformers-pytorch-amd-gpu
+      docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
       report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit
@@ -33,7 +33,7 @@ jobs:
       job: run_pipelines_torch_gpu
       slack_report_channel: "#amd-hf-ci"
       runner_scale_set: amd-mi355-ci
-      docker: huggingface/transformers-pytorch-amd-gpu
+      docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
       report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit
@@ -45,7 +45,7 @@ jobs:
       job: run_examples_gpu
       slack_report_channel: "#amd-hf-ci"
       runner_scale_set: amd-mi355-ci
-      docker: huggingface/transformers-pytorch-amd-gpu
+      docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
       report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit
@@ -57,7 +57,7 @@ jobs:
       job: run_torch_cuda_extensions_gpu
       slack_report_channel: "#amd-hf-ci"
       runner_scale_set: amd-mi355-ci
-      docker: huggingface/transformers-pytorch-deepspeed-amd-gpu
+      docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
       report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit

.github/workflows/self-scheduled-caller.yml

@@ -52,6 +52,7 @@ jobs:
       slack_report_channel: "#transformers-ci-daily-models"
       docker: huggingface/transformers-all-latest-gpu
       ci_event: Daily CI
+      runner_type: "a10"
       report_repo_id: hf-internal-testing/transformers_daily_ci
       commit_sha: ${{ github.sha }}
     secrets: inherit

.github/workflows/self-scheduled.yml

@@ -31,6 +31,9 @@ on:
       commit_sha:
         required: false
         type: string
+      runner_type:
+        required: false
+        type: string
       models:
         default: ""
         required: false
@@ -126,6 +129,8 @@ jobs:
       runner_map: ${{ needs.setup.outputs.runner_map }}
       docker: ${{ inputs.docker }}
       commit_sha: ${{ inputs.commit_sha || github.sha }}
+      runner_type: ${{ inputs.runner_type }}
+      report_repo_id: ${{ inputs.report_repo_id }}
     secrets: inherit
 
   run_trainer_and_fsdp_gpu:

benchmark_v2/.gitignore

@@ -0,0 +1 @@
+benchmark_results/

benchmark_v2/README.md

@@ -0,0 +1,98 @@
+# Benchmarking v2
+
+A comprehensive benchmarking framework for transformer models that supports multiple execution modes (eager, compiled, kernelized), detailed performance metrics collection, and structured output format.
+
+
+## Quick Start
+
+### Running All Benchmarks
+
+```bash
+# Run all benchmarks with default settings
+python run_benchmarks.py
+
+# Specify output directory
+python run_benchmarks.py --output-dir my_results
+
+# Run with custom parameters
+python run_benchmarks.py \
+    --warmup-iterations 5 \
+    --measurement-iterations 10 \
+    --num-tokens-to-generate 200
+```
+
+### Running Specific Benchmarks
+
+```bash
+# Include only specific benchmarks
+python run_benchmarks.py --include llama
+
+# Exclude specific benchmarks
+python run_benchmarks.py --exclude old_benchmark
+
+## Output Format
+
+Results are saved as JSON files with the following structure:
+
+```json
+{
+  "model_name": "llama_2_7b",
+  "benchmark_scenarios": [
+    {
+      "scenario_name": "eager_variant",
+      "metadata": {
+        "timestamp": "2025-01-XX...",
+        "commit_id": "abc123...",
+        "hardware_info": {
+          "gpu_name": "NVIDIA A100",
+          "gpu_memory_total": 40960,
+          "cpu_count": 64
+        },
+        "config": {
+          "variant": "eager",
+          "warmup_iterations": 3,
+          "measurement_iterations": 5
+        }
+      },
+      "measurements": {
+        "latency": {
+          "mean": 2.45,
+          "median": 2.43,
+          "std": 0.12,
+          "min": 2.31,
+          "max": 2.67,
+          "p95": 2.61,
+          "p99": 2.65
+        },
+        "time_to_first_token": {
+          "mean": 0.15,
+          "std": 0.02
+        },
+        "tokens_per_second": {
+          "mean": 87.3,
+          "unit": "tokens/sec"
+        }
+      },
+      "gpu_metrics": {
+        "gpu_utilization_mean": 85.2,
+        "gpu_memory_used_mean": 12450
+      }
+    }
+  ]
+}
+```
+
+### Debug Mode
+
+```bash
+python run_benchmarks.py --log-level DEBUG
+```
+
+## Contributing
+
+To add new benchmarks:
+
+1. Create a new file in `benches/`
+2. Implement the `ModelBenchmark` interface
+3. Add a runner function (`run_<benchmark_name>` or `run_benchmark`)
+4. run_benchmarks.py