[ci]: add transient pytest reruns for Modal tests

docs/contributing/ci_architecture.md

@@ -125,6 +125,18 @@ it), while a GitHub outage or a >25 min wait lets it run anyway (fail open).
 See [Performance Benchmarks](performance_benchmarks.md) for the performance
 lane's thresholds, rolling baseline, artifacts, and reseeding process.
 
+### Modal Pytest Reruns
+
+Modal pytest lanes install `pytest-rerunfailures` through the `test` extra.
+`fastvideo/tests/modal/pr_test.py` applies the shared retry policy through
+`PYTEST_ADDOPTS`, and `fastvideo/tests/modal/ssim_test.py` appends the same
+arguments to each SSIM subprocess.
+
+The policy reruns an individual pytest failure twice with a short delay, but
+only when the failure text matches the transient infrastructure regex in
+`fastvideo/tests/modal/pytest_retry.py`. Plain assertion and numerical parity
+failures are not matched by that policy and should fail without rerun.
+
 ## Slash Commands
 
 Slash commands are handled by `.github/workflows/ci-slash-commands.yml`.
@@ -239,7 +251,9 @@ All Buildkite test jobs go through `.buildkite/scripts/pr_test.sh`, which:
 3. Passes Buildkite metadata into the Modal container.
 4. Runs the selected test command from `fastvideo/tests/modal/pr_test.py` or
    `fastvideo/tests/modal/ssim_test.py`.
-5. Uploads performance artifacts for `TEST_TYPE=performance`.
+5. Applies the shared transient-only pytest rerun policy to Modal pytest
+   commands.
+6. Uploads performance artifacts for `TEST_TYPE=performance`.
 
 If you add a new CI test category:
 

docs/contributing/testing.md

@@ -219,6 +219,12 @@ instances. The main files are:
 For exact tier membership, path filters, slash commands, and aggregate statuses,
 see [CI/CD Architecture](ci_architecture.md).
 
+Modal pytest commands use `pytest-rerunfailures` with the shared transient-only
+policy in `fastvideo/tests/modal/pytest_retry.py`. The retry layer reruns
+individual pytest failures for infrastructure-style errors such as connection,
+Modal worker, NCCL, or CUDA initialization failures. Deterministic assertion
+and numerical parity failures should still fail immediately.
+
 ### Adding A New CI Test Category
 
 If a new test does not fit an existing lane:

fastvideo/tests/contract/test_ci_pytest_reruns.py

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Static checks for Modal pytest rerun wiring.
+
+Pure text analysis: no fastvideo imports, no GPU, no torch.
+"""
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+PYPROJECT = REPO_ROOT / "pyproject.toml"
+MODAL_ROOT = REPO_ROOT / "fastvideo" / "tests" / "modal"
+PYTEST_RETRY = MODAL_ROOT / "pytest_retry.py"
+PR_TEST = MODAL_ROOT / "pr_test.py"
+SSIM_TEST = MODAL_ROOT / "ssim_test.py"
+
+
+def test_pytest_rerunfailures_is_in_test_extra():
+    assert "pytest-rerunfailures" in PYPROJECT.read_text(encoding="utf-8")
+
+
+def test_modal_entrypoints_share_pytest_retry_policy():
+    retry_text = PYTEST_RETRY.read_text(encoding="utf-8")
+    pr_text = PR_TEST.read_text(encoding="utf-8")
+    ssim_text = SSIM_TEST.read_text(encoding="utf-8")
+
+    assert "TRANSIENT_FAILURE_REGEX" in retry_text
+    assert "--only-rerun" in retry_text
+    assert "build_pytest_addopts" in pr_text
+    assert "PYTEST_ADDOPTS" in pr_text
+    assert "_build_pytest_rerun_args()" in ssim_text
+
+
+def test_retry_policy_excludes_plain_assertion_failures():
+    retry_text = PYTEST_RETRY.read_text(encoding="utf-8")
+
+    assert "AssertionError" not in retry_text
+    assert "assert " not in retry_text

fastvideo/tests/modal/pr_test.py

@@ -1,4 +1,5 @@
 import os
+import shlex
 import sys
 
 import modal
@@ -91,6 +92,24 @@ def run_test(pytest_command: str):
     run_test_command(pytest_command, build_kernel=True)
 
 
+def _pytest_retry_setup_command() -> str:
+    addopts_statement = (
+        "import os, sys; "
+        "sys.path.insert(0, '/FastVideo/fastvideo/tests/modal'); "
+        "from pytest_retry import build_pytest_addopts; "
+        "print(build_pytest_addopts(os.environ.get('PYTEST_ADDOPTS', '')))"
+    )
+    describe_statement = (
+        "import sys; "
+        "sys.path.insert(0, '/FastVideo/fastvideo/tests/modal'); "
+        "from pytest_retry import describe_pytest_reruns; "
+        "print(describe_pytest_reruns())"
+    )
+    addopts_command = f"python -c {shlex.quote(addopts_statement)}"
+    describe_command = f"python -c {shlex.quote(describe_statement)}"
+    return f'PYTEST_ADDOPTS="$({addopts_command})" && export PYTEST_ADDOPTS &&\n    {describe_command} &&'
+
+
 def run_test_command(test_command: str,
                      build_kernel: bool,
                      install_command: str = 'uv pip install -e ".[test]"'):
@@ -143,6 +162,7 @@ def run_test_command(test_command: str,
     {checkout_command} &&
     git submodule update --init --recursive &&
     {install_clause}
+    {_pytest_retry_setup_command()}
     {build_kernel_command}
     {test_command}
     """

fastvideo/tests/modal/pytest_retry.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Shared pytest rerun policy for Modal CI entrypoints."""
+
+from __future__ import annotations
+
+import shlex
+
+PYTEST_RERUNS = 2
+PYTEST_RERUNS_DELAY_SECONDS = 10
+
+TRANSIENT_FAILURE_PATTERNS = (
+    r"connection (?:reset|aborted|refused|timed out)",
+    r"read timed out",
+    r"remote end closed connection",
+    r"temporarily unavailable",
+    r"(?:HTTP|status code) 5\d\d",
+    r"502 Bad Gateway",
+    r"503 Service Unavailable",
+    r"504 Gateway Timeout",
+    r"NCCL.*(?:timeout|connection|remote error|unhandled system error|system error)",
+    r"CUDA error: (?:unknown error|initialization error|operation not permitted|operation not supported)",
+    r"Modal.*(?:preempt|interrupt|worker|container|timeout)",
+)
+TRANSIENT_FAILURE_REGEX = "(?i)(" + "|".join(TRANSIENT_FAILURE_PATTERNS) + ")"
+
+
+def build_pytest_rerun_args() -> list[str]:
+    return [
+        "--reruns",
+        str(PYTEST_RERUNS),
+        "--reruns-delay",
+        str(PYTEST_RERUNS_DELAY_SECONDS),
+        "--only-rerun",
+        TRANSIENT_FAILURE_REGEX,
+    ]
+
+
+def build_pytest_addopts(existing_addopts: str = "") -> str:
+    retry_addopts = shlex.join(build_pytest_rerun_args())
+    existing_addopts = existing_addopts.strip()
+    if existing_addopts:
+        return f"{existing_addopts} {retry_addopts}"
+    return retry_addopts
+
+
+def describe_pytest_reruns() -> str:
+    return (
+        "Pytest transient reruns enabled: "
+        f"{PYTEST_RERUNS} reruns, {PYTEST_RERUNS_DELAY_SECONDS}s delay, "
+        f"only-rerun={TRANSIENT_FAILURE_REGEX!r}"
+    )

fastvideo/tests/modal/ssim_test.py

@@ -680,7 +680,7 @@ def _build_pytest_extra_args(
     ssim_bootstrap_mode: bool,
     pytest_k: str,
 ) -> list[str]:
-    args = []
+    args = _build_pytest_rerun_args()
     if ssim_full_quality:
         args.append("--ssim-full-quality")
     if ssim_reference_repo.strip():
@@ -694,6 +694,21 @@ def _build_pytest_extra_args(
     return args
 
 
+def _build_pytest_rerun_args() -> list[str]:
+    helper_paths = [
+        os.path.dirname(os.path.abspath(__file__)),
+        os.path.join("/FastVideo", "fastvideo", "tests", "modal"),
+    ]
+    for helper_path in helper_paths:
+        if helper_path not in sys.path:
+            sys.path.insert(0, helper_path)
+
+    from pytest_retry import build_pytest_rerun_args, describe_pytest_reruns  # noqa: E402
+
+    print(describe_pytest_reruns())
+    return build_pytest_rerun_args()
+
+
 def _schedule_ssim_tasks(
     repo_root: str,
     tasks: list[SSIMTask],

pyproject.toml

@@ -136,6 +136,7 @@ test = [
     "av",
     "pytorch-msssim>=1.0.0",
     "pytest",
+    "pytest-rerunfailures",
     "pandas",
     "plotly",
 ]