[ci]: add transient pytest reruns for Modal tests

.buildkite/pipeline.yml

@@ -99,8 +99,6 @@ steps:
             limit: 3
           - exit_status: -1
             limit: 2
-          - exit_status: 1
-            limit: 2
       agents:
         queue: "default"
     - label: ":test_tube: LoRA Inference Tests"
@@ -167,8 +165,6 @@ steps:
             limit: 3
           - exit_status: -1
             limit: 2
-          - exit_status: 1
-            limit: 2
       agents:
         queue: "default"
     - label: ":test_tube: Training Tests VSA"
@@ -180,8 +176,6 @@ steps:
             limit: 3
           - exit_status: -1
             limit: 2
-          - exit_status: 1
-            limit: 2
       agents:
         queue: "default"
     - label: ":test_tube: Inference Tests VMoBA"
@@ -361,10 +355,6 @@ steps:
                 label: ":bar_chart: SSIM Tests"
                 env:
                   - TEST_TYPE=ssim
-                retry:
-                  automatic:
-                    - exit_status: 1
-                      limit: 2
                 agents:
                   queue: "default"
             - path:
@@ -440,10 +430,6 @@ steps:
                 label: ":test_tube: LoRA Training Tests"
                 env:
                   - TEST_TYPE=training_lora
-                retry:
-                  automatic:
-                    - exit_status: 1
-                      limit: 2
                 agents:
                   queue: "default"
             - path:
@@ -456,10 +442,6 @@ steps:
                 label: ":test_tube: Training Tests VSA"
                 env:
                   - TEST_TYPE=training_vsa
-                retry:
-                  automatic:
-                    - exit_status: 1
-                      limit: 2
                 agents:
                   queue: "default"
             - path:

docs/contributing/ci_architecture.md

@@ -125,6 +125,24 @@ it), while a GitHub outage or a >25 min wait lets it run anyway (fail open).
 See [Performance Benchmarks](performance_benchmarks.md) for the performance
 lane's thresholds, rolling baseline, artifacts, and reseeding process.
 
+### Modal Pytest Reruns
+
+Modal pytest lanes install `pytest-rerunfailures` through the `test` extra.
+`fastvideo/tests/modal/pr_test.py` applies the shared retry policy through
+`PYTEST_ADDOPTS`, and `fastvideo/tests/modal/ssim_test.py` appends the same
+arguments to each SSIM subprocess.
+
+The policy reruns an individual pytest failure twice with a short delay only
+when pytest receives a failure report whose text matches the infrastructure
+regex in `fastvideo/tests/modal/pytest_retry.py`. Rerun attempts retain their
+tracebacks even when a later attempt passes. Plain, nonmatching assertion and
+numerical parity failures fail without rerun; a deterministic failure whose
+message matches the regex can still be rerun.
+
+Process termination, Modal worker or container loss, and preemption do not
+produce a pytest failure report for this policy to inspect. Modal and Buildkite
+own recovery for those failures at the function and job layers.
+
 ## Slash Commands
 
 Slash commands are handled by `.github/workflows/ci-slash-commands.yml`.
@@ -239,7 +257,9 @@ All Buildkite test jobs go through `.buildkite/scripts/pr_test.sh`, which:
 3. Passes Buildkite metadata into the Modal container.
 4. Runs the selected test command from `fastvideo/tests/modal/pr_test.py` or
    `fastvideo/tests/modal/ssim_test.py`.
-5. Uploads performance artifacts for `TEST_TYPE=performance`.
+5. Applies the shared infrastructure-pattern pytest rerun policy to Modal pytest
+   commands.
+6. Uploads performance artifacts for `TEST_TYPE=performance`.
 
 If you add a new CI test category:
 

docs/contributing/testing.md

@@ -219,6 +219,19 @@ instances. The main files are:
 For exact tier membership, path filters, slash commands, and aggregate statuses,
 see [CI/CD Architecture](ci_architecture.md).
 
+Modal pytest commands use `pytest-rerunfailures` with the shared
+infrastructure-pattern policy in `fastvideo/tests/modal/pytest_retry.py`. The
+retry layer reruns an individual failure only when it reaches pytest and its
+report text matches an infrastructure-style connection, NCCL, or CUDA pattern.
+Each rerun preserves
+the prior attempt's traceback. Plain, nonmatching assertion and numerical
+parity failures fail immediately, while matching messages may be rerun even
+when their underlying cause is deterministic.
+
+Killed or preempted Modal workers and containers cannot be recovered by this
+pytest policy. Modal and Buildkite handle those failures outside the test
+process.
+
 ### Adding A New CI Test Category
 
 If a new test does not fit an existing lane:

fastvideo/tests/contract/test_ci_pytest_reruns.py

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Contract and behavior checks for Modal pytest rerun wiring.
+
+No fastvideo imports, GPU, or torch required.
+"""
+import importlib.util
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+PYPROJECT = REPO_ROOT / "pyproject.toml"
+MODAL_ROOT = REPO_ROOT / "fastvideo" / "tests" / "modal"
+PYTEST_RETRY = MODAL_ROOT / "pytest_retry.py"
+PR_TEST = MODAL_ROOT / "pr_test.py"
+SSIM_TEST = MODAL_ROOT / "ssim_test.py"
+
+
+def test_pytest_rerunfailures_is_in_test_extra():
+    assert "pytest-rerunfailures>=16.3" in PYPROJECT.read_text(encoding="utf-8")
+
+
+def test_modal_entrypoints_share_pytest_retry_policy():
+    retry_text = PYTEST_RETRY.read_text(encoding="utf-8")
+    pr_text = PR_TEST.read_text(encoding="utf-8")
+    ssim_text = SSIM_TEST.read_text(encoding="utf-8")
+
+    assert "TRANSIENT_FAILURE_REGEX" in retry_text
+    assert "--only-rerun" in retry_text
+    assert "--rerun-show-tracebacks" in retry_text
+    assert "build_pytest_addopts" in pr_text
+    assert "PYTEST_ADDOPTS" in pr_text
+    assert "_build_pytest_rerun_args()" in ssim_text
+
+
+def test_retry_policy_excludes_plain_assertion_failures():
+    retry_text = PYTEST_RETRY.read_text(encoding="utf-8")
+
+    assert "AssertionError" not in retry_text
+    assert "assert " not in retry_text
+
+
+def test_retry_policy_runtime_behavior(tmp_path, monkeypatch):
+    spec = importlib.util.spec_from_file_location("pytest_retry_under_test", PYTEST_RETRY)
+    assert spec is not None
+    assert spec.loader is not None
+    pytest_retry = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(pytest_retry)
+    monkeypatch.setattr(pytest_retry, "PYTEST_RERUNS_DELAY_SECONDS", 0)
+
+    sample_test = tmp_path / "test_retry_policy_sample.py"
+    sample_test.write_text(
+        textwrap.dedent("""
+            from pathlib import Path
+
+            TRANSIENT_ATTEMPTS = Path(__file__).with_name("transient_attempts.txt")
+            ASSERTION_ATTEMPTS = Path(__file__).with_name("assertion_attempts.txt")
+
+
+            def next_attempt(path):
+                attempt = int(path.read_text(encoding="utf-8")) + 1 if path.exists() else 1
+                path.write_text(str(attempt), encoding="utf-8")
+                return attempt
+
+
+            def test_matching_failure_is_rerun():
+                if next_attempt(TRANSIENT_ATTEMPTS) == 1:
+                    raise RuntimeError("503 Service Unavailable from synthetic test")
+
+
+            def test_plain_assertion_is_not_rerun():
+                next_attempt(ASSERTION_ATTEMPTS)
+                assert False, "synthetic plain assertion"
+        """),
+        encoding="utf-8",
+    )
+    env = os.environ.copy()
+    env.pop("PYTEST_ADDOPTS", None)
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", str(sample_test), "-q", *pytest_retry.build_pytest_rerun_args()],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    output = result.stdout + result.stderr
+
+    assert result.returncode == 1, output
+    assert (tmp_path / "transient_attempts.txt").read_text(encoding="utf-8") == "2"
+    assert (tmp_path / "assertion_attempts.txt").read_text(encoding="utf-8") == "1"
+    assert 'raise RuntimeError("503 Service Unavailable from synthetic test")' in output
+    assert "RuntimeError: 503 Service Unavailable from synthetic test" in output

fastvideo/tests/modal/pr_test.py

@@ -1,4 +1,5 @@
 import os
+import shlex
 import sys
 
 import modal
@@ -91,6 +92,24 @@ def run_test(pytest_command: str):
     run_test_command(pytest_command, build_kernel=True)
 
 
+def _pytest_retry_setup_command() -> str:
+    addopts_statement = (
+        "import os, sys; "
+        "sys.path.insert(0, '/FastVideo/fastvideo/tests/modal'); "
+        "from pytest_retry import build_pytest_addopts; "
+        "print(build_pytest_addopts(os.environ.get('PYTEST_ADDOPTS', '')))"
+    )
+    describe_statement = (
+        "import sys; "
+        "sys.path.insert(0, '/FastVideo/fastvideo/tests/modal'); "
+        "from pytest_retry import describe_pytest_reruns; "
+        "print(describe_pytest_reruns())"
+    )
+    addopts_command = f"python -c {shlex.quote(addopts_statement)}"
+    describe_command = f"python -c {shlex.quote(describe_statement)}"
+    return f'PYTEST_ADDOPTS="$({addopts_command})" && export PYTEST_ADDOPTS &&\n    {describe_command} &&'
+
+
 def run_test_command(test_command: str,
                      build_kernel: bool,
                      install_command: str = 'uv pip install -e ".[test]"'):
@@ -143,6 +162,7 @@ def run_test_command(test_command: str,
     {checkout_command} &&
     git submodule update --init --recursive &&
     {install_clause}
+    {_pytest_retry_setup_command()}
     {build_kernel_command}
     {test_command}
     """

fastvideo/tests/modal/pytest_retry.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Shared pytest rerun policy for Modal CI entrypoints."""
+
+from __future__ import annotations
+
+import shlex
+
+PYTEST_RERUNS = 2
+PYTEST_RERUNS_DELAY_SECONDS = 10
+
+TRANSIENT_FAILURE_PATTERNS = (
+    r"connection (?:reset|aborted|refused|timed out)",
+    r"read timed out",
+    r"remote end closed connection",
+    r"temporarily unavailable",
+    r"(?:HTTP|status code) 5\d\d",
+    r"502 Bad Gateway",
+    r"503 Service Unavailable",
+    r"504 Gateway Timeout",
+    r"NCCL.*(?:timeout|connection|remote error|unhandled system error|system error)",
+    r"CUDA error: (?:unknown error|initialization error|operation not permitted|operation not supported)",
+    r"Modal.*(?:preempt|interrupt|worker|container|timeout)",
+)
+TRANSIENT_FAILURE_REGEX = "(?i)(" + "|".join(TRANSIENT_FAILURE_PATTERNS) + ")"
+
+
+def build_pytest_rerun_args() -> list[str]:
+    return [
+        "--reruns",
+        str(PYTEST_RERUNS),
+        "--reruns-delay",
+        str(PYTEST_RERUNS_DELAY_SECONDS),
+        "--rerun-show-tracebacks",
+        "--only-rerun",
+        TRANSIENT_FAILURE_REGEX,
+    ]
+
+
+def build_pytest_addopts(existing_addopts: str = "") -> str:
+    retry_addopts = shlex.join(build_pytest_rerun_args())
+    existing_addopts = existing_addopts.strip()
+    if existing_addopts:
+        return f"{existing_addopts} {retry_addopts}"
+    return retry_addopts
+
+
+def describe_pytest_reruns() -> str:
+    return (
+        "Pytest transient reruns enabled: "
+        f"{PYTEST_RERUNS} reruns, {PYTEST_RERUNS_DELAY_SECONDS}s delay, "
+        f"only-rerun={TRANSIENT_FAILURE_REGEX!r}"
+    )

fastvideo/tests/modal/ssim_test.py

@@ -680,7 +680,7 @@ def _build_pytest_extra_args(
     ssim_bootstrap_mode: bool,
     pytest_k: str,
 ) -> list[str]:
-    args = []
+    args = _build_pytest_rerun_args()
     if ssim_full_quality:
         args.append("--ssim-full-quality")
     if ssim_reference_repo.strip():
@@ -694,6 +694,21 @@ def _build_pytest_extra_args(
     return args
 
 
+def _build_pytest_rerun_args() -> list[str]:
+    helper_paths = [
+        os.path.dirname(os.path.abspath(__file__)),
+        os.path.join("/FastVideo", "fastvideo", "tests", "modal"),
+    ]
+    for helper_path in helper_paths:
+        if helper_path not in sys.path:
+            sys.path.insert(0, helper_path)
+
+    from pytest_retry import build_pytest_rerun_args, describe_pytest_reruns  # noqa: E402
+
+    print(describe_pytest_reruns())
+    return build_pytest_rerun_args()
+
+
 def _schedule_ssim_tasks(
     repo_root: str,
     tasks: list[SSIMTask],

pyproject.toml

@@ -136,6 +136,7 @@ test = [
     "av",
     "pytorch-msssim>=1.0.0",
     "pytest",
+    "pytest-rerunfailures>=16.3",
     "pandas",
     "plotly",
 ]