Avoid half-deleted cache trees on concurrent pip-compile --rebuild

Two ``pip-compile --rebuild`` invocations sharing a ``cache_dir`` race
inside ``PyPIRepository.clear_caches``. Both call
``shutil.rmtree(self._download_dir, ignore_errors=True)`` on the same
path, and the second resolver walks a tree the first started
removing. ``ignore_errors=True`` masks the error but not the
corruption.

The patch swaps the in-place delete for an atomic ``os.replace`` to a
per-PID sibling directory followed by ``rmtree`` of the renamed copy.
``os.replace`` is atomic on POSIX and NTFS, so a concurrent peer sees
either the old directory or no directory, never a half-deleted tree.

Scope is intentionally narrow: this fixes one observable symptom of
#2393, not the wider race where a process is already using the
cache and a peer ``--rebuild`` later renames the directory out from
under it. Full concurrent-rebuild safety needs locking around the
resolve and is left for future work.

Author: gaborbernat

changelog.d/2393.bugfix.md

@@ -0,0 +1,6 @@
+Atomically rename the download directory before deleting it in
+{meth}`piptools.repositories.PyPIRepository.clear_caches` so a parallel
+{command}`pip-compile --rebuild` no longer trips over a half-deleted
+tree. This narrows one symptom of #2393 and does not fully serialise
+concurrent ``--rebuild`` runs; broader cache locking remains future
+work -- {user}`gaborbernat`.

piptools/repositories/pypi.py

@@ -104,7 +104,20 @@ def __init__(self, pip_args: list[str], cache_dir: str):
         )
 
     def clear_caches(self) -> None:
-        rmtree(self._download_dir, ignore_errors=True)
+        # Atomic rename + opportunistic rmtree narrows one specific race:
+        # two ``pip-compile --rebuild`` runs on the same ``cache_dir`` no
+        # longer leave the second resolver staring at a tree the first
+        # started removing. It does not serialise concurrent ``--rebuild``
+        # runs as a whole; full concurrent-rebuild safety needs locking
+        # around the resolve, which is out of scope here.
+        if not os.path.exists(self._download_dir):
+            return
+        stale = f"{self._download_dir}.stale-{os.getpid()}"
+        try:
+            os.replace(self._download_dir, stale)
+        except OSError:
+            return
+        rmtree(stale, ignore_errors=True)
 
     @property
     def options(self) -> optparse.Values:

pyproject.toml

@@ -58,6 +58,7 @@ changelog = "https://github.com/jazzband/pip-tools/releases"
 [project.optional-dependencies]
 testing = [
   "pytest >= 7.2.0",
+  "pytest-mock >= 3.15.1",
   "pytest-rerunfailures",
   "pytest-xdist",
   "tomli-w",

tests/test_repository_pypi.py

@@ -1,13 +1,15 @@
 from __future__ import annotations
 
 import os
+from pathlib import Path
 from unittest import mock
 
 import pytest
 from pip._internal.models.candidate import InstallationCandidate
 from pip._internal.models.link import Link
 from pip._internal.utils.urls import path_to_url
 from pip._vendor.requests import HTTPError, Session
+from pytest_mock import MockerFixture
 
 from piptools.repositories import PyPIRepository
 from piptools.repositories.pypi import open_local_or_remote_file
@@ -169,6 +171,48 @@ def test_pip_cache_dir_is_empty(from_line, tmpdir):
     assert not pypi_repository.options.cache_dir
 
 
+@pytest.fixture
+def repo_with_cache_dir(tmp_path: Path) -> PyPIRepository:
+    return PyPIRepository([], cache_dir=str(tmp_path / "cache"))
+
+
+def test_clear_caches_removes_existing_download_dir(
+    repo_with_cache_dir: PyPIRepository,
+) -> None:
+    download_dir = Path(repo_with_cache_dir._download_dir)
+    download_dir.mkdir(parents=True, exist_ok=True)
+    (download_dir / "marker").write_text("x")
+
+    repo_with_cache_dir.clear_caches()
+
+    assert not download_dir.exists()
+
+
+def test_clear_caches_when_download_dir_missing_is_a_no_op(
+    repo_with_cache_dir: PyPIRepository,
+) -> None:
+    assert not Path(repo_with_cache_dir._download_dir).exists()
+
+    repo_with_cache_dir.clear_caches()
+
+    assert not Path(repo_with_cache_dir._download_dir).exists()
+
+
+def test_clear_caches_swallows_replace_failure(
+    repo_with_cache_dir: PyPIRepository, mocker: MockerFixture
+) -> None:
+    Path(repo_with_cache_dir._download_dir).mkdir(parents=True, exist_ok=True)
+    mocker.patch(
+        "piptools.repositories.pypi.os.replace",
+        side_effect=OSError("racing peer renamed first"),
+    )
+    rmtree = mocker.patch("piptools.repositories.pypi.rmtree")
+
+    repo_with_cache_dir.clear_caches()
+
+    rmtree.assert_not_called()
+
+
 @pytest.mark.parametrize(
     ("project_data", "expected_hashes"),
     (