Fix source builds failing when the interpreter's bundled pip differs from the pip in the build environment

.changes/next-release/bugfix-Source-64452.json

@@ -0,0 +1,5 @@
+{
+  "type": "bugfix",
+  "category": "Source",
+  "description": "Fix source builds failing when the interpreter's bundled pip differs from the pip in the build environment"
+}

backends/build_system/awscli_venv.py

@@ -64,6 +64,7 @@ def bootstrap(
 
     def _copy_parent_packages(self):
         for site_package in site.getsitepackages():
+            self._remove_packages_being_replaced(site_package)
             self._utils.copy_directory_contents_into(
                 site_package, self._site_packages()
             )
@@ -75,6 +76,38 @@ def _copy_parent_packages(self):
                     source, os.path.join(self.bin_dir, script)
                 )
 
+    def _remove_packages_being_replaced(self, source_site_packages):
+        # Remove packages in the new venv that we're about to replace.
+        # The venv already contains the user's pip from when it was created,
+        # matching their interpreter's version. We then copy packages in to
+        # the venv during 'bootstrap' above when download_deps=False. The pip
+        # we copy in is our pinned pip if the parent environment has our
+        # bootstrap requirements installed, and otherwise the user's own.
+        # copy_directory_contents_into doesn't delete existing content first,
+        # which can lead to a non-functional mix of files from different pip
+        # versions.
+        destination = self._site_packages()
+        replaced = {
+            self._distribution_key(name)
+            for name in self._utils.listdir(source_site_packages)
+        }
+        for name in self._utils.listdir(destination):
+            if self._distribution_key(name) not in replaced:
+                continue
+            path = os.path.join(destination, name)
+            if self._utils.isdir(path) and not self._utils.islink(path):
+                self._utils.rmtree(path)
+            else:
+                self._utils.remove(path)
+
+    def _distribution_key(self, name):
+        # Metadata directory names include the version, so they need to be
+        # matched on the distribution name alone in order to remove metadata
+        # belonging to a different version of the same distribution.
+        if name.endswith((".dist-info", ".egg-info")):
+            return name.split("-")[0]
+        return name
+
     def _install_requirements(self, requirements_file, cwd=None):
         self._pip_install(
             ["--no-build-isolation", "-r", requirements_file],

backends/build_system/utils.py

@@ -202,6 +202,9 @@ class Utils:
     def isdir(self, path: str) -> bool:
         return os.path.isdir(path)
 
+    def listdir(self, path: str) -> List[str]:
+        return os.listdir(path)
+
     def islink(self, path: str) -> bool:
         return os.path.islink(path)
 

tests/backends/build_system/functional/test_aws_cli_venv.py

@@ -136,6 +136,38 @@ def test_create_windows(self, tmp_path_factory):
         for required_file in required_files:
             assert required_file in venv_dirs
 
+    def _touch(self, path):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.touch()
+
+    def test_remove_packages_being_replaced(self, tmp_path):
+        venv_path = tmp_path / "venv"
+        venv = AwsCliVenv(venv_path)
+        venv.create()
+        site_packages = pathlib.Path(self._site_packages_dir(venv_path))
+
+        # The venv has pip from when it was created, along with its metadata
+        # directory carrying that version.
+        assert (site_packages / "pip").is_dir()
+        assert len(list(site_packages.glob("pip-*.dist-info"))) == 1
+        self._touch(site_packages / "unrelated" / "__init__.py")
+
+        # The directory we are about to copy in has a different version of
+        # pip, whose files do not all have the same names.
+        parent = tmp_path / "parent-site-packages"
+        self._touch(parent / "pip" / "_internal" / "build_env.py")
+        self._touch(parent / "pip-25.3.dist-info" / "METADATA")
+
+        venv._remove_packages_being_replaced(str(parent))
+
+        # Nothing is left of the version being replaced, including metadata
+        # that is named after it.
+        assert not (site_packages / "pip").exists()
+        assert list(site_packages.glob("pip-*.dist-info")) == []
+
+        # Distributions the parent environment does not provide are untouched.
+        assert (site_packages / "unrelated" / "__init__.py").is_file()
+
     @skip_if_windows("Posix bootstrap")
     def test_bootstrap(self, cli_venv, venv_path):
         site_package_path = self._site_packages_dir(venv_path)