feat: implement PEP 832 support for managing virtual environments

.gitignore

@@ -133,6 +133,7 @@ caches/
 __pypackages__
 .pdm.toml
 .pdm-python
+.python-envs
 temp.py
 
 # Pyannotate generated stubs

docs/usage/project.md

@@ -20,10 +20,10 @@ pdm init
 
 ## Choose a Python interpreter
 
-At first, you need to choose a Python interpreter from a list of Python versions installed on your machine. The interpreter path
-will be stored in `.pdm-python` and used by subsequent commands. You can also change it later with [`pdm use`](../reference/cli.md#use).
+At first, you need to choose a Python interpreter from a list of Python versions installed on your machine. Its environment
+will be stored as the last entry in `.python-envs` and used by subsequent commands. You can also change it later with [`pdm use`](../reference/cli.md#use).
 
-Alternatively, you can specify the Python interpreter path via `PDM_PYTHON` environment variable. When it is set, the path saved in `.pdm-python` will be ignored.
+Alternatively, you can specify the Python interpreter path via the `PDM_PYTHON` environment variable. When it is set, the environment selected in `.python-envs` will be ignored.
 
 !!! tip
     Added in 2.23.0.
@@ -180,7 +180,7 @@ Also, when you are executing [`pdm init`](../reference/cli.md#init) or [`pdm ins
 
 ## Working with version control
 
-You **must** commit the `pyproject.toml` file. You **should** commit the `pdm.lock` and `pdm.toml` file. **Do not** commit the `.pdm-python` file.
+You **must** commit the `pyproject.toml` file. You **should** commit the `pdm.lock` and `pdm.toml` file. `.python-envs` should normally remain uncommitted when it contains personal environment locations, but may be committed when all listed paths are stable for the project.
 
 The `pyproject.toml` file must be committed as it contains the project's build metadata and dependencies needed for PDM.
 It is also commonly used by other python tools for configuration. Read more about the `pyproject.toml` file at
@@ -191,7 +191,7 @@ To learn how to update dependencies see [update existing dependencies](./depende
 
 `pdm.toml` contains some project-wide configuration and it may be useful to commit it for sharing.
 
-`.pdm-python` stores the **Python path** used by the **current** project and doesn't need to be shared.
+`.python-envs` lists the project's known environments. The last entry selects the current environment, and paths are relative to the project whenever possible.
 
 ## Show the current Python environment
 

docs/usage/venv.md

@@ -7,7 +7,7 @@ Compared to [PEP 582](https://www.python.org/dev/peps/pep-0582/), virtual enviro
 !!! note "Configure pdm to use virtual environment or PEP 582"
     By default pdm is configured to use virtual environment instead of PEP 582. But this behavior can be changed with `pdm config python.use_venv False` config variable.
 
-**Virtual environments will be used if the project interpreter (the interpreter stored in `.pdm-python`, which can be checked by `pdm info`) is from a virtualenv.**
+**Virtual environments will be used if the environment selected by the last line of `.python-envs` (which can be checked by `pdm info`) is a virtualenv.**
 
 ## Virtualenv auto-creation
 
@@ -42,6 +42,8 @@ pdm venv create --with venv 3.10
 ## The location of virtualenvs
 
 If no `--name` is given, PDM will create the venv in `<project_root>/.venv`. Otherwise, virtualenvs go to the location specified by the `venv.location` configuration.
+
+PDM supports the environment discovery convention defined by [PEP 832](https://peps.python.org/pep-0832/). Environments are recorded in the project's `.python-envs` file, using paths relative to the project whenever possible. The last line is the environment selected by `pdm use`. In PEP 582 mode it points to `__pypackages__/<python_identifier>`; otherwise it points to the virtualenv root. PDM removes entries when their environments are removed or purged. A `.pdm-python` file created by an older PDM version is migrated automatically.
 They are named as `<project_name>-<path_hash>-<name_or_python_version>` to avoid name collision.
 You can disable the in-project virtualenv creation by `pdm config venv.in_project false`. And all virtualenvs will be created under `venv.location`.
 

docs/usage/workspace.md

@@ -41,7 +41,7 @@ pdm new packages/foo
 ```
 
 When a project is initialized as a workspace member, PDM reuses the root project's `requires-python` default,
-does not select a separate Python interpreter, does not write `.pdm-python`, and does not initialize a nested Git repository or `.gitignore`.
+does not select a separate Python environment, does not write `.python-envs`, and does not initialize a nested Git repository or `.gitignore`.
 
 ## Remove a member
 

news/3834.feature.md

@@ -0,0 +1 @@
+Replace `.pdm-python` with the PEP 832 `.python-envs` listing, keeping the selected virtualenv or PEP 582 environment as its last entry.

src/pdm/cli/commands/fix/fixers.py

@@ -43,7 +43,7 @@ class ProjectConfigFixer(BaseFixer):
 
     def get_message(self) -> str:
         return (
-            "[success]python.path[/] config needs to be moved to [info].pdm-python[/] and "
+            "[success]python.path[/] config needs to be moved to [info].python-envs[/] and "
             "[info].pdm.toml[/] needs to be renamed to [info]pdm.toml[/]"
         )
 
@@ -52,16 +52,17 @@ def _fix_gitignore(self) -> None:
         if not gitignore.exists():
             return
         content = gitignore.read_text("utf8")
-        if ".pdm-python" not in content:
-            content = re.sub(r"^\.pdm\.toml$", ".pdm-python", content, flags=re.MULTILINE)
+        if ".python-envs" not in content:
+            content = re.sub(r"^\.pdm-python$", ".python-envs", content, flags=re.MULTILINE)
+            content = re.sub(r"^\.pdm\.toml$", ".python-envs", content, flags=re.MULTILINE)
             gitignore.write_text(content, "utf8")
 
     def fix(self) -> None:
         old_file = self.project.root.joinpath(".pdm.toml")
         config = Config(old_file).self_data
-        if not self.project.root.joinpath(".pdm-python").exists() and config.get("python.path"):
-            self.log("Creating .pdm-python...", verbosity=Verbosity.DETAIL)
-            self.project.root.joinpath(".pdm-python").write_text(config["python.path"])
+        if config.get("python.path"):
+            self.log("Updating .python-envs...", verbosity=Verbosity.DETAIL)
+            self.project._saved_python = config["python.path"]
         self.project.project_config  # access the project config to move the config items
         self.log("Moving .pdm.toml to pdm.toml...", verbosity=Verbosity.DETAIL)
         old_file.unlink()

src/pdm/cli/commands/venv/activate.py

@@ -30,7 +30,7 @@ def handle(self, project: Project, options: Namespace) -> None:
         if options.env:
             venv = get_venv_with_name(project, options.env)
         else:
-            # Use what is saved in .pdm-python
+            # Use the selected environment from .python-envs
             interpreter = project._saved_python
             if not interpreter:
                 project.core.ui.warn(

src/pdm/cli/commands/venv/backends.py

@@ -10,7 +10,7 @@
 from typing import TYPE_CHECKING, Any
 
 from pdm import termui
-from pdm.cli.commands.venv.utils import get_venv_prefix
+from pdm.cli.commands.venv.utils import get_venv_prefix, register_venv
 from pdm.exceptions import PdmUsageError, ProjectError
 
 if TYPE_CHECKING:
@@ -131,6 +131,7 @@ def create(
             )
         self._ensure_clean(location, force)
         self.perform_create(location, args, prompt=prompt)
+        register_venv(self.project, location)
         return location
 
     @abc.abstractmethod

src/pdm/cli/commands/venv/purge.py

@@ -34,7 +34,7 @@ def add_arguments(self, parser: ArgumentParser) -> None:
         )
 
     def handle(self, project: Project, options: Namespace) -> None:
-        from pdm.cli.commands.venv.utils import iter_central_venvs
+        from pdm.cli.commands.venv.utils import iter_central_venvs, unregister_venv
 
         all_central_venvs = list(iter_central_venvs(project))
         if not all_central_venvs:
@@ -61,15 +61,20 @@ def handle(self, project: Project, options: Namespace) -> None:
         elif selection != "none":
             for i, venv in enumerate(all_central_venvs):
                 if i == int(selection):
+                    saved_python = project._saved_python
                     shutil.rmtree(venv[1])
+                    unregister_venv(project, venv[1])
+                    if saved_python and Path(saved_python).parent.parent == venv[1]:
+                        project._python = None
             project.core.ui.echo("Purged successfully!")
 
     def del_all_venvs(self, project: Project) -> None:
-        from pdm.cli.commands.venv.utils import iter_central_venvs
+        from pdm.cli.commands.venv.utils import iter_central_venvs, unregister_venv
 
         saved_python = project._saved_python
         for _, venv in iter_central_venvs(project):
             shutil.rmtree(venv)
+            unregister_venv(project, venv)
             if saved_python and Path(saved_python).parent.parent == venv:
-                project._saved_python = None
+                project._python = None
         project.core.ui.echo("Purged successfully!")

src/pdm/cli/commands/venv/remove.py

@@ -29,13 +29,14 @@ def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument("env", help="The key of the virtualenv")
 
     def handle(self, project: Project, options: Namespace) -> None:
-        from pdm.cli.commands.venv.utils import get_venv_with_name
+        from pdm.cli.commands.venv.utils import get_venv_with_name, unregister_venv
 
         project.core.ui.echo("Virtualenvs created with this project:")
         venv = get_venv_with_name(project, options.env)
         if options.yes or termui.confirm(f"[warning]Will remove: [success]{venv.root}[/], continue?", default=True):
-            shutil.rmtree(venv.root)
             saved_python = project._saved_python
+            shutil.rmtree(venv.root)
+            unregister_venv(project, venv.root)
             if saved_python and Path(saved_python).parent.parent == venv.root:
-                project._saved_python = None
+                project._python = None
             project.core.ui.echo("Removed successfully!")

src/pdm/cli/commands/venv/utils.py

@@ -2,13 +2,15 @@
 
 import base64
 import hashlib
+import os
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from findpython import BaseProvider, PythonVersion
 
 from pdm.exceptions import PdmUsageError
 from pdm.models.venv import VirtualEnv
+from pdm.utils import open_for_write_no_symlink
 
 if TYPE_CHECKING:
     import sys
@@ -43,6 +45,105 @@ def get_venv_prefix(project: Project) -> str:
     return f"{path.name}-{name_hash}-"
 
 
+def _get_python_envs_path(project: Project) -> Path:
+    return project.root / ".python-envs"
+
+
+def _read_python_envs(project: Project) -> list[str]:
+    path = _get_python_envs_path(project)
+    if path.is_symlink():
+        raise PdmUsageError(f"Refusing to read from {path} because it is a symlink.")
+    try:
+        return path.read_text("utf-8").splitlines()
+    except FileNotFoundError:
+        return []
+
+
+def _write_python_envs(project: Project, entries: list[str]) -> None:
+    with open_for_write_no_symlink(_get_python_envs_path(project)) as fp:
+        if entries:
+            fp.write("\n".join(entries) + "\n")
+
+
+def _normalize_env_path(project: Project, path: str | Path) -> str:
+    path = Path(path)
+    if not path.is_absolute():
+        path = project.root / path
+    return os.path.normcase(os.path.abspath(path))
+
+
+def _format_env_path(project: Project, path: Path) -> str:
+    absolute_path = os.path.abspath(path)
+    try:
+        entry = os.path.relpath(absolute_path, project.root)
+    except ValueError:  # Paths on different Windows drives cannot be made relative.
+        entry = absolute_path
+    if "\n" in entry or "\r" in entry:
+        raise PdmUsageError(f"Virtualenv path {entry!r} cannot be represented in .python-envs.")
+    return Path(entry).as_posix()
+
+
+def get_default_env_path(project: Project) -> Path | None:
+    """Return the last environment listed in .python-envs."""
+    entries = _read_python_envs(project)
+    if not entries:
+        return None
+    path = Path(entries[-1])
+    return Path(os.path.abspath(path if path.is_absolute() else project.root / path))
+
+
+def set_default_env(project: Project, path: Path) -> None:
+    """Add an environment and make it the last, selected entry."""
+    target = _normalize_env_path(project, path)
+    entries = _read_python_envs(project)
+    entries = [entry for entry in entries if _normalize_env_path(project, entry) != target]
+    entries.append(_format_env_path(project, path))
+    _write_python_envs(project, entries)
+
+
+def pop_default_env(project: Project) -> None:
+    """Remove the last environment from .python-envs."""
+    python_envs = _get_python_envs_path(project)
+    if not python_envs.exists() and not python_envs.is_symlink():
+        return
+    entries = _read_python_envs(project)
+    if entries:
+        _write_python_envs(project, entries[:-1])
+
+
+def clear_envs(project: Project) -> None:
+    """Clear the PEP 832 environment listing."""
+    python_envs = _get_python_envs_path(project)
+    if python_envs.exists() or python_envs.is_symlink():
+        _read_python_envs(project)
+        _write_python_envs(project, [])
+
+
+def register_venv(project: Project, venv: Path) -> None:
+    """Register a virtualenv without changing the selected environment."""
+    target = _normalize_env_path(project, venv)
+    if target == _normalize_env_path(project, project.root / ".venv"):
+        return
+    entries = _read_python_envs(project)
+    if any(_normalize_env_path(project, entry) == target for entry in entries):
+        return
+    entry = _format_env_path(project, venv)
+    entries.insert(max(len(entries) - 1, 0), entry)
+    _write_python_envs(project, entries)
+
+
+def unregister_venv(project: Project, venv: Path) -> None:
+    """Remove all references to a virtualenv from the PEP 832 listing."""
+    python_envs = _get_python_envs_path(project)
+    if not python_envs.exists() and not python_envs.is_symlink():
+        return
+    target = _normalize_env_path(project, venv)
+    entries = _read_python_envs(project)
+    remaining = [entry for entry in entries if _normalize_env_path(project, entry) != target]
+    if remaining != entries:
+        _write_python_envs(project, remaining)
+
+
 def iter_venvs(project: Project) -> Iterable[tuple[str, VirtualEnv]]:
     """Return an iterable of venv paths associated with the project"""
     in_project_venv = get_in_project_venv(project.root)

src/pdm/cli/options.py

@@ -424,7 +424,7 @@ def no_isolation_option(
     "-I",
     "--ignore-python",
     nargs=0,
-    help="Ignore the Python path saved in .pdm-python. [env var: PDM_IGNORE_SAVED_PYTHON]",
+    help="Ignore the environment selected in .python-envs. [env var: PDM_IGNORE_SAVED_PYTHON]",
 )
 def ignore_python_option(
     project: Project,

src/pdm/cli/templates/default/.gitignore

@@ -110,11 +110,11 @@ ipython_config.py
 
 # pdm
 #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .python-envs.
 #   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
 # pdm.lock
 # pdm.toml
-.pdm-python
+.python-envs
 .pdm-build/
 
 # pixi

src/pdm/cli/templates/minimal/.gitignore

@@ -3,4 +3,4 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 .ruff_cache/
-.pdm-python
+.python-envs