Default to loky executor on all platforms (#497)

Python 3.14 changed the default multiprocessing start method on Linux
from "fork" to "forkserver" (python/cpython#84559), so workers of the
stdlib ProcessPoolExecutor now re-import __main__ instead of inheriting
it. Functions defined interactively (notebooks, executed doc pages) are
pickled by reference and fail to unpickle in the workers with e.g.:

    AttributeError: module '__main__' has no attribute 'sphere'

loky's reusable executor serializes functions by value via cloudpickle
and was already the default on macOS and Windows (and loky is already a
required dependency), so use it on Linux too. This also removes the
now-dead lambda-pickling check in AsyncRunner that only applied when the
default executor was ProcessPoolExecutor.

Author: basnijholt

adaptive/runner.py

@@ -6,8 +6,6 @@
 import functools
 import inspect
 import itertools
-import pickle
-import platform
 import time
 import traceback
 import warnings
@@ -44,16 +42,16 @@
 
 
 # -- Runner definitions
-if platform.system() == "Linux":
-    _default_executor = concurrent.ProcessPoolExecutor  # type: ignore[misc]
-else:
-    # On Windows and MacOS functions, the __main__ module must be
-    # importable by worker subprocesses. This means that
-    # ProcessPoolExecutor will not work in the interactive interpreter.
-    # On Linux the whole process is forked, so the issue does not appear.
-    # See https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor
-    # and https://github.com/python-adaptive/adaptive/issues/301
-    _default_executor = loky.get_reusable_executor  # type: ignore[misc]
+# Functions submitted to a stdlib ProcessPoolExecutor must be importable from
+# `__main__` by the worker, which fails for functions defined interactively
+# (notebooks, doc pages). This used to work on Linux because workers were
+# forked, but Python 3.14 changed the default start method on Linux to
+# "forkserver", which re-imports `__main__` like Windows/macOS always did.
+# loky serializes functions by value with cloudpickle, so it works everywhere.
+# See https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor,
+# https://github.com/python-adaptive/adaptive/issues/301,
+# and https://github.com/python/cpython/issues/84559
+_default_executor = loky.get_reusable_executor
 
 
 class BaseRunner(metaclass=abc.ABCMeta):
@@ -86,8 +84,7 @@ class BaseRunner(metaclass=abc.ABCMeta):
                `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\
                `loky.get_reusable_executor`, optional
         The executor in which to evaluate the function to be learned.
-        If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on
-        Linux, and a `loky.get_reusable_executor` on MacOS and Windows.
+        If not provided, a new `loky.get_reusable_executor` is used.
     ntasks : int, optional
         The number of concurrent function evaluations. Defaults to the number
         of cores available in `executor`.
@@ -373,8 +370,7 @@ class BlockingRunner(BaseRunner):
                `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\
                `loky.get_reusable_executor`, optional
         The executor in which to evaluate the function to be learned.
-        If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on
-        Linux, and a `loky.get_reusable_executor` on MacOS and Windows.
+        If not provided, a new `loky.get_reusable_executor` is used.
     ntasks : int, optional
         The number of concurrent function evaluations. Defaults to the number
         of cores available in `executor`.
@@ -520,8 +516,7 @@ class AsyncRunner(BaseRunner):
                `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\
                `loky.get_reusable_executor`, optional
         The executor in which to evaluate the function to be learned.
-        If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on
-        Linux, and a `loky.get_reusable_executor` on MacOS and Windows.
+        If not provided, a new `loky.get_reusable_executor` is used.
     ntasks : int, optional
         The number of concurrent function evaluations. Defaults to the number
         of cores available in `executor`.
@@ -595,22 +590,6 @@ def __init__(
         retries: int = 0,
         raise_if_retries_exceeded: bool = True,
     ) -> None:
-        if (
-            executor is None
-            and _default_executor is concurrent.ProcessPoolExecutor
-            and not inspect.iscoroutinefunction(learner.function)
-        ):
-            try:
-                pickle.dumps(learner.function)
-            except pickle.PicklingError as e:
-                raise ValueError(
-                    "`learner.function` cannot be pickled (is it a lamdba function?)"
-                    " and therefore does not work with the default executor."
-                    " Either make sure the function is pickleble or use an executor"
-                    " that might work with 'hard to pickle'-functions"
-                    " , e.g. `ipyparallel` with `dill`."
-                ) from e
-
         super().__init__(
             learner,
             goal=goal,

docs/source/tutorial/tutorial.parallelism.md

@@ -16,7 +16,8 @@ Often you will want to evaluate the function on some remote computing resources.
 
 ## `concurrent.futures`
 
-On Unix-like systems by default {class}`adaptive.Runner` creates a {class}`~concurrent.futures.ProcessPoolExecutor`, but you can also pass one explicitly e.g. to limit the number of workers:
+By default {class}`adaptive.Runner` creates a `loky.get_reusable_executor`, which serializes functions by value so it also works with functions defined interactively (e.g. in a notebook).
+You can also pass a {class}`~concurrent.futures.ProcessPoolExecutor` explicitly, e.g. to limit the number of workers, but then the function must be importable from `__main__` by the worker processes:
 
 ```python
 from concurrent.futures import ProcessPoolExecutor