feat(bundles): create lfx-bundles metapackage skeleton

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "lfx-arxiv>=0.1.0",
     "lfx-ibm>=0.1.0",
     "lfx-docling>=0.1.0",
+    "lfx-bundles[all]>=1.0,<2.0",
     # langflow-extensions:bundle-deps-end
 ]
 
@@ -84,6 +85,7 @@ lfx-duckduckgo = { workspace = true }
 lfx-arxiv = { workspace = true }
 lfx-ibm = { workspace = true }
 lfx-docling = { workspace = true }
+lfx-bundles = { workspace = true }
 # langflow-extensions:bundle-sources-end
 torch = { index = "pytorch-cpu" }
 torchvision = { index = "pytorch-cpu" }
@@ -100,6 +102,7 @@ members = [
     "src/bundles/arxiv",
     "src/bundles/ibm",
     "src/bundles/docling",
+    "src/bundles/lfx-bundles",
     # langflow-extensions:bundle-members-end
 ]
 

scripts/ci/update_bundle_versions.py

@@ -65,6 +65,7 @@ def rename_bundle_pyproject(pyproject_path: Path, lfx_version: str, dev_n: str)
     - ``[project] version``      → ``<base_version>.dev<N>``
     - entry-point key            → ``<base_name>-nightly``
     - ``"lfx>=...,<..."`` dep    → ``"lfx-nightly==<lfx_version>"``
+    - ``"<base_name>[extras]"`` self-refs → ``"<base_name>-nightly[extras]"``
 
     Returns ``(base_name, nightly_name, nightly_version)`` so the caller can
     update the root pyproject. Returns ``None`` if the file has no
@@ -96,6 +97,15 @@ def rename_bundle_pyproject(pyproject_path: Path, lfx_version: str, dev_n: str)
     # Rewrite the lfx dep regardless of which form it's in.
     content = _LFX_DEP_PATTERN.sub(f'"lfx-nightly=={lfx_version}"', content)
 
+    # Self-referencing extras must follow the distribution rename (e.g. the
+    # lfx-bundles metapackage's generated `all` extra lists
+    # "lfx-bundles[<provider>]" members). Left unrenamed, the nightly package
+    # would resolve the stable distribution from PyPI -- which ships the same
+    # import package and collides at install time -- or fail to resolve
+    # entirely while the stable name is unpublished. Idempotent on re-runs.
+    self_ref_pattern = re.compile(rf'"{re.escape(base_name)}(?:-nightly)?(\[[^\]]*\])')
+    content = self_ref_pattern.sub(rf'"{nightly_name}\g<1>', content)
+
     pyproject_path.write_text(content, encoding="utf-8")
     return base_name, nightly_name, nightly_version
 
@@ -108,22 +118,26 @@ def update_root_pyproject_for_bundle(
 ) -> None:
     """Update root ``pyproject.toml`` to reference the nightly bundle.
 
-    - dependency line  ``"<base_name>[..]"`` → ``"<nightly_name>==<version>"``
+    - dependency line  ``"<base_name>[extras]<spec>"`` → ``"<nightly_name>[extras]==<version>"``
+      (extras carried over verbatim, e.g. ``lfx-bundles[all]`` / ``lfx-docling[local]``)
     - uv.sources entry ``<base_name> = { workspace = true }`` → ``<nightly_name> = ...``
 
     Idempotent: also matches the already-nightly form.
     """
     content = root_pyproject.read_text(encoding="utf-8")
 
-    # Dependency in [project.dependencies] (any PEP 440 specifier or range form).
+    # Dependency in [project.dependencies] or [project.optional-dependencies]
+    # (any PEP 440 specifier or range form, with an optional [extras] group
+    # between the name and the specifier).
     dep_pattern = re.compile(
         rf'"{re.escape(base_name)}(?:-nightly)?'
+        r"(\[[^\]]*\])?"
         r"(?:"
         r"(?:~=|==|>=)[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*"
         r"(?:,\s*<[\d.]+(?:\.(?:post|dev|a|b|rc)\d+)*)?"
         r')"'
     )
-    content = dep_pattern.sub(f'"{nightly_name}=={nightly_version}"', content)
+    content = dep_pattern.sub(rf'"{nightly_name}\g<1>=={nightly_version}"', content)
 
     # uv.sources entry — only the workspace = true form is used by bundles today.
     source_pattern = re.compile(

src/backend/tests/unit/test_update_bundle_versions.py

@@ -0,0 +1,145 @@
+"""Tests for ``scripts/ci/update_bundle_versions.py``.
+
+The nightly build calls this script to rename every ``src/bundles/*``
+package to its ``-nightly`` counterpart and re-point the root
+``pyproject.toml`` at the renamed distributions. These tests exercise the
+real script module so regressions in the rename/dep regexes are caught
+without running a nightly. The extras-suffix cases exist because the
+``lfx-bundles`` metapackage is referenced as ``lfx-bundles[all]`` (and
+docling as ``lfx-docling[local]`` etc.) -- a dep regex that cannot see
+through ``[extras]`` leaves the root pointing at the stable distribution
+while the workspace member is renamed, which breaks the nightly resolve.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+_SCRIPT = REPO_ROOT / "scripts" / "ci" / "update_bundle_versions.py"
+
+
+def _load_module():
+    spec = importlib.util.spec_from_file_location("update_bundle_versions", _SCRIPT)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["update_bundle_versions"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+mod = _load_module()
+
+
+_ROOT_PYPROJECT = """\
+[project]
+name = "langflow"
+version = "1.11.0"
+dependencies = [
+    "lfx-bundles[all]>=1.0,<2.0",
+    "lfx-duckduckgo>=0.1.0,<1.0.0",
+]
+
+[project.optional-dependencies]
+docling = [
+    "lfx-docling[local]>=0.1.0",
+]
+
+[tool.uv.sources]
+lfx-bundles = { workspace = true }
+lfx-duckduckgo = { workspace = true }
+lfx-docling = { workspace = true }
+"""
+
+_METAPACKAGE_PYPROJECT = """\
+[project]
+name = "lfx-bundles"
+version = "1.0.0"
+dependencies = [
+    "lfx>=1.11.0,<2.0.0",
+]
+
+[project.optional-dependencies]
+aiml = ["openai>=1.68.2,<3.0.0"]
+tavily = []
+all = [
+    "lfx-bundles[aiml]",
+    "lfx-bundles[tavily]",
+]
+
+[project.entry-points."lfx.bundles"]
+lfx_bundles = "lfx_bundles"
+"""
+
+
+class TestRenameBundlePyproject:
+    def test_metapackage_self_ref_extras_follow_the_rename(self, tmp_path):
+        pyproject = tmp_path / "pyproject.toml"
+        pyproject.write_text(_METAPACKAGE_PYPROJECT, encoding="utf-8")
+
+        renamed = mod.rename_bundle_pyproject(pyproject, "1.11.0.dev38", "38")
+
+        assert renamed == ("lfx-bundles", "lfx-bundles-nightly", "1.0.0.dev38")
+        content = pyproject.read_text(encoding="utf-8")
+        assert 'name = "lfx-bundles-nightly"' in content
+        assert 'version = "1.0.0.dev38"' in content
+        assert '"lfx-nightly==1.11.0.dev38"' in content
+        assert '"lfx-bundles-nightly[aiml]"' in content
+        assert '"lfx-bundles-nightly[tavily]"' in content
+        # No stable self-ref left behind to pull the stable dist from PyPI.
+        assert '"lfx-bundles[' not in content
+
+    def test_rename_is_idempotent(self, tmp_path):
+        pyproject = tmp_path / "pyproject.toml"
+        pyproject.write_text(_METAPACKAGE_PYPROJECT, encoding="utf-8")
+
+        first = mod.rename_bundle_pyproject(pyproject, "1.11.0.dev38", "38")
+        after_first = pyproject.read_text(encoding="utf-8")
+        second = mod.rename_bundle_pyproject(pyproject, "1.11.0.dev38", "38")
+
+        assert first == second
+        assert pyproject.read_text(encoding="utf-8") == after_first
+
+
+class TestUpdateRootPyprojectForBundle:
+    def test_extras_suffixed_main_dep_is_rewritten_with_extras_preserved(self, tmp_path):
+        root = tmp_path / "pyproject.toml"
+        root.write_text(_ROOT_PYPROJECT, encoding="utf-8")
+
+        mod.update_root_pyproject_for_bundle(root, "lfx-bundles", "lfx-bundles-nightly", "1.0.0.dev38")
+
+        content = root.read_text(encoding="utf-8")
+        assert '"lfx-bundles-nightly[all]==1.0.0.dev38"' in content
+        assert '"lfx-bundles[all]' not in content
+        assert "lfx-bundles-nightly = { workspace = true }" in content
+
+    def test_extras_suffixed_optional_dep_is_rewritten(self, tmp_path):
+        root = tmp_path / "pyproject.toml"
+        root.write_text(_ROOT_PYPROJECT, encoding="utf-8")
+
+        mod.update_root_pyproject_for_bundle(root, "lfx-docling", "lfx-docling-nightly", "0.1.5.dev38")
+
+        content = root.read_text(encoding="utf-8")
+        assert '"lfx-docling-nightly[local]==0.1.5.dev38"' in content
+        assert '"lfx-docling[local]' not in content
+
+    def test_plain_dep_keeps_working(self, tmp_path):
+        root = tmp_path / "pyproject.toml"
+        root.write_text(_ROOT_PYPROJECT, encoding="utf-8")
+
+        mod.update_root_pyproject_for_bundle(root, "lfx-duckduckgo", "lfx-duckduckgo-nightly", "0.1.2.dev38")
+
+        content = root.read_text(encoding="utf-8")
+        assert '"lfx-duckduckgo-nightly==0.1.2.dev38"' in content
+        assert "lfx-duckduckgo-nightly = { workspace = true }" in content
+
+    def test_root_update_is_idempotent(self, tmp_path):
+        root = tmp_path / "pyproject.toml"
+        root.write_text(_ROOT_PYPROJECT, encoding="utf-8")
+
+        mod.update_root_pyproject_for_bundle(root, "lfx-bundles", "lfx-bundles-nightly", "1.0.0.dev38")
+        after_first = root.read_text(encoding="utf-8")
+        mod.update_root_pyproject_for_bundle(root, "lfx-bundles", "lfx-bundles-nightly", "1.0.0.dev38")
+
+        assert root.read_text(encoding="utf-8") == after_first

src/bundles/lfx-bundles/README.md

@@ -0,0 +1,55 @@
+# lfx-bundles
+
+The long tail of Langflow's provider components as a **single manifest-less
+metapackage**, modeled on `langchain-community`. This is the destination for
+every vendor/third-party provider that does not warrant its own standalone
+distribution; the curated partner providers (OpenAI, Anthropic, AWS,
+DataStax, Cohere) ship as separate `lfx-<provider>` packages instead.
+
+## How it works
+
+`lfx-bundles` declares the `lfx.bundles` entry point:
+
+```toml
+[project.entry-points."lfx.bundles"]
+lfx_bundles = "lfx_bundles"
+```
+
+At startup, lfx resolves this package and **folder-walks its immediate
+subdirectories**. Each subdirectory is one bundle, registered at the
+`@official` slot under its directory name — no `extension.json`, no per-provider
+manifest. Adding a provider is just adding a folder.
+
+```
+src/lfx_bundles/
+├── __init__.py        # bare namespace marker
+├── <provider>/        # one bundle, e.g. tavily/, pinecone/, ...
+│   └── *.py           # Component subclasses
+└── ...
+```
+
+A component's identity is its **bundle name** (`ext:<provider>:<Class>@official`),
+which is stable whether the provider ships here or graduates to a standalone
+`lfx-<provider>` package. Because a manifest-shipping package always shadows the
+manifest-less metapackage, a provider can graduate with **no lockstep release**.
+
+## Installing
+
+```bash
+pip install langflow                 # everything (langflow pins lfx-bundles[all])
+pip install lfx                      # engine only, no bundles
+pip install "lfx[bundles]"           # engine + this metapackage (deployment footnote)
+pip install "lfx-bundles[tavily]"    # engine + one provider's deps
+```
+
+`lfx-bundles` itself depends only on `lfx`. Each provider's third-party SDKs are
+**optional extras** (PEP 685-normalized keys, e.g. `lfx-bundles[google-genai]`);
+the generated `all` extra pulls every provider's deps and is what `langflow`
+depends on so `pip install langflow` is unchanged.
+
+## Adding a provider
+
+Providers are moved here by `scripts/migrate/consolidate_bundles.py`, which also
+maintains the per-provider extras and the generated `all` aggregate. **Do not**
+hand-edit the extras block in `pyproject.toml`. Provider folder names must be
+lowercase snake_case (`a-z`, `0-9`, `_`, 2–64 chars).

src/bundles/lfx-bundles/pyproject.toml

@@ -0,0 +1,57 @@
+[project]
+name = "lfx-bundles"
+version = "1.0.0"
+description = "Langflow's long-tail provider bundles as a single manifest-less metapackage (the langchain-community model)."
+readme = "README.md"
+requires-python = ">=3.10,<3.15"
+license = { text = "MIT" }
+authors = [
+    { name = "Langflow", email = "contact@langflow.org" },
+]
+keywords = ["langflow", "lfx", "extension", "bundle", "providers"]
+
+# Runtime: only lfx (the BUNDLE_API surface).  Each provider's third-party SDK
+# is an optional extra (see [project.optional-dependencies]); installing
+# lfx-bundles bare gives the provider *code* but defers each provider's SDK to
+# its extra, so a user opts into exactly the providers they need.  The
+# generated ``all`` extra pulls every provider's deps and is what ``langflow``
+# depends on (``lfx-bundles[all]``) so ``pip install langflow`` stays
+# functionally identical to today.
+dependencies = [
+    "lfx>=1.11.0.dev0,<2.0.0",
+]
+
+[project.optional-dependencies]
+# Per-provider extras + the ``all`` aggregate are populated by the bulk move
+# (scripts/migrate/consolidate_bundles.py) as the long-tail providers land
+# here.  Extra keys are PEP 685-normalized (lowercase, hyphen-separated).
+# ``all`` is GENERATED from the per-provider keys -- never hand-edit it.
+# Empty until the first provider tranche moves in.
+all = []
+
+[project.urls]
+Homepage = "https://github.com/langflow-ai/langflow"
+Documentation = "https://docs.langflow.org/extensions"
+Repository = "https://github.com/langflow-ai/langflow"
+
+# Manifest-less discovery via the ``lfx.bundles`` entry-point group (NOT
+# ``langflow.extensions``).  The loader (lfx.extension.loader._bundles_root)
+# resolves this package with find_spec and folder-walks its immediate
+# subdirectories -- each is one bundle at the @official slot, named after the
+# directory.  No extension.json; exempt from ``lfx extension validate``.
+[project.entry-points."lfx.bundles"]
+lfx_bundles = "lfx_bundles"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lfx_bundles"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/lfx_bundles",
+    "README.md",
+    "pyproject.toml",
+]

src/bundles/lfx-bundles/src/lfx_bundles/__init__.py

@@ -0,0 +1,16 @@
+"""lfx-bundles: the manifest-less metapackage of Langflow's long-tail providers.
+
+This package is a bare namespace marker.  Each immediate subdirectory is one
+provider bundle, discovered at runtime by lfx's ``lfx.bundles`` entry-point
+folder-walk (``lfx.extension.loader._bundles_root``) and registered at the
+``@official`` slot under its directory name.  There are intentionally no
+re-exports here and no ``extension.json`` -- providers are added as folders,
+the langchain-community way.
+
+Provider folders are lowercase snake_case (``BUNDLE_NAME_RE``); a component's
+identity is its bundle name (``ext:<provider>:<Class>@official``), stable
+whether the provider ships here or in a graduated ``lfx-<provider>`` package.
+
+Providers are added by ``scripts/migrate/consolidate_bundles.py``, never by
+hand.
+"""