Bundle Punkt models for offline sentence tokenization

MANIFEST.in

@@ -3,6 +3,8 @@ prune examples
 prune scripts
 prune tests
 
+graft src/pipecat/utils/text/data
+
 # Ship the CLI scaffolding templates in the sdist (jinja + client trees, incl. dotfiles).
 graft src/pipecat/cli/templates
 

changelog/5726.fixed.md

@@ -0,0 +1 @@
+- Sentence tokenization uses bundled Punkt models without runtime downloads or an external NLTK data installation, while preserving lazy imports and background warm-up.

pyproject.toml

@@ -202,6 +202,7 @@ where = ["src"]
 
 [tool.setuptools.package-data]
 "pipecat" = ["py.typed"]
+"pipecat.utils.text.data" = ["punkt_tab.zip", "README.md"]
 "pipecat.audio.dtmf" = [
     "src/pipecat/audio/dtmf/dtmf-0.wav",
     "src/pipecat/audio/dtmf/dtmf-1.wav",

src/pipecat/utils/string.py

@@ -16,20 +16,18 @@
     See: https://www.nltk.org/
     Source: https://www.nltk.org/api/nltk.tokenize.punkt.html
 
-    The tokenizer and its ``punkt_tab`` data load on first use, and the data is
-    downloaded if it isn't already present. Deployments that build their own
-    image should bundle it at build time (``python -m nltk.downloader
-    punkt_tab``) or point ``NLTK_DATA`` at a directory that already has it, so
-    that a slow or unavailable network can't delay the first bot turn.
+    The tokenizer loads on first use from Pipecat's bundled ``punkt_tab`` data.
+    It requires no network access or external NLTK data directory.
 """
 
 import re
 import threading
 from collections.abc import Callable, Sequence
 from dataclasses import dataclass
 from functools import cache
-
-from loguru import logger
+from importlib.resources import files
+from io import TextIOWrapper
+from zipfile import ZipFile
 
 _load_lock = threading.Lock()
 
@@ -44,30 +42,32 @@ def _sent_tokenizer() -> Callable[[str], list[str]]:
     never tokenize.
 
     A caller arriving while the pipeline's background warming is still loading
-    waits on the lock rather than loading alongside it, so the one-time
-    ``punkt_tab`` download cannot run twice at once. The cache keeps the lock
-    off the path once the tokenizer is loaded.
+    waits on the lock rather than loading alongside it. The cache keeps the
+    lock off the path once the tokenizer is loaded. Model parameters are read
+    directly from the packaged archive without modifying NLTK's data paths.
     """
     with _load_lock:
-        import nltk
-        from nltk.tokenize import sent_tokenize
-
-        try:
-            nltk.data.find("tokenizers/punkt_tab")
-        except LookupError:
-            try:
-                nltk.download("punkt_tab", quiet=True)
-            except (OSError, PermissionError) as e:
-                logger.error(
-                    f"Failed to download NLTK 'punkt_tab' tokenizer data: {e}. "
-                    "This data is required for sentence tokenization features. "
-                    "The download failed due to filesystem permissions. "
-                    "To resolve: pre-install the data in a location with appropriate read "
-                    "permissions, or set the NLTK_DATA environment variable to point to a "
-                    "writable directory. See https://www.nltk.org/data.html for more information."
-                )
-
-        return sent_tokenize
+        return _load_punkt_tokenizer("english")
+
+
+def _load_punkt_tokenizer(language: str) -> Callable[[str], list[str]]:
+    """Read a bundled Punkt model into memory without NLTK filesystem lookups."""
+    from nltk.tabdata import PunktDecoder
+    from nltk.tokenize.punkt import PunktParameters, PunktSentenceTokenizer
+
+    params = PunktParameters()
+    decoder = PunktDecoder()
+    resource = files("pipecat.utils.text.data").joinpath("punkt_tab.zip")
+    with resource.open("rb") as stream, ZipFile(stream) as archive:
+        with archive.open(f"punkt_tab/{language}/collocations.tab") as data:
+            params.collocations = set(decoder.tab2tups(TextIOWrapper(data, encoding="utf-8")))
+        with archive.open(f"punkt_tab/{language}/sent_starters.txt") as data:
+            params.sent_starters = decoder.txt2set(TextIOWrapper(data, encoding="utf-8"))
+        with archive.open(f"punkt_tab/{language}/abbrev_types.txt") as data:
+            params.abbrev_types = decoder.txt2set(TextIOWrapper(data, encoding="utf-8"))
+        with archive.open(f"punkt_tab/{language}/ortho_context.tab") as data:
+            params.ortho_context = decoder.tab2intdict(TextIOWrapper(data, encoding="utf-8"))
+    return PunktSentenceTokenizer(params).tokenize
 
 
 SENTENCE_ENDING_PUNCTUATION: frozenset[str] = frozenset(

src/pipecat/utils/text/data/README.md

@@ -0,0 +1,19 @@
+# Punkt tokenizer models
+
+`punkt_tab.zip` is an unmodified copy of the full NLTK Punkt model package.
+Its embedded `punkt_tab/README` records the model authors and training sources.
+
+- Source: https://github.com/nltk/nltk_data/blob/4f15a3d89eefe9748ec1c05be495d91289197155/packages/tokenizers/punkt_tab.zip
+- Revision date: February 17, 2025
+- SHA-256: `e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106`
+
+NLTK's code license does not establish the license of these model files.
+Upstream explicitly identifies Punkt's data license as unclear:
+https://github.com/nltk/nltk_data/blob/gh-pages/LICENSE-OVERVIEW.md.
+Redistribution clearance is unresolved; release of this bundle requires a
+licensing determination.
+
+To update, obtain the archive from a pinned upstream revision, verify its
+checksum, update this record, and run the sentence-tokenization tests for all
+bundled languages. The loader reads the models directly from the archive;
+no runtime download or persistent extraction is needed.

src/pipecat/utils/text/data/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+"""Packaged sentence tokenizer models."""

tests/test_bundled_tokenizer.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+"""Offline loading and isolation of the packaged Punkt models."""
+
+import subprocess
+import sys
+from importlib.resources import as_file, files
+from zipfile import ZipFile
+
+import nltk
+
+from pipecat.utils.string import _load_punkt_tokenizer, _sent_tokenizer, match_endofsentence
+
+
+def test_sentence_detection_without_external_data(monkeypatch):
+    _sent_tokenizer.cache_clear()
+    monkeypatch.setattr(nltk.data, "path", [])
+
+    def download(*args, **kwargs):
+        raise AssertionError("Sentence detection must not download data")
+
+    monkeypatch.setattr(nltk, "download", download)
+    try:
+        assert match_endofsentence("For Mr. Smith. Next") == len("For Mr. Smith.")
+        assert match_endofsentence("こんにちは。次") == len("こんにちは。")
+        assert nltk.data.path == []
+    finally:
+        _sent_tokenizer.cache_clear()
+
+
+def test_all_bundled_models_load():
+    with as_file(files("pipecat.utils.text.data").joinpath("punkt_tab.zip")) as archive:
+        with ZipFile(archive) as zipped:
+            languages = {
+                name.split("/")[1]
+                for name in zipped.namelist()
+                if name.endswith("/ortho_context.tab")
+            }
+        assert len(languages) == 19
+        for language in languages:
+            assert _load_punkt_tokenizer(language)("Hello world.") == ["Hello world."]
+
+
+def test_import_does_not_load_nltk():
+    subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "import pipecat.utils.string; import sys; assert 'nltk' not in sys.modules",
+        ],
+        check=True,
+    )