add pdf and docx for knowledge bases

Author: Cristhianzl

src/backend/base/langflow/api/utils/kb_helpers.py

@@ -15,6 +15,7 @@
 from langchain_chroma import Chroma
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from lfx.base.data.utils import extract_text_from_bytes
 from lfx.base.models.unified_models import get_embedding_model_options
 from lfx.components.models_and_agents.embedding_model import EmbeddingModelComponent
 from lfx.log import logger
@@ -330,7 +331,7 @@ async def perform_ingestion(
             job_id_str = str(task_job_id)
             for file_name, file_content in files_data:
                 await logger.ainfo("Starting ingestion of %s for %s", file_name, kb_name)
-                content = file_content.decode("utf-8", errors="ignore")
+                content = extract_text_from_bytes(file_name, file_content)
                 if not content.strip():
                     continue
 

src/backend/base/langflow/api/v1/knowledge_bases.py

@@ -12,6 +12,7 @@
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile
 from langchain_chroma import Chroma
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from lfx.base.data.utils import extract_text_from_bytes
 from lfx.log import logger
 
 from langflow.api.utils import CurrentActiveUser
@@ -170,7 +171,7 @@ async def preview_chunks(
             try:
                 file_content = await uploaded_file.read()
                 file_name = uploaded_file.filename or "unknown"
-                text_content = file_content.decode("utf-8", errors="ignore")
+                text_content = extract_text_from_bytes(file_name, file_content)
 
                 if not text_content.strip():
                     file_previews.append(

src/backend/tests/unit/test_extract_text_from_bytes.py

@@ -0,0 +1,176 @@
+from io import BytesIO
+from unittest.mock import MagicMock, patch
+
+import pytest
+from pypdf import PdfWriter
+
+from lfx.base.data.utils import extract_text_from_bytes
+
+
+def _make_blank_pdf(num_pages: int = 1) -> bytes:
+    """Create a valid PDF with blank pages."""
+    writer = PdfWriter()
+    for _ in range(num_pages):
+        writer.add_blank_page(width=612, height=792)
+    buf = BytesIO()
+    writer.write(buf)
+    return buf.getvalue()
+
+
+def _mock_pdf_reader(pages_text: list[str]):
+    """Create a mock PdfReader that returns pages with given text."""
+    mock_reader = MagicMock()
+    mock_pages = []
+    for text in pages_text:
+        page = MagicMock()
+        page.extract_text.return_value = text
+        mock_pages.append(page)
+    mock_reader.pages = mock_pages
+    mock_reader.__enter__ = MagicMock(return_value=mock_reader)
+    mock_reader.__exit__ = MagicMock(return_value=False)
+    return mock_reader
+
+
+class TestExtractTextFromBytesPDF:
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_extract_text_from_valid_pdf(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Hello World"])
+        result = extract_text_from_bytes("document.pdf", _make_blank_pdf())
+        assert "Hello World" in result
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_extract_text_from_multi_page_pdf(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Page one content", "Page two content"])
+        result = extract_text_from_bytes("multi.pdf", _make_blank_pdf(2))
+        assert "Page one content" in result
+        assert "Page two content" in result
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_join_pages_with_double_newline(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["First", "Second"])
+        result = extract_text_from_bytes("test.pdf", _make_blank_pdf(2))
+        assert result == "First\n\nSecond"
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_be_case_insensitive_on_extension(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Test"])
+        result = extract_text_from_bytes("DOC.PDF", _make_blank_pdf())
+        assert "Test" in result
+
+    def test_should_raise_value_error_for_corrupted_pdf(self):
+        with pytest.raises(ValueError, match="Failed to parse PDF file"):
+            extract_text_from_bytes("bad.pdf", b"this is not a pdf")
+
+    def test_should_raise_value_error_for_empty_pdf_bytes(self):
+        with pytest.raises(ValueError, match="Failed to parse PDF file"):
+            extract_text_from_bytes("empty.pdf", b"")
+
+    def test_should_handle_pdf_with_blank_pages(self):
+        result = extract_text_from_bytes("blank.pdf", _make_blank_pdf())
+        assert isinstance(result, str)
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_handle_page_returning_none(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Text"])
+        mock_reader_cls.return_value.pages[0].extract_text.return_value = None
+        mock_reader_cls.return_value.__enter__.return_value = mock_reader_cls.return_value
+        result = extract_text_from_bytes("null_page.pdf", _make_blank_pdf())
+        assert isinstance(result, str)
+
+
+class TestExtractTextFromBytesDOCX:
+    def test_should_extract_text_from_valid_docx(self):
+        from docx import Document
+
+        doc = Document()
+        doc.add_paragraph("Hello from DOCX")
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("file.docx", buf.getvalue())
+        assert "Hello from DOCX" in result
+
+    def test_should_extract_multiple_paragraphs(self):
+        from docx import Document
+
+        doc = Document()
+        doc.add_paragraph("First paragraph")
+        doc.add_paragraph("Second paragraph")
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("file.docx", buf.getvalue())
+        assert "First paragraph" in result
+        assert "Second paragraph" in result
+        assert "\n\n" in result
+
+    def test_should_be_case_insensitive_on_extension(self):
+        from docx import Document
+
+        doc = Document()
+        doc.add_paragraph("Case test")
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("FILE.DOCX", buf.getvalue())
+        assert "Case test" in result
+
+    def test_should_raise_value_error_for_corrupted_docx(self):
+        with pytest.raises(ValueError, match="Failed to parse DOCX file"):
+            extract_text_from_bytes("bad.docx", b"not a valid docx")
+
+    def test_should_raise_value_error_for_empty_docx_bytes(self):
+        with pytest.raises(ValueError, match="Failed to parse DOCX file"):
+            extract_text_from_bytes("empty.docx", b"")
+
+    def test_should_handle_docx_with_no_paragraphs(self):
+        from docx import Document
+
+        doc = Document()
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("empty_doc.docx", buf.getvalue())
+        assert isinstance(result, str)
+
+
+class TestExtractTextFromBytesPlainText:
+    def test_should_decode_utf8_text(self):
+        content = "Hello plain text".encode("utf-8")
+        result = extract_text_from_bytes("readme.txt", content)
+        assert result == "Hello plain text"
+
+    def test_should_handle_non_utf8_gracefully(self):
+        content = b"\xff\xfe\x00\x01 some text"
+        result = extract_text_from_bytes("binary.txt", content)
+        assert isinstance(result, str)
+        assert "some text" in result
+
+    def test_should_handle_empty_content(self):
+        result = extract_text_from_bytes("empty.txt", b"")
+        assert result == ""
+
+    def test_should_handle_csv_as_plain_text(self):
+        content = "col1,col2\nval1,val2".encode("utf-8")
+        result = extract_text_from_bytes("data.csv", content)
+        assert "col1,col2" in result
+
+    def test_should_handle_json_as_plain_text(self):
+        content = '{"key": "value"}'.encode("utf-8")
+        result = extract_text_from_bytes("data.json", content)
+        assert '"key"' in result
+
+    def test_should_handle_unknown_extension_as_plain_text(self):
+        content = "some content".encode("utf-8")
+        result = extract_text_from_bytes("file.xyz", content)
+        assert result == "some content"
+
+    def test_should_handle_file_without_extension(self):
+        content = "no extension".encode("utf-8")
+        result = extract_text_from_bytes("Makefile", content)
+        assert result == "no extension"
+
+    def test_should_preserve_unicode_characters(self):
+        content = "café résumé naïve".encode("utf-8")
+        result = extract_text_from_bytes("unicode.txt", content)
+        assert result == "café résumé naïve"

src/frontend/src/modals/knowledgeBaseUploadModal/constants.ts

@@ -26,6 +26,8 @@ export const KB_INGEST_FORMATS: Record<string, string[]> = {
     "adoc",
     "asciidoc",
     "asc",
+    "pdf",
+    "docx",
   ],
   spreadsheets: ["csv"],
   code: ["py", "js", "ts", "tsx", "sh", "sql"],

src/lfx/src/lfx/base/data/utils.py

@@ -247,9 +247,35 @@ async def read_docx_file_async(file_path: str) -> str:
             Path(temp_path).unlink()
 
 
-def parse_pdf_to_text(file_path: str) -> str:
-    from pypdf import PdfReader
+def extract_text_from_bytes(file_name: str, file_content: bytes) -> str:
+    """Extract text from binary file content based on file extension.
+
+    Supports PDF (via pypdf), DOCX (via python-docx), and plain text files.
+
+    Raises:
+        ValueError: If the file content is corrupted or cannot be parsed.
+    """
+    lower_name = file_name.lower()
+    if lower_name.endswith(".pdf"):
+        try:
+            with BytesIO(file_content) as f, PdfReader(f) as reader:
+                return "\n\n".join(page.extract_text() or "" for page in reader.pages)
+        except Exception as e:
+            msg = f"Failed to parse PDF file '{file_name}': {e}"
+            raise ValueError(msg) from e
+    if lower_name.endswith(".docx"):
+        try:
+            from docx import Document
+
+            doc = Document(BytesIO(file_content))
+            return "\n\n".join(p.text for p in doc.paragraphs)
+        except Exception as e:
+            msg = f"Failed to parse DOCX file '{file_name}': {e}"
+            raise ValueError(msg) from e
+    return file_content.decode("utf-8", errors="ignore")
 
+
+def parse_pdf_to_text(file_path: str) -> str:
     with Path(file_path).open("rb") as f, PdfReader(f) as reader:
         return "\n\n".join([page.extract_text() for page in reader.pages])