Merge branch 'release-1.8.0' into fix-knowledge-global-llm

Author: erichare

src/backend/base/langflow/api/utils/kb_helpers.py

@@ -15,6 +15,7 @@
 from langchain_chroma import Chroma
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from lfx.base.data.utils import extract_text_from_bytes
 from lfx.base.models.unified_models import get_embedding_model_options
 from lfx.components.models_and_agents.embedding_model import EmbeddingModelComponent
 from lfx.log import logger
@@ -330,7 +331,7 @@ async def perform_ingestion(
             job_id_str = str(task_job_id)
             for file_name, file_content in files_data:
                 await logger.ainfo("Starting ingestion of %s for %s", file_name, kb_name)
-                content = file_content.decode("utf-8", errors="ignore")
+                content = extract_text_from_bytes(file_name, file_content)
                 if not content.strip():
                     continue
 

src/backend/base/langflow/api/v1/knowledge_bases.py

@@ -12,6 +12,7 @@
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile
 from langchain_chroma import Chroma
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from lfx.base.data.utils import extract_text_from_bytes
 from lfx.log import logger
 
 from langflow.api.utils import CurrentActiveUser
@@ -170,7 +171,7 @@ async def preview_chunks(
             try:
                 file_content = await uploaded_file.read()
                 file_name = uploaded_file.filename or "unknown"
-                text_content = file_content.decode("utf-8", errors="ignore")
+                text_content = extract_text_from_bytes(file_name, file_content)
 
                 if not text_content.strip():
                     file_previews.append(

src/backend/tests/unit/test_extract_text_from_bytes.py

@@ -0,0 +1,175 @@
+from io import BytesIO
+from unittest.mock import MagicMock, patch
+
+import pytest
+from lfx.base.data.utils import extract_text_from_bytes
+from pypdf import PdfWriter
+
+
+def _make_blank_pdf(num_pages: int = 1) -> bytes:
+    """Create a valid PDF with blank pages."""
+    writer = PdfWriter()
+    for _ in range(num_pages):
+        writer.add_blank_page(width=612, height=792)
+    buf = BytesIO()
+    writer.write(buf)
+    return buf.getvalue()
+
+
+def _mock_pdf_reader(pages_text: list[str]):
+    """Create a mock PdfReader that returns pages with given text."""
+    mock_reader = MagicMock()
+    mock_pages = []
+    for text in pages_text:
+        page = MagicMock()
+        page.extract_text.return_value = text
+        mock_pages.append(page)
+    mock_reader.pages = mock_pages
+    mock_reader.__enter__ = MagicMock(return_value=mock_reader)
+    mock_reader.__exit__ = MagicMock(return_value=False)
+    return mock_reader
+
+
+class TestExtractTextFromBytesPDF:
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_extract_text_from_valid_pdf(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Hello World"])
+        result = extract_text_from_bytes("document.pdf", _make_blank_pdf())
+        assert "Hello World" in result
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_extract_text_from_multi_page_pdf(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Page one content", "Page two content"])
+        result = extract_text_from_bytes("multi.pdf", _make_blank_pdf(2))
+        assert "Page one content" in result
+        assert "Page two content" in result
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_join_pages_with_double_newline(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["First", "Second"])
+        result = extract_text_from_bytes("test.pdf", _make_blank_pdf(2))
+        assert result == "First\n\nSecond"
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_be_case_insensitive_on_extension(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Test"])
+        result = extract_text_from_bytes("DOC.PDF", _make_blank_pdf())
+        assert "Test" in result
+
+    def test_should_raise_value_error_for_corrupted_pdf(self):
+        with pytest.raises(ValueError, match="Failed to parse PDF file"):
+            extract_text_from_bytes("bad.pdf", b"this is not a pdf")
+
+    def test_should_raise_value_error_for_empty_pdf_bytes(self):
+        with pytest.raises(ValueError, match="Failed to parse PDF file"):
+            extract_text_from_bytes("empty.pdf", b"")
+
+    def test_should_handle_pdf_with_blank_pages(self):
+        result = extract_text_from_bytes("blank.pdf", _make_blank_pdf())
+        assert isinstance(result, str)
+
+    @patch("lfx.base.data.utils.PdfReader")
+    def test_should_handle_page_returning_none(self, mock_reader_cls):
+        mock_reader_cls.return_value = _mock_pdf_reader(["Text"])
+        mock_reader_cls.return_value.pages[0].extract_text.return_value = None
+        mock_reader_cls.return_value.__enter__.return_value = mock_reader_cls.return_value
+        result = extract_text_from_bytes("null_page.pdf", _make_blank_pdf())
+        assert isinstance(result, str)
+
+
+class TestExtractTextFromBytesDOCX:
+    def test_should_extract_text_from_valid_docx(self):
+        from docx import Document
+
+        doc = Document()
+        doc.add_paragraph("Hello from DOCX")
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("file.docx", buf.getvalue())
+        assert "Hello from DOCX" in result
+
+    def test_should_extract_multiple_paragraphs(self):
+        from docx import Document
+
+        doc = Document()
+        doc.add_paragraph("First paragraph")
+        doc.add_paragraph("Second paragraph")
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("file.docx", buf.getvalue())
+        assert "First paragraph" in result
+        assert "Second paragraph" in result
+        assert "\n\n" in result
+
+    def test_should_be_case_insensitive_on_extension(self):
+        from docx import Document
+
+        doc = Document()
+        doc.add_paragraph("Case test")
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("FILE.DOCX", buf.getvalue())
+        assert "Case test" in result
+
+    def test_should_raise_value_error_for_corrupted_docx(self):
+        with pytest.raises(ValueError, match="Failed to parse DOCX file"):
+            extract_text_from_bytes("bad.docx", b"not a valid docx")
+
+    def test_should_raise_value_error_for_empty_docx_bytes(self):
+        with pytest.raises(ValueError, match="Failed to parse DOCX file"):
+            extract_text_from_bytes("empty.docx", b"")
+
+    def test_should_handle_docx_with_no_paragraphs(self):
+        from docx import Document
+
+        doc = Document()
+        buf = BytesIO()
+        doc.save(buf)
+
+        result = extract_text_from_bytes("empty_doc.docx", buf.getvalue())
+        assert isinstance(result, str)
+
+
+class TestExtractTextFromBytesPlainText:
+    def test_should_decode_utf8_text(self):
+        content = b"Hello plain text"
+        result = extract_text_from_bytes("readme.txt", content)
+        assert result == "Hello plain text"
+
+    def test_should_handle_non_utf8_gracefully(self):
+        content = b"\xff\xfe\x00\x01 some text"
+        result = extract_text_from_bytes("binary.txt", content)
+        assert isinstance(result, str)
+        assert "some text" in result
+
+    def test_should_handle_empty_content(self):
+        result = extract_text_from_bytes("empty.txt", b"")
+        assert result == ""
+
+    def test_should_handle_csv_as_plain_text(self):
+        content = b"col1,col2\nval1,val2"
+        result = extract_text_from_bytes("data.csv", content)
+        assert "col1,col2" in result
+
+    def test_should_handle_json_as_plain_text(self):
+        content = b'{"key": "value"}'
+        result = extract_text_from_bytes("data.json", content)
+        assert '"key"' in result
+
+    def test_should_handle_unknown_extension_as_plain_text(self):
+        content = b"some content"
+        result = extract_text_from_bytes("file.xyz", content)
+        assert result == "some content"
+
+    def test_should_handle_file_without_extension(self):
+        content = b"no extension"
+        result = extract_text_from_bytes("Makefile", content)
+        assert result == "no extension"
+
+    def test_should_preserve_unicode_characters(self):
+        content = "café résumé naïve".encode()
+        result = extract_text_from_bytes("unicode.txt", content)
+        assert result == "café résumé naïve"

src/frontend/src/components/core/parameterRenderComponent/components/modelInputComponent/__tests__/ModelInputComponent.test.tsx

@@ -16,6 +16,20 @@ jest.mock("@/stores/alertStore", () => ({
   }),
 }));
 
+// Mock useRefreshModelInputs with controllable promise
+let mockRefreshResolve: () => void;
+const mockRefreshAllModelInputs = jest.fn(
+  () =>
+    new Promise<void>((resolve) => {
+      mockRefreshResolve = resolve;
+    }),
+);
+jest.mock("@/hooks/use-refresh-model-inputs", () => ({
+  useRefreshModelInputs: () => ({
+    refreshAllModelInputs: mockRefreshAllModelInputs,
+  }),
+}));
+
 jest.mock("@/stores/flowStore", () => ({
   __esModule: true,
   default: {
@@ -310,6 +324,92 @@ describe("ModelInputComponent", () => {
     });
   });
 
+  describe("Refresh List", () => {
+    it("should close popover before entering loading state when refresh is clicked", async () => {
+      const user = userEvent.setup();
+      renderWithQueryClient(<ModelInputComponent {...defaultProps} />);
+
+      const trigger = screen.getByRole("combobox");
+      await user.click(trigger);
+
+      await waitFor(() => {
+        expect(screen.getByTestId("refresh-model-list")).toBeInTheDocument();
+      });
+
+      const refreshButton = screen.getByTestId("refresh-model-list");
+      await user.click(refreshButton);
+
+      await waitFor(() => {
+        expect(screen.getByText("Loading models")).toBeInTheDocument();
+      });
+
+      mockRefreshResolve();
+
+      await waitFor(() => {
+        expect(screen.getByRole("combobox")).toBeInTheDocument();
+      });
+
+      // Popover must be closed after refresh to prevent width measurement glitch
+      expect(screen.queryByTestId("gpt-4-option")).not.toBeInTheDocument();
+      expect(screen.queryByText("OpenAI")).not.toBeInTheDocument();
+    });
+
+    it("should not crash when component renders without popover open during refresh", () => {
+      mockRefreshAllModelInputs.mockImplementationOnce(() => Promise.resolve());
+      renderWithQueryClient(<ModelInputComponent {...defaultProps} />);
+
+      expect(screen.getByRole("combobox")).toBeInTheDocument();
+      expect(screen.queryByTestId("gpt-4-option")).not.toBeInTheDocument();
+    });
+
+    it("should call refresh with silent flag exactly once per click", async () => {
+      const user = userEvent.setup();
+      renderWithQueryClient(<ModelInputComponent {...defaultProps} />);
+
+      const trigger = screen.getByRole("combobox");
+      await user.click(trigger);
+
+      await waitFor(() => {
+        expect(screen.getByTestId("refresh-model-list")).toBeInTheDocument();
+      });
+
+      const refreshButton = screen.getByTestId("refresh-model-list");
+      await user.click(refreshButton);
+
+      expect(mockRefreshAllModelInputs).toHaveBeenCalledTimes(1);
+      expect(mockRefreshAllModelInputs).toHaveBeenCalledWith({ silent: true });
+
+      mockRefreshResolve();
+    });
+
+    it("should recover to normal state when refresh rejects", async () => {
+      // handleRefreshButtonPress uses try/finally, so refreshOptions resets even on error
+      mockRefreshAllModelInputs.mockImplementationOnce(() =>
+        Promise.reject(new Error("Network error")),
+      );
+
+      const user = userEvent.setup();
+      renderWithQueryClient(<ModelInputComponent {...defaultProps} />);
+
+      const trigger = screen.getByRole("combobox");
+      await user.click(trigger);
+
+      await waitFor(() => {
+        expect(screen.getByTestId("refresh-model-list")).toBeInTheDocument();
+      });
+
+      const refreshButton = screen.getByTestId("refresh-model-list");
+      await user.click(refreshButton);
+
+      // finally block sets refreshOptions=false, restoring the combobox
+      await waitFor(() => {
+        expect(screen.getByRole("combobox")).toBeInTheDocument();
+      });
+
+      expect(screen.queryByText("Loading models")).not.toBeInTheDocument();
+    });
+  });
+
   describe("Edge Cases", () => {
     it("should filter out disabled provider models from grouped options", () => {
       const optionsWithDisabled: ModelOption[] = [

src/frontend/src/components/core/parameterRenderComponent/components/modelInputComponent/index.tsx

@@ -185,9 +185,12 @@ export default function ModelInputComponent({
   );
 
   const handleRefreshButtonPress = useCallback(async () => {
+    setOpen(false);
     setRefreshOptions(true);
     try {
       await refreshAllModelInputs({ silent: true });
+    } catch {
+      // refreshAllModelInputs handles its own error notifications via alertStore
     } finally {
       setRefreshOptions(false);
     }

src/frontend/src/modals/knowledgeBaseUploadModal/__tests__/KnowledgeBaseUploadModal.test.tsx

@@ -559,8 +559,8 @@ describe("KnowledgeBaseUploadModal", () => {
       const validFile = new File(["content"], "valid.md", {
         type: "text/markdown",
       });
-      const invalidFile = new File(["content"], "invalid.pdf", {
-        type: "application/pdf",
+      const invalidFile = new File(["content"], "invalid.exe", {
+        type: "application/x-msdownload",
       });
 
       // Manually trigger the change event
@@ -573,11 +573,11 @@ describe("KnowledgeBaseUploadModal", () => {
       fireEvent.change(folderInput, event);
 
       expect(screen.getByText("valid.md")).toBeInTheDocument();
-      expect(screen.queryByText("invalid.pdf")).not.toBeInTheDocument();
+      expect(screen.queryByText("invalid.exe")).not.toBeInTheDocument();
 
       expect(mockSetErrorData).toHaveBeenCalledWith(
         expect.objectContaining({
-          list: expect.arrayContaining(["invalid.pdf"]),
+          list: expect.arrayContaining(["invalid.exe"]),
         }),
       );
     });

src/frontend/src/modals/knowledgeBaseUploadModal/constants.ts

@@ -26,6 +26,8 @@ export const KB_INGEST_FORMATS: Record<string, string[]> = {
     "adoc",
     "asciidoc",
     "asc",
+    "pdf",
+    "docx",
   ],
   spreadsheets: ["csv"],
   code: ["py", "js", "ts", "tsx", "sh", "sql"],