Skip to content

fix: bound web search result content #14470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Oxygen56 wants to merge 6 commits into
base: main
Choose a base branch
from
Open

fix: bound web search result content #14470

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8454392
fix: bound web search result content
Oxygen56 Aug 8, 2026
5df6cce
[autofix.ci] apply automated fixes
autofix-ci[bot] Aug 8, 2026
65cbdeb
[autofix.ci] apply automated fixes (attempt 2/3)
autofix-ci[bot] Aug 8, 2026
23ba428
test: cover nonpositive web search limits
Oxygen56 Aug 8, 2026
cf37613
chore: resolve component index merge conflict
Oxygen56 Aug 8, 2026
26bfacc
chore: merge main into web search fix
Oxygen56 Aug 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions src/backend/tests/unit/components/data_source/test_web_search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ def default_kwargs(self):
return {
"search_mode": "Web",
"query": "OpenAI GPT-4",
"max_results": 5,
"max_content_length": 4000,
"timeout": 5,
}

Expand Down Expand Up @@ -123,6 +125,15 @@ def test_clean_html(self):
expected = "Title Paragraph"
assert component.clean_html(html) == expected

def test_web_search_limit_inputs(self):
"""Web search should expose conservative advanced limits."""
inputs = {input_.name: input_ for input_ in WebSearchComponent.inputs}

assert inputs["max_results"].value == 5
assert inputs["max_results"].advanced is True
assert inputs["max_content_length"].value == 4000
assert inputs["max_content_length"].advanced is True

def test_update_build_config_web_mode(self):
"""Test build config update for Web mode."""
component = WebSearchComponent()
Expand Down Expand Up @@ -187,6 +198,46 @@ def test_perform_web_search_success(self, mock_get, mock_safe_get):
assert result.iloc[0]["snippet"] == "Test snippet content"
assert "Page content" in result.iloc[0]["content"]

@patch.object(WebSearchComponent, "_safe_get_url")
@patch("lfx.components.data_source.web_search.requests.get")
def test_perform_web_search_limits_results_and_content(self, mock_get, mock_safe_get):
"""Web search should bound both fetched result count and returned page text."""
component = WebSearchComponent()
component.query = "test query"
component.max_results = 2
component.max_content_length = 12
component.timeout = 5

mock_response = Mock()
mock_response.text = """
<html>
<div class="result">
<a class="result__a" href="?uddg=https%3A%2F%2Fexample.com%2F1">First</a>
</div>
<div class="result">
<a class="result__a" href="?uddg=https%3A%2F%2Fexample.com%2F2">Second</a>
</div>
<div class="result">
<a class="result__a" href="?uddg=https%3A%2F%2Fexample.com%2F3">Third</a>
</div>
</html>
"""
mock_response.headers = {"content-type": "text/html"}
mock_response.raise_for_status.return_value = None
mock_get.return_value = mock_response

mock_page_response = Mock()
mock_page_response.text = "<html><body>abcdefghijklmnopqrstuvwxyz</body></html>"
mock_page_response.raise_for_status.return_value = None
mock_safe_get.return_value = mock_page_response

with patch("lfx.components.data_source.web_search.get_user_agent", return_value="test-agent"):
result = component.perform_web_search()

assert len(result) == 2
assert mock_safe_get.call_count == 2
assert result["content"].tolist() == ["abcdefghijkl", "abcdefghijkl"]
Comment on lines +201 to +239

@coderabbitai coderabbitai Bot Aug 8, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

Test zero and negative limits.

The implementation clamps both limits to zero. This test only covers positive values. Add cases for max_results=0, max_results=-1, max_content_length=0, and max_content_length=-1. Assert that a zero result limit fetches no result pages and that a zero content limit returns empty content.

As per coding guidelines, backend tests must cover positive, negative, edge, and error cases.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/backend/tests/unit/components/data_source/test_web_search.py` around
lines 201 - 239, Add test coverage around
test_perform_web_search_limits_and_content for zero and negative max_results and
max_content_length values. Verify max_results=0 and -1 fetch no result pages,
and max_content_length=0 and -1 return empty content, while preserving the
existing positive-limit assertions and mocks.

Source: Coding guidelines


@patch("lfx.components.data_source.web_search.requests.get")
def test_perform_web_search_no_results(self, mock_get):
"""Test web search with no results."""
Expand Down
21 changes: 20 additions & 1 deletion src/lfx/src/lfx/components/data_source/web_search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,22 @@ class WebSearchComponent(Component):
required=False,
advanced=True,
),
IntInput(
name="max_results",
display_name="Max Results",
info="Maximum number of web results to fetch and return.",
value=5,
required=False,
advanced=True,
),
IntInput(
name="max_content_length",
display_name="Max Content Length",
info="Maximum number of characters of scraped page content returned per web result.",
value=4000,
required=False,
advanced=True,
),
IntInput(
name="timeout",
display_name="Timeout",
Expand Down Expand Up @@ -200,8 +216,10 @@ def perform_web_search(self) -> DataFrame:

soup = BeautifulSoup(response.text, "html.parser")
results = []
max_results = max(self.max_results or 0, 0)
max_content_length = max(self.max_content_length or 0, 0)

for result in soup.select("div.result"):
for result in soup.select("div.result")[:max_results]:
title_tag = result.select_one("a.result__a")
snippet_tag = result.select_one("a.result__snippet")
if title_tag:
Expand All @@ -221,6 +239,7 @@ def perform_web_search(self) -> DataFrame:
content = f"(Blocked by SSRF protection: {e!s})"
else:
content = f"(Failed to fetch: {e!s})"
content = content[:max_content_length]

results.append(
{
Expand Down
Loading