Adding internet and research paper tools for open world search tasks

pyproject.toml

@@ -15,6 +15,9 @@ dependencies = [
     "tqdm>=4.64.0",
     "numpy>=1.22.0",
     "python-dotenv>=1.2.1",
+    "beautifulsoup4>=4.12.0",
+    "httpx>=0.27.0",
+    "requests>=2.31.0",
 ]
 
 [project.optional-dependencies]
@@ -115,6 +118,7 @@ include = ["skydiscover*"]
 [tool.setuptools.package-data]
 skydiscover = [
     "context_builder/*/templates/*.txt",
+    "llm/tool_schemas/*.json",
     "search/evox/config/*.yaml",
     "search/evox/config/*.txt",
     "extras/external/defaults/*.yaml",

skydiscover/context_builder/default/templates/agentic_system_message.txt

@@ -1,13 +1,51 @@
-You are in agentic mode. You have tools to explore the codebase before writing code.
+You are in agentic mode. You have tools to explore the codebase and the internet before writing code.
 
-Tools: read_file, search
+Tools: read_file, search, web_search, fetch_webpage, research_papers, run_command
 
 Workflow:
-1. Review the project structure.
-2. Read files that look useful.
-3. When ready, output your complete improved program as text.
+1. Review the project structure and read files that look useful.
+2. Use `web_search` to find relevant algorithms, papers, or implementations.
+3. Use `fetch_webpage` to download the full content of promising URLs returned by web_search.
+   Then use `read_file` with the returned path to read the saved page content.
+   Example:
+     Step A → web_search(query="circle packing algorithms")  → get list of URLs
+     Step B → fetch_webpage(url="https://en.wikipedia.org/wiki/Circle_packing")  → returns reference/web_<slug>.txt
+     Step C → read_file(path="reference/web_<slug>.txt")  → read the actual content
+4. Use `research_papers` to search for academic research relevant to the task.
+5. Use `run_command` to execute commands inside the codebase root and inspect outputs (e.g. run existing scripts, sanity-check data files, compute simple statistics).
+
+`run_command` usage:
+- Call with `{"command": "<your command>"}`. The tool runs in the codebase root and returns a transcript including:
+  - the command (`$ ...`)
+  - `[mode: exec]` (safe mode) or `[mode: shell]` (unsafe mode)
+  - `[exit code: N]`
+  - stdout/stderr output
+- Prefer short commands that print useful information (avoid huge outputs).
+- Before running a command, mentally estimate its runtime and output size. Only
+  run it if you expect it to finish quickly and print bounded output. For most
+  exploratory commands, target under 20 seconds and under a few dozen lines.
+- Do not run broad filesystem searches (`find /`, `find /home`, or recursive
+  scans from a high-level directory). Use the project structure, known paths,
+  `read_file`, or narrow commands scoped to the codebase root.
+
+Safe mode (default):
+- Intended for inspection and analysis.
+- Only a limited set of executables is allowed.
+- Shell operators like pipes and redirects (`|`, `>`, `&&`, `;`, etc.) are blocked.
+- Because safe mode does not invoke a shell, do not include pipes, redirects,
+  command chaining, `2>/dev/null`, `head`/`tail` pipelines, glob tricks, or
+  other shell syntax. If you need filtering, write a short Python command that
+  prints only the exact bounded information you need.
+
+Unsafe mode (optional):
+- You may set `unsafe: true` to run via the system shell only when the runtime allows it.
+- This enables pipes/redirects/compound commands, but is higher risk and should be used sparingly.
+- If unsafe mode is disabled, `unsafe: true` will fail with an error; fall back to safe mode.
+
+When you use `run_command`, read the output carefully and explicitly incorporate what you learned into your next step. If two tool attempts fail because of path or safe-mode restrictions, stop exploring and output the best complete solution you can from the prompt and prior context.
+6. When ready, output your complete improved program as text.
 
 The code you output must be complete and self-contained.
 It will be evaluated in isolation — do NOT import from the codebase.
 Inline any constants or logic you find in other files.
-Be efficient — don't read files you don't need.
+Be efficient — fetch only the pages most likely to contain algorithmic details.

skydiscover/llm/agentic_generator.py

@@ -1,4 +1,4 @@
-"""Agentic code generator -- multi-turn tool-calling loop with read_file and search."""
+"""Agentic code generator -- multi-turn tool-calling loop with codebase and research tools."""
 
 import asyncio
 import concurrent.futures
@@ -8,6 +8,7 @@
 import os
 import re
 import time
+from importlib import resources
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -20,9 +21,11 @@
 
 logger = logging.getLogger(__name__)
 
-_TOOL_SCHEMAS_PATH = Path(__file__).parent / "tool_schemas" / "agentic_tools.json"
-with open(_TOOL_SCHEMAS_PATH, "r") as _f:
-    TOOL_SCHEMAS = json.load(_f)
+TOOL_SCHEMAS = json.loads(
+    resources.files("skydiscover.llm.tool_schemas")
+    .joinpath("agentic_tools.json")
+    .read_text(encoding="utf-8")
+)
 
 # Responses API uses a flattened tool format (name/description/parameters at top level)
 TOOL_SCHEMAS_RESPONSES = [
@@ -50,7 +53,8 @@ class AgenticGenerator:
     """
     V0 [simple version]: Multi-turn tool-calling agent that explores a codebase before generating code.
 
-    Tools: read_file, search. When it stops calling tools, its text output
+    Tools: read_file, search, web_search, research_papers, fetch_webpage, run_command.
+    When it stops calling tools, its text output
     is the final answer. Returns None if no output is produced (caller falls
     back to direct generation).
     """
@@ -150,7 +154,7 @@ async def generate(self, system_message: str, user_message: str) -> Optional[str
                     },
                 )
 
-                result = self._run_tool(name, args, files_read)
+                result = await self._run_tool(name, args, files_read)
                 conversation.append(
                     {"role": "tool", "tool_call_id": tc_id, "content": result["content"]}
                 )
@@ -272,13 +276,53 @@ async def _call_llm_responses(
     # Tools
     # ------------------------------------------------------------------
 
-    def _run_tool(self, name: str, args: Dict[str, Any], files_read: set) -> Dict[str, Any]:
+    async def _run_tool(self, name: str, args: Dict[str, Any], files_read: set) -> Dict[str, Any]:
         try:
             if name == "read_file":
                 return self._tool_read_file(args, files_read)
             elif name == "search":
                 return self._tool_search(args)
-            return _err(f"Unknown tool '{name}'. Available: read_file, search.")
+            elif name == "web_search":
+                from skydiscover.llm.tools.web_search_tool import web_search_handler
+
+                output, success = await web_search_handler(args)
+                return {"content": output, "_error": not success}
+            elif name in ("research_papers", "hf_papers"):
+                from skydiscover.llm.tools.papers_tool import research_papers_handler
+
+                output, success = await research_papers_handler(args)
+                return {"content": output, "_error": not success}
+            elif name == "fetch_webpage":
+                from skydiscover.llm.tools.fetch_webpage_tool import fetch_webpage_handler
+
+                output, success = await fetch_webpage_handler(
+                    args, codebase_root=self.config.codebase_root
+                )
+                return {"content": output, "_error": not success}
+            elif name == "run_command":
+                from skydiscover.llm.tools.run_command_tool import run_command_handler
+
+                if getattr(self.config, "run_command_enabled", True) is False:
+                    return _err(
+                        "run_command is disabled (agentic.run_command_enabled=false)."
+                    )
+                output, success = await run_command_handler(
+                    args,
+                    codebase_root=self.config.codebase_root,
+                    run_command_default_timeout=getattr(
+                        self.config, "run_command_default_timeout", 30
+                    ),
+                    run_command_max_timeout=getattr(self.config, "run_command_max_timeout", 120),
+                    run_command_max_output_chars=getattr(
+                        self.config, "run_command_max_output_chars", 20_000
+                    ),
+                    allow_unsafe_commands=getattr(self.config, "allow_unsafe_commands", False),
+                )
+                return {"content": output, "_error": not success}
+            return _err(
+                f"Unknown tool '{name}'. Available: read_file, search, web_search, "
+                "research_papers, fetch_webpage, run_command."
+            )
         except Exception as e:
             return _err(f"Tool '{name}' error: {e}")
 

skydiscover/llm/tool_schemas/agentic_tools.json

@@ -20,7 +20,9 @@
             "description": "End line (1-indexed). Omit for end of file."
           }
         },
-        "required": ["path"]
+        "required": [
+          "path"
+        ]
       }
     }
   },
@@ -41,7 +43,190 @@
             "description": "Glob to filter files (default: '*.py')."
           }
         },
-        "required": ["pattern"]
+        "required": [
+          "pattern"
+        ]
+      }
+    }
+  },
+  {
+    "type": "function",
+    "function": {
+      "name": "web_search",
+      "description": "Search the web for current information and return cited results.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "query": {
+            "type": "string",
+            "minLength": 2
+          },
+          "allowed_domains": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "Optional allowlist of domains or URLs. Subdomains match."
+          },
+          "blocked_domains": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "Optional blocklist of domains or URLs. Subdomains match."
+          }
+        },
+        "required": [
+          "query"
+        ],
+        "additionalProperties": false
+      }
+    }
+  },
+  {
+    "type": "function",
+    "function": {
+      "name": "research_papers",
+      "description": "Discover ML research papers, analyze citations, search paper contents, and find linked resources.\n\nCombines HuggingFace Hub, arXiv, and Semantic Scholar. Use for exploring research areas, finding datasets for a task, tracing citation chains, or implementing a paper's approach.\n\nTypical flows:\n  search \u2192 read_paper \u2192 find_all_resources \u2192 hf_inspect_dataset\n  search \u2192 paper_details \u2192 citation_graph \u2192 read_paper (trace influence)\n  snippet_search \u2192 paper_details \u2192 read_paper (find specific claims)\n\nOperations:\n- trending: Get trending daily papers, optionally filter by topic keyword\n- search: Search papers. Uses HF by default (ML-tuned). Add date_from/min_citations/categories to use Semantic Scholar with filters\n- paper_details: Metadata, abstract, AI summary, github link\n- read_paper: Read paper contents \u2014 without section: abstract + TOC; with section: full text\n- citation_graph: Get references and citations for a paper with influence flags and citation intents\n- snippet_search: Semantic search over full-text passages from 12M+ papers\n- recommend: Find similar papers (single paper or positive/negative examples)\n- find_datasets: Find datasets linked to a paper\n- find_models: Find models linked to a paper\n- find_collections: Find collections that include a paper\n- find_all_resources: Parallel fetch of datasets + models + collections for a paper",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "operation": {
+            "type": "string",
+            "enum": [
+              "trending",
+              "search",
+              "paper_details",
+              "read_paper",
+              "citation_graph",
+              "snippet_search",
+              "recommend",
+              "find_datasets",
+              "find_models",
+              "find_collections",
+              "find_all_resources"
+            ],
+            "description": "Operation to execute."
+          },
+          "query": {
+            "type": "string",
+            "description": "Search query. Required for: search, snippet_search. Optional for: trending (filters by keyword). Supports boolean syntax for Semantic Scholar: '\"exact phrase\" term1 | term2'."
+          },
+          "arxiv_id": {
+            "type": "string",
+            "description": "ArXiv paper ID (e.g. '2305.18290'). Required for: paper_details, read_paper, citation_graph, find_datasets, find_models, find_collections, find_all_resources. Optional for: recommend (single-paper recs). Get IDs from search results first."
+          },
+          "section": {
+            "type": "string",
+            "description": "Section name or number to read (e.g. '3', 'Experiments', '4.2'). Optional for: read_paper. Without this, returns abstract + TOC."
+          },
+          "direction": {
+            "type": "string",
+            "enum": [
+              "citations",
+              "references",
+              "both"
+            ],
+            "description": "Direction for citation_graph. Default: both."
+          },
+          "date": {
+            "type": "string",
+            "description": "Date in YYYY-MM-DD format. Optional for: trending (defaults to recent papers)."
+          },
+          "date_from": {
+            "type": "string",
+            "description": "Start date (YYYY-MM-DD). Triggers Semantic Scholar search. For: search, snippet_search."
+          },
+          "date_to": {
+            "type": "string",
+            "description": "End date (YYYY-MM-DD). Triggers Semantic Scholar search. For: search, snippet_search."
+          },
+          "categories": {
+            "type": "string",
+            "description": "Field of study filter (e.g. 'Computer Science'). Triggers Semantic Scholar search."
+          },
+          "min_citations": {
+            "type": "integer",
+            "description": "Minimum citation count filter. Triggers Semantic Scholar search."
+          },
+          "sort_by": {
+            "type": "string",
+            "enum": [
+              "relevance",
+              "citationCount",
+              "publicationDate"
+            ],
+            "description": "Sort order for Semantic Scholar search. Default: relevance."
+          },
+          "positive_ids": {
+            "type": "string",
+            "description": "Comma-separated arxiv IDs for multi-paper recommendations. For: recommend."
+          },
+          "negative_ids": {
+            "type": "string",
+            "description": "Comma-separated arxiv IDs as negative examples. For: recommend."
+          },
+          "sort": {
+            "type": "string",
+            "enum": [
+              "downloads",
+              "likes",
+              "trending"
+            ],
+            "description": "Sort order for find_datasets and find_models. Default: downloads."
+          },
+          "limit": {
+            "type": "integer",
+            "description": "Maximum results to return (default: 10, max: 50)."
+          }
+        },
+        "required": [
+          "operation"
+        ]
+      }
+    }
+  },
+  {
+    "type": "function",
+    "function": {
+      "name": "fetch_webpage",
+      "description": "Fetch a webpage by URL, convert it to plain text, and save it as a local file so you can read it with read_file.\n\nTypical flow:\n  1. web_search → get URLs\n  2. fetch_webpage(url=...) → get saved_path\n  3. read_file(path=saved_path) → read content\n\nThe saved file is plain text (HTML tags stripped), stored under reference/web_<slug>.txt inside the codebase root. Returns the relative path to use with read_file.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "url": {
+            "type": "string",
+            "description": "The full URL to fetch (must start with http:// or https://)."
+          }
+        },
+        "required": ["url"],
+        "additionalProperties": false
+      }
+    }
+  },
+  {
+    "type": "function",
+    "function": {
+      "name": "run_command",
+      "description": "Execute a read-only shell command inside the codebase root directory and return its stdout/stderr output.\n\nUse this to inspect and analyse data files, run existing scripts, count lines, pretty-print JSON/CSV, profile code, etc.\n\nRestrictions:\n- Only a fixed allowlist of executables is permitted (python, python3, cat, head, tail, wc, grep, rg, find, ls, awk, sed, jq, sort, uniq, etc.).\n- Shell operators (|, &, ;, >, <) are NOT allowed — pipe output by saving to a file first or use awk/python inline.\n- Destructive commands (rm, mv, cp, pip, curl, wget, etc.) are blocked.\n- The working directory is always the codebase root.\n- Maximum runtime: 120 seconds (default 30).",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "command": {
+            "type": "string",
+            "description": "The command to run (e.g. 'python analyse.py', 'head -n 20 data/results.csv', 'wc -l src/*.py')."
+          },
+          "timeout": {
+            "type": "integer",
+            "description": "Optional wall-clock timeout in seconds (1–120, default 30)."
+          },
+          "unsafe": {
+            "type": "boolean",
+            "description": "If true, run the command through the system shell for full commandline capabilities (pipes, redirects, compound commands). Disabled by default."
+          }
+        },
+        "required": ["command"],
+        "additionalProperties": false
       }
     }
   }