feat (ml-tooling): cap by lines instead of number of chars

Author: monatis

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "isanagent"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/altaidevorg/isanagent"

Cargo.toml

@@ -473,7 +473,6 @@ Enable [api], [slack], or [email] (with enabled = true) so the agent can receive
         max_output_chars: max_web_output_chars,
     }));
     tools.register(Box::new(ArxivFetchTool {
-        max_output_chars: max_web_output_chars,
         workspace_dir: workspace.dir.clone(),
     }));
     tools.register(Box::new(HfHubFileFetchTool {

src/main.rs

@@ -138,7 +138,7 @@ impl Tool for ReadFileTool {
     }
 
     fn description(&self) -> &str {
-        "Read the contents of a local file. Provide the absolute or relative path to the file. You can optionally read specific lines by specifying start_line and end_line (1-indexed, inclusive) capped at a maximum of 100 lines per call."
+        "Read the contents of a local file. Provide the absolute or relative path to the file. You can read specific lines by specifying start_line and end_line (1-indexed, inclusive) capped at a maximum of 100 lines per call, so you can call this tool multiple times to read a file with more than 100 lines when needed."
     }
 
     fn parameters(&self) -> Value {
@@ -151,14 +151,14 @@ impl Tool for ReadFileTool {
                 },
                 "start_line": {
                     "type": "integer",
-                    "description": "Optional starting line number (1-indexed, inclusive)"
+                    "description": "Starting line number (1-indexed, inclusive)"
                 },
                 "end_line": {
                     "type": "integer",
-                    "description": "Optional ending line number (1-indexed, inclusive)"
+                    "description": "Ending line number (1-indexed, inclusive)"
                 }
             },
-            "required": ["path"]
+            "required": ["path", "start_line", "end_line"]
         })
     }
 
@@ -170,22 +170,19 @@ impl Tool for ReadFileTool {
 
         let actual_path = resolve_path(path_str, &self.workspace_dir, self.restrict_to_workspace)?;
 
-        let start_line = args.get("start_line").and_then(|v| v.as_u64());
-        let end_line = args.get("end_line").and_then(|v| v.as_u64());
+        let start_line = args
+            .get("start_line")
+            .and_then(|v| v.as_u64())
+            .ok_or("Missing 'start_line' argument")?;
+        let end_line = args
+            .get("end_line")
+            .and_then(|v| v.as_u64())
+            .ok_or("Missing 'end_line' argument")?;
 
         let content = fs::read_to_string(&actual_path).map_err(|e| e.to_string())?;
 
-        if start_line.is_none() && end_line.is_none() {
-            // No lines specified, check total lines
-            let total_lines = content.lines().count();
-            if total_lines > 100 {
-                return Err(format!("File is too large ({} lines). Please specify start_line and end_line to read a maximum of 100 lines at a time.", total_lines));
-            }
-            return Ok(content);
-        }
-
-        let start = start_line.unwrap_or(1).max(1) as usize;
-        let end = end_line.unwrap_or(start as u64 + 99) as usize;
+        let start = start_line.max(1) as usize;
+        let end = end_line as usize;
 
         if end < start {
             return Err("end_line must be greater than or equal to start_line".to_string());
@@ -201,8 +198,7 @@ impl Tool for ReadFileTool {
 
         let snippet: Vec<String> = lines[actual_start - 1..actual_end]
             .iter()
-            .enumerate()
-            .map(|(i, l)| format!("{:4}: {}", actual_start + i, l))
+            .map(|l| l.to_string())
             .collect();
 
         Ok(snippet.join("\n"))

src/tools/builtin.rs

@@ -638,10 +638,10 @@ impl Tool for ExecutionReadLogTool {
                 "job_id": { "type": "string", "description": "The job ID to read logs from. Provide either this or run_id." },
                 "run_id": { "type": "string", "description": "The run ID to read logs from. Provide either this or job_id." },
                 "stream": { "type": "string", "enum": ["stdout", "stderr"], "description": "Which stream to read." },
-                "start_line": { "type": "integer", "description": "Optional starting line number (1-indexed, inclusive)" },
-                "end_line": { "type": "integer", "description": "Optional ending line number (1-indexed, inclusive)" }
+                "start_line": { "type": "integer", "description": "Starting line number (1-indexed, inclusive)" },
+                "end_line": { "type": "integer", "description": "Ending line number (1-indexed, inclusive)" }
             },
-            "required": ["stream"]
+            "required": ["stream", "start_line", "end_line"]
         })
     }
 
@@ -650,8 +650,14 @@ impl Tool for ExecutionReadLogTool {
             .get("stream")
             .and_then(|v| v.as_str())
             .unwrap_or("stdout");
-        let start_line = args.get("start_line").and_then(|v| v.as_u64());
-        let end_line = args.get("end_line").and_then(|v| v.as_u64());
+        let start_line = args
+            .get("start_line")
+            .and_then(|v| v.as_u64())
+            .ok_or("Missing 'start_line' argument")?;
+        let end_line = args
+            .get("end_line")
+            .and_then(|v| v.as_u64())
+            .ok_or("Missing 'end_line' argument")?;
 
         let run_id;
         let mut sid = None;
@@ -725,8 +731,8 @@ impl Tool for ExecutionReadLogTool {
 
         let mut reader = BufReader::new(file);
 
-        let start = start_line.unwrap_or(1).max(1) as usize;
-        let end = end_line.unwrap_or(start as u64 + 99) as usize;
+        let start = start_line.max(1) as usize;
+        let end = end_line as usize;
 
         if end < start {
             return Err("end_line must be greater than or equal to start_line".to_string());
@@ -744,11 +750,7 @@ impl Tool for ExecutionReadLogTool {
                 Ok(0) => break,
                 Ok(_) => {
                     if current_line >= start && current_line <= actual_end {
-                        lines.push(format!(
-                            "{:4}: {}",
-                            current_line,
-                            buf.trim_end_matches(&['\r', '\n'][..])
-                        ));
+                        lines.push(buf.trim_end_matches(&['\r', '\n'][..]).to_string());
                     }
                     current_line += 1;
                     if current_line > actual_end {

src/tools/execution.rs

@@ -131,7 +131,6 @@ impl Tool for ArxivSearchTool {
 
 /// Fetch one arXiv abstract page (abs HTML) by id.
 pub struct ArxivFetchTool {
-    pub max_output_chars: usize,
     pub workspace_dir: std::path::PathBuf,
 }
 
@@ -166,20 +165,27 @@ impl Tool for ArxivFetchTool {
             return Err("invalid arxiv_id".to_string());
         }
 
-        let url = format!("https://arxiv.org/html/{}", id);
+        let arxiv2md_url = format!("https://arxiv2md.org/api/markdown?url={}", id);
 
         let client = reqwest::Client::builder()
             .user_agent(HF_USER_AGENT)
             .build()
             .map_err(|e| e.to_string())?;
 
-        let resp = client
-            .get(&url)
-            .send()
-            .await
-            .map_err(|e| format!("arxiv_fetch request: {}", e))?;
+        let arxiv2md_resp = client.get(&arxiv2md_url).send().await;
 
-        let full_content = if !resp.status().is_success() {
+        let mut html_markdown_content = String::new();
+        if let Ok(resp) = arxiv2md_resp {
+            if resp.status().is_success() {
+                if let Ok(text) = resp.text().await {
+                    if text.lines().count() >= 30 {
+                        html_markdown_content = text;
+                    }
+                }
+            }
+        }
+
+        let full_content = if html_markdown_content.is_empty() {
             let pdf_url = format!("https://arxiv.org/pdf/{}.pdf", id);
             let pdf_resp = client
                 .get(&pdf_url)
@@ -189,7 +195,7 @@ impl Tool for ArxivFetchTool {
 
             if !pdf_resp.status().is_success() {
                 return Err(format!(
-                    "arxiv_fetch HTTP {} (HTML not found, and PDF not found for {})",
+                    "arxiv_fetch HTTP {} (PDF not found for {})",
                     pdf_resp.status(),
                     id
                 ));
@@ -203,12 +209,7 @@ impl Tool for ArxivFetchTool {
 
             crate::utils::extract_markdown_from_pdf_bytes(&pdf_bytes)?
         } else {
-            let html_content = resp
-                .text()
-                .await
-                .map_err(|e| format!("arxiv_fetch body: {}", e))?;
-
-            htmd::convert(&html_content).map_err(|e| format!("html to markdown error: {}", e))?
+            html_markdown_content
         };
 
         let downloads_dir = self
@@ -222,20 +223,28 @@ impl Tool for ArxivFetchTool {
             .await
             .map_err(|e| e.to_string())?;
 
-        let safe_limit = self.max_output_chars.saturating_sub(1000).max(1000);
-        let mut body = full_content.clone();
-        crate::utils::truncate_utf8_safe(&mut body, safe_limit, "\n... [TRUNCATED]");
-
         let total_lines = full_content.lines().count();
+        let max_preview_lines = 50;
 
-        Ok(format!(
-            "{body}\n\n---\nSystem: The full response ({} lines, {} bytes) was saved to `{}`. \
-            If this preview is truncated, use the `read_file` tool with `start_line` and `end_line` arguments \
-            on that path to incrementally read the rest of the content and/or use the `search_text` tool to find specific information.",
-            total_lines,
-            full_content.len(),
-            file_path.display()
-        ))
+        if total_lines <= max_preview_lines {
+            Ok(format!(
+                "{full_content}\n\n---\nFull paper ({total_lines} lines) saved to `{}`.",
+                file_path.display()
+            ))
+        } else {
+            let preview: String = full_content
+                .lines()
+                .take(max_preview_lines)
+                .collect::<Vec<_>>()
+                .join("\n");
+
+            Ok(format!(
+                "{preview}\n\n---\n[TRUNCATED] Showing first {max_preview_lines} of {total_lines} lines. \
+                Full content saved to `{}`. Use `read_file` with `start_line` and `end_line` to read \
+                the rest, or `search_text` to find specific information.",
+                file_path.display()
+            ))
+        }
     }
 }