fix: deterministic path resolution in scoring

CLAUDE.md

@@ -0,0 +1,78 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Corral is a benchmarking framework for evaluating AI agents on science tasks (materials science, chemistry, etc.). It provides standardized environments, tools, and metrics to test agent performance across diverse scientific challenges.
+
+## Common Commands
+
+```bash
+# Install dependencies
+uv sync --all-extras --dev
+
+# Run all tests
+uv run pytest tests
+
+# Run a single test file
+uv run pytest tests/agents/test_react.py
+
+# Run a specific test
+uv run pytest tests/agents/test_react.py::test_name -v
+
+# Lint
+ruff check .
+
+# Format
+ruff format .
+
+# Install a task environment for development
+cd tasks/<task_name> && uv pip install -e .
+
+# Run task-specific tests
+cd tasks/<task_name> && CORRAL_WORK_DIR=$(mktemp -d) uv run python -m pytest tests -v --rootdir=.
+
+# Pre-commit (includes ruff, formatting, commitizen)
+pre-commit run --all-files
+```
+
+## Architecture
+
+### Core flow: Runner → Router → Server → Environment
+
+- **CorralRunner** (`src/corral/run.py`) — Orchestrates benchmark execution: manages trials, checkpointing (auto-save/resume), budget tracking, and W&B logging.
+- **CorralRouter** (`src/corral/router/routes.py`) — HTTP client that agents use to interact with the benchmark server (list tasks, get tools, execute tools, submit answers).
+- **Benchmark Server** (`src/corral/backend/server.py`) — FastAPI server created via `create_benchmark_server()` that hosts task environments.
+- **Environment** (`src/corral/backend/env.py`) — Abstract base class for task environments. Each task implements this with its own tools and scoring. `TaskState` tracks trial state (messages, tool calls, score).
+
+### Agent types (`src/corral/agents/`)
+
+All extend `BaseAgent` (`base_agent.py`). Four strategies:
+- **ReActAgent** — Reasoning + Acting loop
+- **ToolCallingAgent** — Native LLM function calling
+- **LLMPlanner** — Hierarchical planning
+- **ReflexionAgent** — Self-reflection and learning from mistakes
+
+Agents use LiteLLM for unified LLM API access. Agent utilities (token counting, LLM calls) are in `agents/utils.py`.
+
+### Tool system (`src/corral/backend/tool.py`)
+
+Tools are defined with the `@tool` decorator. Supports MCP (Model Context Protocol) conversion and Modal cloud execution (`@modal_tool`). Tool verbosity levels (FULL, BRIEF, MINIMAL, NONE) control description detail via `src/corral/router/verbosity.py`.
+
+### Metrics and reporting (`src/corral/report/`)
+
+Registry-based metric system. `BenchmarkResult` aggregates results; `TaskTrialResult` holds individual trial data. Default metrics include average score, success rate, and pass@k. Custom metrics register via `MetricRegistry`.
+
+### Task environments (`tasks/`)
+
+Each task is its own Python package with `pyproject.toml`, `env.py`, `tools.py`, `tests/`, and `config/`. Tasks include: samplemath, spectra_elucidation, corral_md (LAMMPS MD), catalyst, afm, ml, retrosynthesis, resistor_network, wetlab.
+
+## Key Conventions
+
+- **Commits**: Uses commitizen for conventional commits (enforced by pre-commit hook on commit-msg stage). Install hooks with: `pre-commit install --hook-type commit-msg --hook-type pre-push`
+- **Imports**: `src` layout — the package is `corral` under `src/corral/`. pytest is configured with `pythonpath = "src"`.
+- **Ruff**: Extensive rule set enabled (see `pyproject.toml`). Notable ignores: E501 (line length), PLR (design pylint). Print statements (T20) and unused variables (F841) are unfixable (won't be auto-removed). `__init__.py` files allow unused imports (F401). Tests ignore ANN, ARG, D, E402, PTH, S101.
+- **Pre-commit excludes**: Ruff, formatting, and whitespace hooks exclude `tasks/` (except `tasks/*/src/`) and `reports/afm/`.
+- **Git LFS**: Used for large files (e.g., MD environment data).
+- **Environment variable**: `CORRAL_WORK_DIR` — needed by some task tests for working directory.

src/corral/backend/env.py

@@ -1,5 +1,6 @@
 import inspect
 import time
+import uuid
 from abc import ABC, abstractmethod
 from copy import deepcopy
 from dataclasses import dataclass, field
@@ -41,6 +42,8 @@ class TaskState:
     submitted_answer: str | None = None
     feedback: str | None = None
     surrendered: bool = False
+    workspace_id: str | None = None
+    workspace_path: str | None = None
     env_start_time: datetime = field(
         default_factory=lambda: datetime.now(tz=timezone.utc)
     )
@@ -81,6 +84,15 @@ def __init__(self, task_id: str, base_work_dir: str, fs_manager=None):
         self.trial_states: dict[str, TaskState] = {}
         self.trial_counter = -1
         self.fs_manager = fs_manager
+        self.current_workspace_id: str | None = None
+        if base_work_dir:
+            from corral.utils.workspace_registry import WorkspaceRegistry
+
+            self.workspace_registry: WorkspaceRegistry | None = WorkspaceRegistry(
+                base_work_dir
+            )
+        else:
+            self.workspace_registry = None
         self.reset_state()
 
     def save_current_state(self) -> TaskState:
@@ -100,25 +112,47 @@ def reset_state(self) -> str:
                 self.state.env_end_time = datetime.now(tz=timezone.utc)
             archived_snapshot = self.save_current_state()
             self.trial_states[self.state.trial_id] = archived_snapshot
+            # Mark previous workspace as completed in registry
+            if self.workspace_registry and self.current_workspace_id:
+                self.workspace_registry.update_status(
+                    self.current_workspace_id,
+                    "completed",
+                    completed_at=datetime.now(tz=timezone.utc).isoformat(),
+                )
 
         self.trial_counter += 1
         new_trial_id = str(self.trial_counter)
 
         if self.base_work_dir:
-            self.current_work_dir = self._create_trial_workspace(new_trial_id)
+            self.current_workspace_id = str(uuid.uuid4())
+            self.current_work_dir = self._create_trial_workspace(
+                new_trial_id, self.current_workspace_id
+            )
+            if self.workspace_registry:
+                self.workspace_registry.register_workspace(
+                    self.task_id,
+                    new_trial_id,
+                    self.current_workspace_id,
+                    self.current_work_dir,
+                )
         else:
             self.current_work_dir = None
+            self.current_workspace_id = None
 
         self.state = TaskState(
             task_id=self.task_id,
             trial_id=new_trial_id,
             task_prompt=self.get_task_prompt(),
+            workspace_id=self.current_workspace_id,
+            workspace_path=self.current_work_dir,
         )
         return self.state.trial_id
 
-    def _create_trial_workspace(self, trial_id: str) -> str:
-        """Create workspace directory for this trial"""
-        workspace = Path(self.base_work_dir) / f"{self.task_id}_trial_{trial_id}"
+    def _create_trial_workspace(self, trial_id: str, workspace_id: str) -> str:
+        """Create workspace directory for this trial with UUID identifier."""
+        workspace = (
+            Path(self.base_work_dir) / f"{self.task_id}_trial_{trial_id}_{workspace_id}"
+        )
         logger.info(f"Creating workspace: {workspace}")
         if self.fs_manager:
             self.fs_manager.mkdir(str(workspace), create_parents=True)
@@ -381,6 +415,8 @@ def get_completed_trial_data(self) -> dict:
             "submitted_answer": self.state.submitted_answer,
             "surrendered": self.state.surrendered,
             "tool_statistics": self._get_complete_tool_statistics(),
+            "workspace_id": self.state.workspace_id,
+            "workspace_path": self.state.workspace_path,
         }
 
         return {"trial_id": self.state.trial_id, "state": state_data}

src/corral/backend/schema.py

@@ -56,6 +56,7 @@ class TrialCompletionResponse(BaseModel):
     state: dict[str, Any]
     trial_id: str
     surrendered: bool = False
+    workspace_id: str | None = None
 
 
 class ToLatexRequest(BaseModel):

src/corral/backend/server.py

@@ -47,6 +47,7 @@ def _finalize_trial(
         state=completed_trial["state"],
         trial_id=finished_trial_id,
         surrendered=surrendered,
+        workspace_id=completed_trial["state"].get("workspace_id"),
     )
 
 
@@ -332,6 +333,52 @@ def generate_latex(task_id: str, request: ToLatexRequest):
                 status_code=500, detail=f"Failed to generate LaTeX: {e!s}"
             ) from e
 
+    @app.get("/tasks/{task_id}/workspaces")
+    def get_workspaces(task_id: str):
+        """Get all workspace entries for a task from the registry."""
+        if task_id not in environments:
+            raise HTTPException(status_code=404, detail="Task not found")
+        env = environments[task_id]
+        if env.workspace_registry:
+            entries = env.workspace_registry.get_workspaces(task_id=task_id)
+            return {
+                "workspaces": [
+                    {
+                        "workspace_id": e.workspace_id,
+                        "task_id": e.task_id,
+                        "trial_id": e.trial_id,
+                        "path": e.path,
+                        "status": e.status,
+                        "created_at": e.created_at,
+                        "completed_at": e.completed_at,
+                    }
+                    for e in entries
+                ]
+            }
+        return {"workspaces": []}
+
+    @app.get("/tasks/{task_id}/workspaces/{workspace_id}")
+    def get_workspace(task_id: str, workspace_id: str):
+        """Look up a specific workspace by UUID."""
+        if task_id not in environments:
+            raise HTTPException(status_code=404, detail="Task not found")
+        env = environments[task_id]
+        if env.workspace_registry:
+            entry = env.workspace_registry.get_by_id(workspace_id)
+            if entry:
+                return {
+                    "workspace": {
+                        "workspace_id": entry.workspace_id,
+                        "task_id": entry.task_id,
+                        "trial_id": entry.trial_id,
+                        "path": entry.path,
+                        "status": entry.status,
+                        "created_at": entry.created_at,
+                        "completed_at": entry.completed_at,
+                    }
+                }
+        raise HTTPException(status_code=404, detail="Workspace not found")
+
     return app
 
 

src/corral/report/results.py

@@ -22,6 +22,8 @@ class TaskTrialResult:
     token_usage: dict[str, int] | None = None
     error_message: str | None = None
     surrendered: bool = False
+    workspace_id: str | None = None
+    workspace_path: str | None = None
 
     @property
     def success(self) -> bool:

src/corral/router/routes.py

@@ -32,6 +32,8 @@ def _parse_trial_completion(task_id: str, response_data: dict) -> TaskTrialResul
         tool_statistics=completion.state["tool_statistics"],
         surrendered=completion.surrendered,
         duration=completion.state.get("duration"),
+        workspace_id=completion.workspace_id,
+        workspace_path=completion.state.get("workspace_path"),
     )
 
 

src/corral/utils/io_tools.py

@@ -1,13 +1,19 @@
+from __future__ import annotations
+
 import json
 import re
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import fsspec
 import modal
 
 from corral.backend.schema import ToolArgument
 from corral.backend.tool import Tool
 
+if TYPE_CHECKING:
+    from corral.utils.workspace_registry import WorkspaceRegistry
+
 
 class FSManager:
     """A file-system abstraction layer using fsspec.
@@ -19,12 +25,16 @@ def __init__(
         protocol: str = "file",
         base_path: str = "./",
         app: str | None = None,
+        registry: WorkspaceRegistry | None = None,
+        workspace_id: str | None = None,
         **kwargs,
     ):
         self.protocol = protocol
         self.base_path = Path(base_path) if base_path else None
         self.fs = fsspec.filesystem(protocol, **kwargs)
         self.app = app
+        self.registry = registry
+        self.workspace_id = workspace_id
 
     def _resolve_path(self, path: str) -> str:
         """Resolve a relative path against the base_path"""
@@ -38,19 +48,6 @@ def _resolve_path(self, path: str) -> str:
         resolved = self.base_path / path_obj
         return str(resolved)
 
-    def _resolve_path(self, path: str) -> str:
-        """Resolve a relative path against the base_path"""
-        if self.base_path is None:
-            return path
-
-        path_obj = Path(path)
-        if path_obj.is_absolute():
-            return str(path_obj)
-        else:
-            # Relative path - resolve against base_path
-            resolved = self.base_path / path_obj
-            return str(resolved)
-
     def list_files(self, path: str, recursive: bool = False) -> list[str]:
         """List files in a directory"""
         try:
@@ -88,6 +85,11 @@ def write_file(self, path: str, content: str) -> None:
 
                 with self.fs.open(resolved_path, "w") as f:
                     f.write(content)
+
+            # Track file in registry
+            if self.registry and self.workspace_id:
+                filename = Path(resolved_path).name
+                self.registry.register_file(self.workspace_id, filename, resolved_path)
         except Exception as e:
             raise RuntimeError(f"Error writing file {path}: {e}") from e
 
@@ -247,7 +249,8 @@ def __init__(self, fs_manager: FSManager):
 
     def execute(self, **kwargs) -> str:
         self.fs_manager.write_file(kwargs["path"], kwargs["content"])
-        return f"Successfully wrote to {kwargs['path']}"
+        resolved = self.fs_manager._resolve_path(kwargs["path"])
+        return f"Successfully wrote to {resolved}"
 
 
 class FileInfoTool(Tool):
@@ -343,7 +346,8 @@ def __init__(self, fs_manager: FSManager):
     def execute(self, **kwargs) -> str:
         try:
             self.fs_manager.copy_file(kwargs["source"], kwargs["destination"])
-            return f"Copied {kwargs['source']} to {kwargs['destination']}"
+            resolved_dest = self.fs_manager._resolve_path(kwargs["destination"])
+            return f"Copied {kwargs['source']} to {resolved_dest}"
         except Exception as e:
             return f"Error: {e}"