Gym/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml at main · NousResearch/Gym · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
legal_agent_bench:
  resources_servers:
    legal_agent_bench:
      entrypoint: app.py
      domain: agent
      verified: false
      description: Harbor-native integration of <a href='https://github.com/harveyai/harvey-labs/tree/f46ef86e4788545622db25dcffa3aebb7a139929'>Legal Agent Benchmark (LAB)</a>
      value: Improve legal-agent document review, drafting, and analysis capability
      harbor_tasks_cache_dir: ${oc.env:LEGAL_AGENT_BENCH_TASK_CACHE_DIR,resources_servers/legal_agent_bench/data/cache/harbor_tasks/legal_agent_bench}
      harbor_tasks_dir: ${oc.env:LEGAL_AGENT_BENCH_RUNTIME_TASKS_DIR,resources_servers/legal_agent_bench/data/runtime/harbor_tasks/legal_agent_bench}
      harness_skills_dir: ${oc.env:LEGAL_AGENT_BENCH_SKILLS_DIR,resources_servers/legal_agent_bench/data/cache/harness/skills}
      auto_prepare_assets: true
      reward_mode: full_task  # official all-criteria score; criteria_pass_rate is diagnostic
      judge_base_url: ${oc.select:judge_base_url,null}
      judge_api_key: ${oc.select:judge_api_key,null}
      judge_model_name: ${oc.select:judge_model_name,null}
      judge_temperature: null
      judge_request_timeout_seconds: 90
      judge_max_retries: 1
      judge_structured_output: true
      judge_parse_repair_attempts: 1
      judge_repair_max_tokens: 4096
      judge_max_tokens: 4096
      judge_parallelism: 6

responses_create_params:
  temperature: 1.0
  top_p: 0.95

legal_agent_bench_harbor_agent:
  responses_api_agents:
    harbor_agent:
      entrypoint: ../../resources_servers/legal_agent_bench/harbor_bridge.py
      concurrency: 1

      harbor_datasets:
        legal_agent_bench:
          local_dataset_path: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir}

      harbor_agent_name: null
      harbor_agent_import_path: resources_servers.legal_agent_bench.legal_harbor_agent:LegalAgentBenchHarborAgent
      harbor_agent_kwargs:
        agent_id: legal-agent-bench
        agent_config_id: legal-agent-bench
        run_timestamp: nemo-gym
        skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir}
        max_turns: 60
        shell_timeout: 60
        agent_model_max_tokens: 32768
        agent_model_timeout_seconds: 1800

      harbor_environment_type: docker
      harbor_environment_import_path: null
      harbor_environment_kwargs: {}

      model_server:
        type: responses_api_models
        name: policy_model

      datasets:
      - name: example
        type: example
        jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl
        license: MIT
      - name: legal_agent_bench
        type: validation
        jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl
        license: MIT

      harbor_agent_override_timeout: 10800
      harbor_agent_max_timeout: null
      harbor_verifier_override_timeout: 3600
      harbor_verifier_max_timeout: null
      harbor_timeout_multiplier: null
      harbor_jobs_dir: results/legal_agent_bench/harbor_jobs