Skip to content

Commit 6ebd67b

Browse files
PanAndyPanAndy
committed
(feat): agentic add gigpo.
1 parent 7ff3a3e commit 6ebd67b

35 files changed

Lines changed: 1860 additions & 894 deletions

‎examples/config/envs.yaml‎

Lines changed: 0 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +0,0 @@
1-
custom_env:
2-
SimpleSokoban:
3-
env_type: sokoban
4-
max_tokens_per_step: ${max_tokens_per_step}
5-
user_prompt_format: ${user_prompt_no_think_format}
6-
added_text: ${added_text_no_think}
7-
env_manager_cls: roll.pipeline.agentic.env_manager.traj_env_manager.TrajEnvManager
8-
use_thread_lock: true
9-
env_config: # keys should be a subset of SokobanConfig
10-
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
11-
action_pattern: ${action_pattern}
12-
max_steps: ${max_actions_per_traj}
13-
dim_x: 6
14-
dim_y: 6
15-
num_boxes: 1
16-
LargerSokoban:
17-
env_type: sokoban
18-
max_tokens_per_step: ${max_tokens_per_step}
19-
user_prompt_format: ${user_prompt_no_think_format}
20-
added_text: ${added_text_no_think}
21-
env_manager_cls: roll.pipeline.agentic.env_manager.traj_env_manager.TrajEnvManager
22-
use_thread_lock: true
23-
env_config:
24-
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
25-
action_pattern: ${action_pattern}
26-
dim_x: 8
27-
dim_y: 8
28-
num_boxes: 2
29-
search_depth: 10
30-
SokobanDifferentGridVocab:
31-
env_type: sokoban
32-
max_tokens_per_step: ${max_tokens_per_step}
33-
user_prompt_format: ${user_prompt_no_think_format}
34-
added_text: ${added_text_no_think}
35-
env_manager_cls: roll.pipeline.agentic.env_manager.traj_env_manager.TrajEnvManager
36-
use_thread_lock: true
37-
env_config: # keys should be a subset of SokobanConfig
38-
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
39-
action_pattern: ${action_pattern}
40-
search_depth: 30
41-
dim_x: 6
42-
dim_y: 6
43-
num_boxes: 1
44-
max_steps: ${max_actions_per_traj}
45-
grid_lookup: { 0: "W", 1: ".", 2: "G", 3: "C", 4: "B", 5: "A", 6: "@" }
46-
grid_vocab: { "W": "wall", ".": "empty", "G": "target", "C": "box on target", "B": "box", "A": "player", "@": "player on target" }
47-
FrozenLake:
48-
env_type: frozen_lake
49-
max_tokens_per_step: ${max_tokens_per_step}
50-
user_prompt_format: ${user_prompt_no_think_format}
51-
added_text: ${added_text_no_think}
52-
env_manager_cls: roll.pipeline.agentic.env_manager.traj_env_manager.TrajEnvManager
53-
use_thread_lock: true
54-
env_config:
55-
env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
56-
action_pattern: ${action_pattern}
57-
is_slippery: false
58-
FrozenLakeThink:
59-
env_type: frozen_lake
60-
max_tokens_per_step: ${max_tokens_per_step}
61-
user_prompt_format: ${user_prompt_think_format}
62-
added_text: ${added_text_think}
63-
env_manager_cls: roll.pipeline.agentic.env_manager.traj_env_manager.TrajEnvManager
64-
use_thread_lock: true
65-
env_config:
66-
env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
67-
action_pattern: ${think_action_pattern}
68-
is_slippery: false

‎examples/config/step_envs.yaml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
action_pattern: ^<answer>(.*?)</answer>$
2+
think_action_pattern: <think>(.*?)</think>\s*<answer>(.*?)</answer>
3+
user_prompt_no_think_format: <answer> [your answer] </answer>
4+
user_prompt_think_format: <think> [Your thoughts] </think> <answer> [your answer] </answer>
5+
6+
max_tokens_per_step: 128
7+
max_actions_per_traj: 10
8+
default_history_length: 5
9+
10+
env_manager_cls: roll.pipeline.agentic.env_manager.step_env_manager.StepEnvManager
11+
custom_env:
12+
SimpleSokoban:
13+
env_type: sokoban
14+
max_tokens_per_step: ${max_tokens_per_step}
15+
user_prompt_format: ${user_prompt_no_think_format}
16+
env_manager_cls: ${env_manager_cls}
17+
use_thread_lock: true
18+
history_length: ${default_history_length}
19+
agent_system_template: ${agent_system_template}
20+
agent_template: ${agent_template}
21+
env_config: # keys should be a subset of SokobanConfig
22+
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
23+
action_pattern: ${action_pattern}
24+
max_steps: ${max_actions_per_traj}
25+
dim_x: 6
26+
dim_y: 6
27+
num_boxes: 1
28+
LargerSokoban:
29+
env_type: sokoban
30+
max_tokens_per_step: ${max_tokens_per_step}
31+
user_prompt_format: ${user_prompt_no_think_format}
32+
env_manager_cls: roll.pipeline.agentic.env_manager.step_env_manager.StepEnvManager
33+
use_thread_lock: true
34+
history_length: ${default_history_length}
35+
agent_system_template: ${agent_system_template}
36+
agent_template: ${agent_template}
37+
env_config:
38+
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
39+
action_pattern: ${action_pattern}
40+
max_steps: ${max_actions_per_traj}
41+
dim_x: 8
42+
dim_y: 8
43+
num_boxes: 2
44+
search_depth: 10
45+
SokobanDifferentGridVocab:
46+
env_type: sokoban
47+
max_tokens_per_step: ${max_tokens_per_step}
48+
user_prompt_format: ${user_prompt_no_think_format}
49+
env_manager_cls: roll.pipeline.agentic.env_manager.step_env_manager.StepEnvManager
50+
use_thread_lock: true
51+
history_length: ${default_history_length}
52+
agent_system_template: ${agent_system_template}
53+
agent_template: ${agent_template}
54+
env_config: # keys should be a subset of SokobanConfig
55+
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
56+
action_pattern: ${action_pattern}
57+
max_steps: ${max_actions_per_traj}
58+
search_depth: 30
59+
dim_x: 6
60+
dim_y: 6
61+
num_boxes: 1
62+
grid_lookup: { 0: "W", 1: ".", 2: "G", 3: "C", 4: "B", 5: "A", 6: "@" }
63+
grid_vocab: { "W": "wall", ".": "empty", "G": "target", "C": "box on target", "B": "box", "A": "player", "@": "player on target" }
64+
FrozenLake:
65+
env_type: frozen_lake
66+
max_tokens_per_step: ${max_tokens_per_step}
67+
user_prompt_format: ${user_prompt_no_think_format}
68+
env_manager_cls: roll.pipeline.agentic.env_manager.step_env_manager.StepEnvManager
69+
use_thread_lock: true
70+
history_length: ${default_history_length}
71+
agent_system_template: ${agent_system_template}
72+
agent_template: ${agent_template}
73+
env_config:
74+
env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
75+
action_pattern: ${action_pattern}
76+
max_steps: ${max_actions_per_traj}
77+
is_slippery: false
78+
FrozenLakeThink:
79+
env_type: frozen_lake
80+
max_tokens_per_step: ${max_tokens_per_step}
81+
user_prompt_format: ${user_prompt_think_format}
82+
env_manager_cls: roll.pipeline.agentic.env_manager.step_env_manager.StepEnvManager
83+
use_thread_lock: true
84+
history_length: ${default_history_length}
85+
agent_system_template: ${agent_system_template}
86+
agent_template: ${agent_template}
87+
env_config:
88+
env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
89+
action_pattern: ${think_action_pattern}
90+
max_steps: ${max_actions_per_traj}
91+
is_slippery: false
92+
WebShopEnv:
93+
env_type: webshop
94+
max_tokens_per_step: ${max_tokens_per_step}
95+
user_prompt_format: ${user_prompt_no_think_format}
96+
env_manager_cls: ${env_manager_cls}
97+
use_thread_lock: true
98+
history_length: ${default_history_length}
99+
agent_system_template: ${agent_system_template}
100+
agent_template: ${agent_template}
101+
env_config:
102+
observation_mode: text
103+
max_steps: ${max_actions_per_traj}
104+
105+
agent_system_template: |
106+
You're a helpful assistant. You are a good game player. You are aiming to get high reward in the game.
107+
108+
agent_template: |
109+
## Env Instruction: {env_instruction}
110+
111+
## State Description:
112+
Prior to this step, you have completed {step_count} steps.
113+
Recent History: Below are the most recent {history_length} observations, the corresponding actions you took, and the environmental reward feedback:
114+
[{history}]
115+
116+
Current State:
117+
You are currently at step {current_step}. Your current observation is: [{current_observation}]
118+
119+
## Output Format Requirement:
120+
Your response *must* strictly adhere to the following format: <answer> [your answer] </answer>, like <answer> [your answer] </answer>, with no extra text.
121+
Response Length Limit: Your output must not exceed {max_response_length} words (tokens).
122+
123+
Determine the Next Action:

‎examples/config/traj_envs.yaml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
action_pattern: <answer>(.*?)</answer>
2+
think_action_pattern: <think>(.*?)</think>\s*<answer>(.*?)</answer>
3+
user_prompt_no_think_format: <answer> [your answer] </answer>
4+
user_prompt_think_format: <think> [Your thoughts] </think> <answer> [your answer] </answer>
5+
6+
max_tokens_per_step: 128
7+
max_actions_per_traj: 10
8+
9+
env_manager_cls: roll.pipeline.agentic.env_manager.traj_env_manager.TrajEnvManager
10+
custom_env:
11+
SimpleSokoban:
12+
env_type: sokoban
13+
max_tokens_per_step: ${max_tokens_per_step}
14+
user_prompt_format: ${user_prompt_no_think_format}
15+
env_manager_cls: ${env_manager_cls}
16+
use_thread_lock: true
17+
agent_system_template: ${agent_system_template}
18+
agent_template: ${agent_template}
19+
reward_template: ${reward_template}
20+
env_config: # keys should be a subset of SokobanConfig
21+
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
22+
action_pattern: ${action_pattern}
23+
max_steps: ${max_actions_per_traj}
24+
dim_x: 6
25+
dim_y: 6
26+
num_boxes: 1
27+
LargerSokoban:
28+
env_type: sokoban
29+
max_tokens_per_step: ${max_tokens_per_step}
30+
user_prompt_format: ${user_prompt_no_think_format}
31+
env_manager_cls: ${env_manager_cls}
32+
use_thread_lock: true
33+
agent_system_template: ${agent_system_template}
34+
agent_template: ${agent_template}
35+
reward_template: ${reward_template}
36+
env_config:
37+
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
38+
action_pattern: ${action_pattern}
39+
max_steps: ${max_actions_per_traj}
40+
dim_x: 8
41+
dim_y: 8
42+
num_boxes: 2
43+
search_depth: 10
44+
SokobanDifferentGridVocab:
45+
env_type: sokoban
46+
max_tokens_per_step: ${max_tokens_per_step}
47+
user_prompt_format: ${user_prompt_no_think_format}
48+
env_manager_cls: ${env_manager_cls}
49+
use_thread_lock: true
50+
agent_system_template: ${agent_system_template}
51+
agent_template: ${agent_template}
52+
reward_template: ${reward_template}
53+
env_config: # keys should be a subset of SokobanConfig
54+
env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
55+
action_pattern: ${action_pattern}
56+
max_steps: ${max_actions_per_traj}
57+
search_depth: 30
58+
dim_x: 6
59+
dim_y: 6
60+
num_boxes: 1
61+
grid_lookup: { 0: "W", 1: ".", 2: "G", 3: "C", 4: "B", 5: "A", 6: "@" }
62+
grid_vocab: { "W": "wall", ".": "empty", "G": "target", "C": "box on target", "B": "box", "A": "player", "@": "player on target" }
63+
FrozenLake:
64+
env_type: frozen_lake
65+
max_tokens_per_step: ${max_tokens_per_step}
66+
user_prompt_format: ${user_prompt_no_think_format}
67+
env_manager_cls: ${env_manager_cls}
68+
use_thread_lock: true
69+
agent_system_template: ${agent_system_template}
70+
agent_template: ${agent_template}
71+
reward_template: ${reward_template}
72+
env_config:
73+
env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
74+
action_pattern: ${action_pattern}
75+
max_steps: ${max_actions_per_traj}
76+
is_slippery: false
77+
FrozenLakeThink:
78+
env_type: frozen_lake
79+
max_tokens_per_step: ${max_tokens_per_step}
80+
user_prompt_format: ${user_prompt_think_format}
81+
env_manager_cls: ${env_manager_cls}
82+
use_thread_lock: true
83+
agent_system_template: ${agent_system_template}
84+
agent_template: ${agent_template}
85+
reward_template: ${reward_template}
86+
env_config:
87+
env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
88+
action_pattern: ${think_action_pattern}
89+
max_steps: ${max_actions_per_traj}
90+
is_slippery: false
91+
WebShopEnv:
92+
env_type: webshop
93+
max_tokens_per_step: ${max_tokens_per_step}
94+
user_prompt_format: ${user_prompt_no_think_format}
95+
env_manager_cls: ${env_manager_cls}
96+
use_thread_lock: true
97+
agent_system_template: ${agent_system_template}
98+
agent_template: ${agent_template}
99+
reward_template: ${reward_template}
100+
env_config:
101+
observation_mode: text
102+
max_steps: ${max_actions_per_traj}
103+
104+
agent_system_template: |
105+
You're a helpful assistant. You are a good game player. You are aiming to get high reward in the game.
106+
agent_template: |
107+
Turn {turn_idx}:
108+
State:
109+
{state}
110+
You have {actions_left} actions left.
111+
Always output: <answer> [your answer] </answer> with no extra text. Strictly follow this format.
112+
Max response length: {max_response_length} words (tokens).
113+
Decide the next action:
114+
115+
reward_template: "Reward:\n{reward}\n"

0 commit comments

Comments
 (0)