(fix): update demo yaml configs

Author: jingyushen

examples/agentic_demo/agent_val_frozen_lake_multi_nodes_demo.yaml

@@ -1,5 +1,5 @@
 defaults:
-  - ../config/envs@_here_
+  - ../config/traj_envs@_here_
   - ../config/deepspeed_zero@_here_
   - ../config/deepspeed_zero2@_here_
   - ../config/deepspeed_zero3@_here_
@@ -14,7 +14,7 @@ exp_name: "agentic_pipeline"
 seed: 42
 logging_dir: ./output/logs
 output_dir: ./output
-render_save_dir: /data/oss_bucket_0/yali/output/render
+render_save_dir: ./output/render
 system_envs:
   USE_MODELSCOPE: '1'
 
@@ -29,10 +29,13 @@ system_envs:
 #    - roll
 #    - baseline
 
-
 track_with: tensorboard
 tracker_kwargs:
-  log_dir: /data/oss_bucket_0/yali/llm/tensorboard/roll_exp/agentic_sokoban
+  log_dir: /data/tensorboard/roll_exp/agentic_frozen_lake
+
+checkpoint_config:
+  type: file_system
+  output_dir: /data/cpfs_0/rl_examples/models/${exp_name}  
 
 num_gpus_per_node: 2
 
@@ -46,15 +49,15 @@ rollout_batch_size: 64
 val_batch_size: 16
 sequence_length: 4096
 
-reward_clip: 20
-advantage_clip: 10.0
+advantage_clip: 0.2
 ppo_epochs: 1
-adv_estimator: "reinforce"
+adv_estimator: "grpo"
 #pg_clip: 0.1
 #dual_clip_loss: True
 init_kl_coef: 0.0
 whiten_advantages: true
 entropy_loss_coef: 0
+max_grad_norm: 1.0
 
 pretrain: Qwen/Qwen2.5-0.5B-Instruct
 reward_pretrain: Qwen/Qwen2.5-0.5B-Instruct
@@ -71,6 +74,7 @@ actor_train:
     per_device_train_batch_size: 1
     gradient_accumulation_steps: 16
     warmup_steps: 10
+    lr_scheduler_type: cosine
   data_args:
     template: qwen2_5
   strategy_args:
@@ -123,71 +127,34 @@ reference:
   device_mapping: list(range(0,4))
   infer_batch_size: 1
 
-enable_response_mask: True
-action_sep: "||"
-use_turn_scores: False # important to GAE when applying token-level rewards to token-level advantages. If False, will take the sum of scores as the reward for the last turn.
-enable_think: False # False -> no think RL
-max_actions_per_traj: 10
 reward_normalization:
   grouping: tags # 可以tags(env_type)/traj_group_id(group)/batch(rollout_batch)... group_by计算reward/adv
   method: identity # asym_clip / identity / mean_std
 
-custom_envs:
-  SimpleSokoban:
-    env_type: sokoban
-    max_actions_per_traj:  ${max_actions_per_traj} # used in environment state manager to control the actual max actions executed per trajectory
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100 # used to curate llm prompt "max words", not used for rollout
-    env_config: # keys should be a subset of SokobanConfig
-      dim_x: 6
-      dim_y: 6
-      num_boxes: 1
-      max_steps: ${max_actions_per_traj}
-  LargerSokoban:
-    env_type: sokoban
-    max_actions_per_traj:  ${max_actions_per_traj}
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100
-    env_config:
-      dim_x: 8
-      dim_y: 8
-      num_boxes: 2
-      max_steps: ${max_actions_per_traj}
-      search_depth: 10
-  SokobanDifferentGridVocab:
-    env_type: sokoban
-    max_actions_per_traj:  ${max_actions_per_traj}
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100
-    env_config: # keys should be a subset of SokobanConfig
-      search_depth: 30
-      dim_x: 6
-      dim_y: 6
-      num_boxes: 1
-      max_steps: ${max_actions_per_traj}
-      grid_lookup: { 0: "W", 1: ".", 2: "G", 3: "C", 4: "B", 5: "A", 6: "@" }
-      grid_vocab: { "W": "wall", ".": "empty", "G": "target", "C": "box on target", "B": "box", "A": "player", "@": "player on target" }
-  FrozenLake:
-    env_type: frozen_lake
-    max_actions_per_traj:  ${max_actions_per_traj}
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100
-    env_config:
-      is_slippery: false
-
 train_env_manager:
-  format_penalty: -0.001
-  env_groups: 1
+  format_penalty: -0.15 # sokoban env penalty_for_step=-0.1
+  max_env_num_per_worker: 16
+  num_env_groups: 1
+  # under the same group, the env config and env seed are ensured to be equal
   group_size: 1
   tags: [FrozenLake]
-  n_groups: [1] # If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
+  num_groups_partition: [1] # If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
 
 val_env_manager:
-  env_groups: 2
+  max_env_num_per_worker: 1
+  num_env_groups: 2
   group_size: 1 # should be set to 1 because val temperature is set to 0 and same prompt leads to same output
-  tags: [SimpleSokoban, FrozenLake]
-  n_groups: [1, 1] # TODO: If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
+  tags: [SimpleSokoban, LargerSokoban]
+  num_groups_partition: [1, 1] # TODO: If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
+
+max_tokens_per_step: 64
+
+custom_envs:
+  SimpleSokoban:
+    ${custom_env.SimpleSokoban}
+  LargerSokoban:
+    ${custom_env.LargerSokoban}
+  SokobanDifferentGridVocab:
+    ${custom_env.SokobanDifferentGridVocab}
+  FrozenLake:
+    ${custom_env.FrozenLake}

examples/agentic_demo/agent_val_frozen_lake_single_node_demo.yaml

@@ -1,5 +1,5 @@
 defaults:
-  - ../config/envs@_here_
+  - ../config/traj_envs@_here_
   - ../config/deepspeed_zero@_here_
   - ../config/deepspeed_zero2@_here_
   - ../config/deepspeed_zero3@_here_
@@ -29,10 +29,13 @@ system_envs:
 #    - roll
 #    - baseline
 
-
 track_with: tensorboard
 tracker_kwargs:
-  log_dir: /data/oss_bucket_0/yali/llm/tensorboard/roll_exp/agentic_sokoban
+  log_dir: /data/tensorboard/roll_exp/agentic_frozen_lake
+
+checkpoint_config:
+  type: file_system
+  output_dir: ./output/render
 
 num_gpus_per_node: 1
 
@@ -46,15 +49,15 @@ rollout_batch_size: 16
 val_batch_size: 16
 sequence_length: 4096
 
-reward_clip: 20
-advantage_clip: 10.0
+advantage_clip: 0.2
 ppo_epochs: 1
-adv_estimator: "reinforce"
+adv_estimator: "grpo"
 #pg_clip: 0.1
 #dual_clip_loss: True
 init_kl_coef: 0.0
 whiten_advantages: true
 entropy_loss_coef: 0
+max_grad_norm: 1.0
 
 pretrain: Qwen/Qwen2.5-0.5B-Instruct
 reward_pretrain: Qwen/Qwen2.5-0.5B-Instruct
@@ -71,6 +74,7 @@ actor_train:
     per_device_train_batch_size: 1
     gradient_accumulation_steps: 16
     warmup_steps: 10
+    lr_scheduler_type: cosine
   data_args:
     template: qwen2_5
   strategy_args:
@@ -123,71 +127,34 @@ reference:
   device_mapping: list(range(0,1))
   infer_batch_size: 1
 
-enable_response_mask: True
-action_sep: "||"
-use_turn_scores: False # important to GAE when applying token-level rewards to token-level advantages. If False, will take the sum of scores as the reward for the last turn.
-enable_think: False # False -> no think RL
-max_actions_per_traj: 10
 reward_normalization:
   grouping: tags # 可以tags(env_type)/traj_group_id(group)/batch(rollout_batch)... group_by计算reward/adv
   method: identity # asym_clip / identity / mean_std
 
-custom_envs:
-  SimpleSokoban:
-    env_type: sokoban
-    max_actions_per_traj:  ${max_actions_per_traj} # used in environment state manager to control the actual max actions executed per trajectory
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100 # used to curate llm prompt "max words", not used for rollout
-    env_config: # keys should be a subset of SokobanConfig
-      dim_x: 6
-      dim_y: 6
-      num_boxes: 1
-      max_steps: ${max_actions_per_traj}
-  LargerSokoban:
-    env_type: sokoban
-    max_actions_per_traj:  ${max_actions_per_traj}
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100
-    env_config:
-      dim_x: 8
-      dim_y: 8
-      num_boxes: 2
-      max_steps: ${max_actions_per_traj}
-      search_depth: 10
-  SokobanDifferentGridVocab:
-    env_type: sokoban
-    max_actions_per_traj:  ${max_actions_per_traj}
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the Sokoban puzzle. You are the player and you need to push all boxes to targets. When you are right next to a box, you can push it by moving in the same direction. You cannot push a box through a wall, and you cannot pull a box. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100
-    env_config: # keys should be a subset of SokobanConfig
-      search_depth: 30
-      dim_x: 6
-      dim_y: 6
-      num_boxes: 1
-      max_steps: ${max_actions_per_traj}
-      grid_lookup: { 0: "W", 1: ".", 2: "G", 3: "C", 4: "B", 5: "A", 6: "@" }
-      grid_vocab: { "W": "wall", ".": "empty", "G": "target", "C": "box on target", "B": "box", "A": "player", "@": "player on target" }
-  FrozenLake:
-    env_type: frozen_lake
-    max_actions_per_traj:  ${max_actions_per_traj}
-    max_steps_per_traj: ${max_actions_per_traj}
-    env_instruction: "You are solving the FrozenLake puzzle. Forbid the whole and go to the target. You may move to the unintended direction due to the slippery ice. The answer must be one of action in a turn, format is <answer>Right</answer>"
-    max_tokens: 100
-    env_config:
-      is_slippery: false
-
 train_env_manager:
-  format_penalty: -0.001
-  env_groups: 1
+  format_penalty: -0.15 # sokoban env penalty_for_step=-0.1
+  max_env_num_per_worker: 16
+  num_env_groups: 1
+  # under the same group, the env config and env seed are ensured to be equal
   group_size: 1
   tags: [FrozenLake]
-  n_groups: [1] # If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
+  num_groups_partition: [1] # If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
 
 val_env_manager:
-  env_groups: 2
+  max_env_num_per_worker: 1
+  num_env_groups: 2
   group_size: 1 # should be set to 1 because val temperature is set to 0 and same prompt leads to same output
-  tags: [SimpleSokoban, FrozenLake]
-  n_groups: [1, 1] # TODO: If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
+  tags: [SimpleSokoban, LargerSokoban]
+  num_groups_partition: [1, 1] # TODO: If not set, all env names divide nums equally. Under the same group, the env config and env seed (prompt) are equal in each generation
+
+max_tokens_per_step: 64
+
+custom_envs:
+  SimpleSokoban:
+    ${custom_env.SimpleSokoban}
+  LargerSokoban:
+    ${custom_env.LargerSokoban}
+  SokobanDifferentGridVocab:
+    ${custom_env.SokobanDifferentGridVocab}
+  FrozenLake:
+    ${custom_env.FrozenLake}