fix bugs in dapo config (no dynamic sampling, no token-level loss)

recipe/dapo/blockelite/dapo_7b_test.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 1))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=1024 \
     data.max_response_length=3072 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/blockelite/run_dapo_codev_7b_3.1k.sh

@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/dapo_7b_test.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 1))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=1024 \
     data.max_response_length=3072 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_16k.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH  \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_16k_err_l0.2_continuous_reward_threshold.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_16k_err_l0.2_continuous_reward_zero.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_16k_err_l1.0_continuous_reward.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_16k_inaccurate_16kdata_err_l0.2_threshold.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_20k_err_l0.2_r1_continuous_reward.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_3.4k.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH  \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_8k.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -69,6 +69,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=16384 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH  \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \

recipe/dapo/run_dapo_codev_7b_8k_terminal.sh

@@ -18,12 +18,12 @@ overlong_buffer_len=$((1024 * 4))
 overlong_penalty_factor=1.0
 
 # An early version for DAPO
-enable_filter_groups=False
+enable_filter_groups=True
 gen_prompt_bsz=512 # NOTE: no filtering here
 train_prompt_bsz=512
 train_prompt_mini_bsz=32
 n_resp_per_prompt=16
-use_token_level_loss=False
+use_token_level_loss=True
 
 # Ray
 RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
@@ -77,6 +77,7 @@ python3 -m verl.trainer.main_ppo \
     data.val_batch_size=512 \
     data.max_prompt_length=2048 \
     data.max_response_length=8192 \
+    algorithm.filter_groups.enable=${enable_filter_groups} \
     actor_rollout_ref.model.path=$MODEL_PATH  \
     +actor_rollout_ref.model.override_config.attention_dropout=0. \
     +actor_rollout_ref.model.override_config.embd_pdrop=0. \