add reward_mapping into reward function and add permission

recipe/dapo/dapo_7b_test.sh

@@ -110,7 +110,8 @@ python3 -m verl.trainer.main_ppo \
     custom_reward_function.path=verl/utils/reward_score/codev.py \
     custom_reward_function.name=compute_score_wrapper \
     custom_reward_function.continuous_reward.enable=True \
-    custom_reward_function.continuous_reward.error_ratio_threshold=0.5 \
+    custom_reward_function.continuous_reward.err_threshold=0.5 \
+    custom_reward_function.continuous_reward.reward_mapping='zero' \
     algorithm.kl_ctrl.kl_coef=0.0 \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \

recipe/dapo/run_dapo_codev_7b_16k_err_l0.2_continuous_reward.sh → recipe/dapo/run_dapo_codev_7b_16k_err_l0.2_continuous_reward_threshold.sh

@@ -110,12 +110,12 @@ python3 -m verl.trainer.main_ppo \
     custom_reward_function.path=verl/utils/reward_score/codev.py \
     custom_reward_function.name=compute_score_wrapper \
     custom_reward_function.continuous_reward.enable=True \
-    custom_reward_function.continuous_reward.error_ratio_threshold=0.2 \
+    custom_reward_function.continuous_reward.err_threshold=0.2 \
     algorithm.kl_ctrl.kl_coef=0.0 \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='codev' \
-    trainer.experiment_name='codev-7b-16k' \
+    trainer.experiment_name='codev-7b-16k-err0.2-threshold' \
     trainer.n_gpus_per_node=$USER_GPUS_PER_NODE \
     trainer.nnodes=$SLURM_JOB_NUM_NODES \
     +trainer.val_before_train=False \

recipe/dapo/run_dapo_codev_7b_16k_err_l0.2_continuous_reward_zero.sh

+#!/bin/bash
+set -x
+set -euxo pipefail
+
+project_name='DAPO'
+exp_name='DAPO-Early-Qwen2.5-32B'
+
+adv_estimator=grpo
+
+kl_coef=0.0
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# An early version for DAPO
+enable_filter_groups=False
+gen_prompt_bsz=512 # NOTE: no filtering here
+train_prompt_bsz=512
+train_prompt_mini_bsz=32
+n_resp_per_prompt=16
+use_token_level_loss=False
+
+# Ray
+RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+NNODES=${NNODES:-16}
+# Paths
+# Algorithm
+## Train
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 20))
+## Validation
+val_top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+
+
+# Performance Related Parameter
+sp_size=8
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$((max_prompt_length + max_response_length))
+infer_ppo_max_token_len=$((max_prompt_length + max_response_length))
+offload=True
+gen_tp=4
+
+
+
+
+export VLLM_USE_V1=1
+
+echo "$WANDB_DIR"
+echo "$SAVE_DIR"
+echo "$WANDB_API_KEY"
+
+
+# Set default model path if not provided
+MODEL_PATH="/nfs_global/S/lvhanqi/LLaMA-Factory/saves/Qwen2.5-Coder-7B-Instruct-codev-r1-87k/full/sft_6epoch"
+
+# Train over a single node, 8 A100-80GB GPUs.
+python3 -m verl.trainer.main_ppo \
+    algorithm.adv_estimator=grpo \
+    data.train_files=/nfs_global/S/zhuyaoyu/projects/verl/data/codev/v1/16k_r1_filtered/train.parquet \
+    data.val_files=/nfs_global/S/zhuyaoyu/projects/verl/data/codev/v1/16k_r1_filtered/test.parquet \
+    data.train_batch_size=128 \
+    data.val_batch_size=512 \
+    data.max_prompt_length=2048 \
+    data.max_response_length=16384 \
+    actor_rollout_ref.model.path=$MODEL_PATH \
+    +actor_rollout_ref.model.override_config.attention_dropout=0. \
+    +actor_rollout_ref.model.override_config.embd_pdrop=0. \
+    +actor_rollout_ref.model.override_config.resid_pdrop=0. \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.weight_decay=0.0 \
+    actor_rollout_ref.actor.use_dynamic_bsz=True\
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.ppo_mini_batch_size=64 \
+    actor_rollout_ref.actor.use_kl_loss=True \
+    actor_rollout_ref.actor.kl_loss_coef=0.00 \
+    actor_rollout_ref.actor.kl_loss_type=low_var_kl \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=0.5 \
+    actor_rollout_ref.actor.use_token_level_loss=${use_token_level_loss} \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.fsdp_config.param_offload=False \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=32768 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.n=16 \
+    actor_rollout_ref.rollout.val_kwargs.n=2 \
+    actor_rollout_ref.rollout.temperature=1.0 \
+    actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \
+    actor_rollout_ref.rollout.enforce_eager=False \
+    actor_rollout_ref.rollout.free_cache_engine=False \
+    reward_model.reward_manager=prime \
+    actor_rollout_ref.ref.fsdp_config.param_offload=True \
+    custom_reward_function.overlong_buffer.enable=${enable_overlong_buffer} \
+    custom_reward_function.overlong_buffer.len=${overlong_buffer_len} \
+    custom_reward_function.overlong_buffer.penalty_factor=${overlong_penalty_factor} \
+    custom_reward_function.path=verl/utils/reward_score/codev.py \
+    custom_reward_function.name=compute_score_wrapper \
+    custom_reward_function.continuous_reward.enable=True \
+    custom_reward_function.continuous_reward.err_threshold=0.2 \
+    custom_reward_function.continuous_reward.reward_mapping='zero' \
+    algorithm.kl_ctrl.kl_coef=0.0 \
+    trainer.critic_warmup=0 \
+    trainer.logger=['console','wandb'] \
+    trainer.project_name='codev' \
+    trainer.experiment_name='codev-7b-16k-err0.2-zero' \
+    trainer.n_gpus_per_node=$USER_GPUS_PER_NODE \
+    trainer.nnodes=$SLURM_JOB_NUM_NODES \
+    +trainer.val_before_train=False \
+    trainer.default_local_dir=$SAVE_DIR \
+    trainer.resume_mode=auto \
+    trainer.default_hdfs_dir=null \
+    trainer.save_freq=20 \
+    trainer.test_freq=20 \
+    trainer.total_epochs=100 "${@:1}"
\ No newline at end of file

recipe/dapo/run_dapo_codev_7b_16k_err_l1.0_continuous_reward.sh

@@ -110,12 +110,12 @@ python3 -m verl.trainer.main_ppo \
     custom_reward_function.path=verl/utils/reward_score/codev.py \
     custom_reward_function.name=compute_score_wrapper \
     custom_reward_function.continuous_reward.enable=True \
-    custom_reward_function.continuous_reward.error_ratio_threshold=1.0 \
+    custom_reward_function.continuous_reward.err_threshold=1.0 \
     algorithm.kl_ctrl.kl_coef=0.0 \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='codev' \
-    trainer.experiment_name='codev-7b-16k' \
+    trainer.experiment_name='codev-7b-16k-err1.0' \
     trainer.n_gpus_per_node=$USER_GPUS_PER_NODE \
     trainer.nnodes=$SLURM_JOB_NUM_NODES \
     +trainer.val_before_train=False \

results/dapo_7b_test/dapo_7b_test.sh

+#!/bin/bash
+set -x
+set -euxo pipefail
+
+project_name='DAPO'
+exp_name='DAPO-Early-Qwen2.5-32B'
+
+adv_estimator=grpo
+
+kl_coef=0.0
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 1))
+overlong_penalty_factor=1.0
+
+# An early version for DAPO
+enable_filter_groups=False
+gen_prompt_bsz=512 # NOTE: no filtering here
+train_prompt_bsz=512
+train_prompt_mini_bsz=32
+n_resp_per_prompt=16
+use_token_level_loss=False
+
+# Ray
+RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+NNODES=${NNODES:-16}
+# Paths
+# Algorithm
+## Train
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 20))
+## Validation
+val_top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+
+
+# Performance Related Parameter
+sp_size=8
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$((max_prompt_length + max_response_length))
+infer_ppo_max_token_len=$((max_prompt_length + max_response_length))
+offload=True
+gen_tp=4
+
+
+
+
+export VLLM_USE_V1=1
+
+echo "$WANDB_DIR"
+echo "$SAVE_DIR"
+echo "$WANDB_API_KEY"
+
+
+# Set default model path if not provided
+MODEL_PATH="/share/collab/codemodel/models/Qwen2.5-Coder-7B-Instruct"
+
+# Train over a single node, 8 A100-80GB GPUs.
+python3 -m verl.trainer.main_ppo \
+    algorithm.adv_estimator=grpo \
+    data.train_files=/nfs_global/S/zhuyaoyu/projects/verl/data/codev/v1/1.6k/train.parquet \
+    data.val_files=/nfs_global/S/zhuyaoyu/projects/verl/data/codev/v1/1.6k/test.parquet \
+    data.train_batch_size=128 \
+    data.val_batch_size=512 \
+    data.max_prompt_length=1024 \
+    data.max_response_length=3072 \
+    actor_rollout_ref.model.path=$MODEL_PATH \
+    +actor_rollout_ref.model.override_config.attention_dropout=0. \
+    +actor_rollout_ref.model.override_config.embd_pdrop=0. \
+    +actor_rollout_ref.model.override_config.resid_pdrop=0. \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.weight_decay=0.0 \
+    actor_rollout_ref.actor.use_dynamic_bsz=True\
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=12000 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.ppo_mini_batch_size=64 \
+    actor_rollout_ref.actor.use_kl_loss=True \
+    actor_rollout_ref.actor.kl_loss_coef=0.00 \
+    actor_rollout_ref.actor.kl_loss_type=low_var_kl \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=0.5 \
+    actor_rollout_ref.actor.use_token_level_loss=${use_token_level_loss} \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.fsdp_config.param_offload=False \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.n=8 \
+    actor_rollout_ref.rollout.val_kwargs.n=2 \
+    actor_rollout_ref.rollout.temperature=1.0 \
+    actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \
+    actor_rollout_ref.rollout.enforce_eager=False \
+    actor_rollout_ref.rollout.free_cache_engine=False \
+    reward_model.reward_manager=prime \
+    actor_rollout_ref.ref.fsdp_config.param_offload=True \
+    custom_reward_function.overlong_buffer.enable=${enable_overlong_buffer} \
+    custom_reward_function.overlong_buffer.len=${overlong_buffer_len} \
+    custom_reward_function.overlong_buffer.penalty_factor=${overlong_penalty_factor} \
+    custom_reward_function.path=verl/utils/reward_score/codev.py \
+    custom_reward_function.name=compute_score_wrapper \
+    custom_reward_function.continuous_reward.enable=True \
+    custom_reward_function.continuous_reward.err_threshold=0.5 \
+    custom_reward_function.continuous_reward.reward_mapping='zero' \
+    algorithm.kl_ctrl.kl_coef=0.0 \
+    trainer.critic_warmup=0 \
+    trainer.logger=['console','wandb'] \
+    trainer.project_name='codev' \
+    trainer.experiment_name='codev-7b-16k' \
+    trainer.n_gpus_per_node=$USER_GPUS_PER_NODE \
+    trainer.nnodes=$SLURM_JOB_NUM_NODES \
+    +trainer.val_before_train=False \
+    trainer.default_local_dir=$SAVE_DIR \
+    trainer.resume_mode=auto \
+    trainer.default_hdfs_dir=null \
+    trainer.save_freq=20 \
+    trainer.test_freq=20 \
+    trainer.total_epochs=100 "${@:1}"
+
+    
+    # custom_reward_function.path=/nfs_global/S/zhuyaoyu/projects/dapo/verl/utils/reward_score/codev.py \
\ No newline at end of file

results/run_dapo_codev_7b_16k_continuous_reward_0.2_zero/run_dapo_codev_7b_16k_err_l0.2_continuous_reward_zero.sh

+#!/bin/bash
+set -x
+set -euxo pipefail
+
+project_name='DAPO'
+exp_name='DAPO-Early-Qwen2.5-32B'
+
+adv_estimator=grpo
+
+kl_coef=0.0
+kl_loss_coef=0.0
+
+clip_ratio_low=0.2
+clip_ratio_high=0.28
+
+enable_overlong_buffer=True
+overlong_buffer_len=$((1024 * 4))
+overlong_penalty_factor=1.0
+
+# An early version for DAPO
+enable_filter_groups=False
+gen_prompt_bsz=512 # NOTE: no filtering here
+train_prompt_bsz=512
+train_prompt_mini_bsz=32
+n_resp_per_prompt=16
+use_token_level_loss=False
+
+# Ray
+RAY_ADDRESS=${RAY_ADDRESS:-"http://localhost:8265"}
+WORKING_DIR=${WORKING_DIR:-"${PWD}"}
+RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/trainer/runtime_env.yaml"}
+NNODES=${NNODES:-16}
+# Paths
+# Algorithm
+## Train
+max_prompt_length=$((1024 * 2))
+max_response_length=$((1024 * 20))
+## Validation
+val_top_k=-1 # 0 for HF rollout, -1 for vLLM rollout
+
+
+# Performance Related Parameter
+sp_size=8
+use_dynamic_bsz=True
+actor_ppo_max_token_len=$((max_prompt_length + max_response_length))
+infer_ppo_max_token_len=$((max_prompt_length + max_response_length))
+offload=True
+gen_tp=4
+
+
+
+
+export VLLM_USE_V1=1
+
+echo "$WANDB_DIR"
+echo "$SAVE_DIR"
+echo "$WANDB_API_KEY"
+
+
+# Set default model path if not provided
+MODEL_PATH="/nfs_global/S/lvhanqi/LLaMA-Factory/saves/Qwen2.5-Coder-7B-Instruct-codev-r1-87k/full/sft_6epoch"
+
+# Train over a single node, 8 A100-80GB GPUs.
+python3 -m verl.trainer.main_ppo \
+    algorithm.adv_estimator=grpo \
+    data.train_files=/nfs_global/S/zhuyaoyu/projects/verl/data/codev/v1/16k_r1_filtered/train.parquet \
+    data.val_files=/nfs_global/S/zhuyaoyu/projects/verl/data/codev/v1/16k_r1_filtered/test.parquet \
+    data.train_batch_size=128 \
+    data.val_batch_size=512 \
+    data.max_prompt_length=2048 \
+    data.max_response_length=16384 \
+    actor_rollout_ref.model.path=$MODEL_PATH \
+    +actor_rollout_ref.model.override_config.attention_dropout=0. \
+    +actor_rollout_ref.model.override_config.embd_pdrop=0. \
+    +actor_rollout_ref.model.override_config.resid_pdrop=0. \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.optim.weight_decay=0.0 \
+    actor_rollout_ref.actor.use_dynamic_bsz=True\
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \
+    actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \
+    actor_rollout_ref.actor.ppo_mini_batch_size=64 \
+    actor_rollout_ref.actor.use_kl_loss=True \
+    actor_rollout_ref.actor.kl_loss_coef=0.00 \
+    actor_rollout_ref.actor.kl_loss_type=low_var_kl \
+    actor_rollout_ref.actor.entropy_coeff=0 \
+    actor_rollout_ref.actor.grad_clip=0.5 \
+    actor_rollout_ref.actor.use_token_level_loss=${use_token_level_loss} \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.fsdp_config.param_offload=False \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=32768 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.n=16 \
+    actor_rollout_ref.rollout.val_kwargs.n=2 \
+    actor_rollout_ref.rollout.temperature=1.0 \
+    actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \
+    actor_rollout_ref.rollout.val_kwargs.do_sample=True \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \
+    actor_rollout_ref.rollout.enforce_eager=False \
+    actor_rollout_ref.rollout.free_cache_engine=False \
+    reward_model.reward_manager=prime \
+    actor_rollout_ref.ref.fsdp_config.param_offload=True \
+    custom_reward_function.overlong_buffer.enable=${enable_overlong_buffer} \
+    custom_reward_function.overlong_buffer.len=${overlong_buffer_len} \
+    custom_reward_function.overlong_buffer.penalty_factor=${overlong_penalty_factor} \
+    custom_reward_function.path=verl/utils/reward_score/codev.py \
+    custom_reward_function.name=compute_score_wrapper \
+    custom_reward_function.continuous_reward.enable=True \
+    custom_reward_function.continuous_reward.err_threshold=0.2 \
+    custom_reward_function.continuous_reward.reward_mapping='zero' \
+    algorithm.kl_ctrl.kl_coef=0.0 \
+    trainer.critic_warmup=0 \
+    trainer.logger=['console','wandb'] \
+    trainer.project_name='codev' \
+    trainer.experiment_name='codev-7b-16k-err0.2-zero' \
+    trainer.n_gpus_per_node=$USER_GPUS_PER_NODE \
+    trainer.nnodes=$SLURM_JOB_NUM_NODES \
+    +trainer.val_before_train=False \
+    trainer.default_local_dir=$SAVE_DIR \
+    trainer.resume_mode=auto \
+    trainer.default_hdfs_dir=null \
+    trainer.save_freq=20 \
+    trainer.test_freq=20 \
+    trainer.total_epochs=100 "${@:1}"
\ No newline at end of file

slurm_submit.py

@@ -116,9 +116,9 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Submit a Slurm job with specified parameters.")
 
     # 添加命令行参数
-    parser.add_argument("--node_count", type=int, default=1, help="Number of nodes required.")
+    parser.add_argument("--node_count", type=int, default=2, help="Number of nodes required.")
     parser.add_argument("--gpus_per_node", type=int, default=8, help="Number of GPUs per node (4 or 8).")
-    parser.add_argument("--node_type", type=str, default="r8l40", help="Node type (r8l40/r8l40s/r8a100).")
+    parser.add_argument("--node_type", type=str, default="r8l40s", help="Node type (r8l40/r8l40s/r8a100).")
     parser.add_argument("--partition", type=str, default=None, help="Partition name. (r8nv-gpu-dedicated needs to be specified)")
     parser.add_argument("--qos", type=str, default=None, help="QOS type. (gpu-long needs to be specified)")
 

verl/trainer/config/ppo_trainer.yaml

@@ -180,7 +180,8 @@ custom_reward_function:
     log: False
   continuous_reward:
     enable: False
-    error_ratio_threshold: 0.0 # between 0 and 1, set to 0 is equivalent to discrete reward; set to 1 lets all reward pass
+    err_threshold: 0.0 # between 0 and 1, set to 0 is equivalent to discrete reward; set to 1 lets all reward pass
+    reward_mapping: 'threshold' # 'threshold' or 'zero'
 
 algorithm:
   gamma: 1.0

verl/trainer/main_ppo.py

@@ -155,7 +155,8 @@ class TaskRunner:
         print(compute_score)
         if config.custom_reward_function.continuous_reward.enable and compute_score:
             import functools
-            compute_score = functools.partial(compute_score, reward_mode='continuous', error_ratio_threshold=config.custom_reward_function.continuous_reward.error_ratio_threshold)
+            cfg = config.custom_reward_function.continuous_reward
+            compute_score = functools.partial(compute_score, reward_mode='continuous', err_threshold=cfg.err_threshold, reward_mapping=cfg.reward_mapping)
         reward_fn = reward_manager_cls(tokenizer=tokenizer,
                                     num_examine=1,
                                     compute_score=compute_score,

verl/utils/reward_score/codev.py

@@ -74,15 +74,19 @@ def compute_score_618832(solution_str, ground_truth, exceed_length=False):
     return reward
 
 
-def compute_score(solution_str, ground_truth, reward_mode='discrete', error_ratio_threshold=None):
+def compute_score(solution_str, ground_truth, **kwargs):
+    reward_mode = kwargs.get('reward_mode', 'discrete')
+    err_threshold = kwargs.get('err_threshold', None)
+    reward_mapping = kwargs.get('reward_mapping', None)
     # if isinstance(ground_truth, dict):
     #     reward_mode = ground_truth.get("reward_mode", "discrete")
-    #     error_ratio_threshold = ground_truth.get("error_ratio_threshold", 1.0)
+    #     err_threshold = ground_truth.get("err_threshold", 1.0)
     #     ground_truth = ground_truth.get("answer", "")
     # else:
     #     reward_mode = "discrete"
     assert reward_mode in ['discrete', 'continuous'], "mode should be either 'discrete' or 'continuous'"
-    assert reward_mode != 'continuous' or error_ratio_threshold is not None, "error_ratio_threshold should be given when using continuous reward!"
+    assert reward_mode != 'continuous' or err_threshold is not None, "err_threshold should be given when using continuous reward!"
+    assert reward_mapping in ['threshold', 'zero'], "reward_mapping should be either 'threshold' or 'zero'"
     # print("Reward mode is:", reward_mode)
     # model_output= re.sub(r'^.*?<\|im_start\|>assistant', '<|im_start|>assistant', model_output, flags=re.DOTALL,count = 1)
     # print("compute_score for codev is called !!!")
@@ -110,18 +114,18 @@ def compute_score(solution_str, ground_truth, reward_mode='discrete', error_rati
     else:
         result = verify_one_sample_wrapper((ground_truth, extracted_answer))
         # print("result is", result)
-        if reward_mode == 'discrete':
-            if result["correct"] == True:
-                reward = 1.0
-            else:
-                reward = 0.0
+        if result["correct"] == True:
+            reward = 1.0
         else:
-            # GRPO对比单题用error_rate应该问题不大，别的算法不好说
-            if 'error_rate' in result:
-                reward = 1 - result['error_rate'] if result['error_rate'] <= error_ratio_threshold else 0.0
-                # print('Error rate is', result['error_rate'])
-            else:
+            if reward_mode == 'discrete':
                 reward = 0.0
+            else:
+                # GRPO对比单题用error_rate应该问题不大，别的算法不好说
+                if 'error_rate' in result and result['error_rate'] <= err_threshold:
+                    reward = 1 - result['error_rate'] if reward_mapping == 'threshold' else 1 - err_threshold - result['error_rate']
+                    # print('Error rate is', result['error_rate'])
+                else:
+                    reward = 0.0
         
         # if "test_error" in result:
         #     print("=============test error=============")
@@ -136,8 +140,8 @@ def compute_score(solution_str, ground_truth, reward_mode='discrete', error_rati
     return reward
 
 
-def compute_score_wrapper(data_source, solution_str, ground_truth, extra_info, reward_mode='discrete', error_ratio_threshold=None):
-    return compute_score(solution_str, ground_truth, reward_mode, error_ratio_threshold)
+def compute_score_wrapper(data_source, solution_str, ground_truth, extra_info, **kwargs):
+    return compute_score(solution_str, ground_truth, **kwargs)
 
 
 if __name__ == '__main__':