# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/decontamination_sft_model_filter_0320_correct_synthesizable_r1_system_prompt_filter_codev_dataset_165k_v3.jsonl --local_dir data/codev/v1/3.1k_r1_filtered --train_size 3033 --test_size 100
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/sft_model_filter_0320_correct_synthesizable_r1_system_prompt_filter_codev_dataset_165k_v3.jsonl --local_dir data/codev/v1/3.4k_r1_filtered --train_size 3306 --test_size 100
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/0320_correct_synthesizable_r1_system_prompt_filter_codev_dataset_165k_v3.jsonl --local_dir data/codev/v1/4.8k_r1 --train_size 4776 --test_size 100
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/decontamination_sft_model_filter_4.8k_and_qwen_32b_correct1234_system_prompt_codev_dataset_v3.jsonl --local_dir data/codev/v1/16k_r1_filtered --train_size 15491 --test_size 500
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/zhangxiaoyun/CodeV-o1/data/sft/codev_o1_qwq_output.jsonl --local_dir data/codev/v1/1.6k --train_size 1614 --test_size 100
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/zhangxiaoyun/CodeV-o1/data/sft/codev_o1_qwq_10k.jsonl --local_dir data/codev/v1/10k_qwq --train_size 9740 --test_size 500
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/error_rate_l_0.2_from_87k_and_decontamination_qwen_32b_correct_1234.jsonl --local_dir data/codev/v1/err_l0.2_32k_r1_qwen --train_size 32064 --test_size 500
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/sft_model_filter_error_rate_l_0.2_from_87k_and_decontamination_qwen_32b_correct_1234.jsonl --local_dir data/codev/v1/err_l0.2_28k_r1_qwen_filtered --train_size 28161 --test_size 500
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/error_rate_l_0.2_from_87k.jsonl --local_dir data/codev/v1/err_l0.2_20k_r1 --train_size 20067 --test_size 500
# python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/sft_model_filter_error_rate_l_0.2_from_87k.jsonl --local_dir data/codev/v1/err_l0.2_16k_r1_filtered --train_size 16364 --test_size 300
python examples/data_preprocess/codev.py --data_path /nfs_global/S/lvhanqi/codev_data/sft_model_filter_error_rate_l_0.2_from_87k.jsonl --local_dir data/codev/v1/err_l0.2_16k_r1_filtered_double_gt --double_gt --train_size 16364 --test_size 300
