fix bugs

codecritic/cli/select_preference_pairs.py

@@ -98,20 +98,13 @@ if __name__ == "__main__":
 
 
     # select pairs
-    ds = defaultdict(dict)
-    for item in dataset:
-        ds[item["task_id"]][item["solution_id"]] = item
-
-    sorted_pairinfo = sorted(pairinfo, key=lambda x: x["similarity"])
     task_groups = defaultdict(list)
     for item in pairinfo:
         task_groups[item["task_id"]].append(item)
 
-    # Step 2: Select the 4 pairs with the smallest score for each task
     selected_pairs = []
     for task, items in task_groups.items():
-        # Sort items for this task by score and select the top 4
-        sorted_items = sorted(items, key=lambda x: x["similarity"])[:4]
+        sorted_items = sorted(items, key=lambda x: x["similarity"], reverse=True)[:4]
         selected_pairs.extend(sorted_items)
 
     save_jsonl(selected_pairs, args.output)

scripts/gen_dataset.sh

@@ -4,15 +4,22 @@ model="/lustre/S/huangdi/open_for_out/models/Qwen2.5-Coder-7B-Instruct/"
 project="/lustre/S/nanziyuan/projects/ccc"
 modelname="qwen25_coder_inst"
 
-# APPS
-# CUDA_VISIBLE_DEVICES=0,1,2,3 \
-python -m codecritic.cli.gen_dataset \
-    --model ${model} \
-    --apps /lustre/S/nanziyuan/datasets/apps/ \
-    --train "${project}/data/train/${modelname}-apps-train.jsonl" \
-    --test "${project}/data/test/${modelname}-apps-test.jsonl"
-
-# HumanEval & MBPP
+trainset="${project}/data/train/${modelname}-apps-train.jsonl"
+testset="${project}/data/test/${modelname}-apps-test.jsonl"
+
+train_selected_pairs="${project}/data/train/${modelname}-apps-train-selected_pairs.jsonl"
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+
+## Sampling
+## APPS
+# python -m codecritic.cli.gen_dataset \
+#     --model ${model} \
+#     --apps /lustre/S/nanziyuan/datasets/apps/ \
+#     --train ${trainset} \
+#     --test ${testset}
+
+## HumanEval & MBPP
 # evalplus.evaluate \
 #     --model ${model} \
 #     --n_samples 50 \
@@ -29,6 +36,11 @@ python -m codecritic.cli.gen_dataset \
 #     --root "${project}/data/test/${modelname}-mbpp" \
 #     --backend vllm
 
-# HumanEvalPack
+## HumanEvalPack
+
+## BigCodeBench
 
-# BigCodeBench
+## Training dataset
+python -m codecritic.cli.select_preference_pairs \
+       --dataset ${trainset} \
+       --output ${train_selected_pairs}