step2: cov_dataset fix bugs

step2_cov_dataset.py

@@ -3,7 +3,7 @@
 import argparse
 from itertools import chain
 from utils import load_json, extract_code, code_template
-from utils_dataset import mk_critic_qa, mk_sft_item, mk_sft_dataset_info, save_dataset, SPLITTER
+from utils_dataset import mk_critic_qa, mk_sft_item, mk_critic_verify, mk_sft_dataset_info, save_dataset, SPLITTER
 from utils_vllm import vllm_chatcomplete
 
 COV_PROMPT = "Please verify your code step by step using Markdown code blocks. After each step, explain whether it's correct or not, and if not, explain the issue."
@@ -32,12 +32,11 @@ result = add_numbers(5, '10')
 **Explanation:** Incorrect. The second argument is a string (`'10'`), which will cause a TypeError when trying to add it to an integer.
 """
 
+CORRECT_PROMPT = "Your code is correct."
+INCORRECT_PROMPT = "Your code is incorrect."
 
 def mk_cov_prompt(is_correct):
-    if is_correct:
-        prompt1 = "Your code is correct."
-    else:
-        prompt1 = "Your code is incorrect."
+    prompt1 = CORRECT_PROMPT if is_correct else INCORRECT_PROMPT
 
     return [{"role": "user", "content": prompt1 + "\n" + COV_PROMPT + '\n' + COV_EXAMPLE},
             {"role": "assistant", "content": "Here's a step-by-step verification of the code." + SPLITTER}]
@@ -57,7 +56,15 @@ def convert_preference_to_vot_prompt(item):
 
 
 def convert_cov_to_cov_dataset(item):
+    user_content = item["messages"][2]["content"]
     item["messages"][2]["content"] = COV_PROMPT
+    if CORRECT_PROMPT in user_content:
+        is_correct = True
+    elif INCORRECT_PROMPT in user_content:
+        is_correct = False
+    else:
+        raise ValueError("Invalid prompt")
+    item["messages"] += mk_critic_verify(is_correct)
     return item