rename data to dataset & prepare apps refactor

codecritic/cli/convert_pairwise_to_sft.py

-# Additional Experiment:
-# As we know, there are two primary methods for training a reward model:
-# 1. Using reward loss
-# 2. Using SFT (Supervised Fine-Tuning) directly
-# This experiment aims to fairly compare these two approaches.
 import argparse
-from codecritic.data.utils import mk_message, save_jsonl_dataset
+from codecritic.dataset.utils import mk_message, save_jsonl_dataset
 from codecritic.utils.json import load_jsonl
-from codecritic.data.verify import mk_critic_verify
+from codecritic.dataset.verify import mk_critic_verify
 
 
 def mk_sft_dataset(messages):
@@ -20,8 +15,6 @@ def convert_preference_to_sft(item):
     chosen = item["chosen"]
     rejected = item["rejected"]
 
-
-
     messages1 = message + chosen + mk_critic_verify(True)
     messages2 = message + rejected + mk_critic_verify(False)
     return mk_sft_dataset(messages1), mk_sft_dataset(messages2)

codecritic/cli/eval.py

+from codecritic.evaluation.metric import group_results, score_pass_at_k
+
+
+def eval():
+    # compute pass@k
+    eval_result_path = result_dir / "passk.jsonl"
+    # results = load_jsonl(score_path)
+    groups = group_results(results, apps_path)
+    eval_results = [score_pass_at_k(groups, k, home_path.stem) for k in range(1, 16)]
+    save_jsonl(eval_results, eval_result_path)
+    pprint.pp(eval_results)
\ No newline at end of file

codecritic/cli/mk_cov_dataset.py

 import argparse
 from itertools import chain
 
-from codecritic.data.cov import (
+from codecritic.dataset.cov import (
     convert_preference_to_vot_prompt,
     convert_cov_to_cov_dataset,
 )
 
 from codecritic.utils.json import load_json
-from codecritic.data.utils import save_jsonl_dataset
+from codecritic.dataset.utils import save_jsonl_dataset
 from codecritic.utils.vllm import vllm_chatcomplete
 
 

codecritic/cli/mk_cov_from_api.py

@@ -8,7 +8,7 @@ from threading import Lock
 from openai import OpenAI
 import json
 
-from codecritic.data.cov import (
+from codecritic.dataset.cov import (
     convert_preference_to_vot_prompt,
     convert_sft_to_vot_prompt,
     convert_cov_to_cov_dataset,

codecritic/cli/mk_rm_dataset.py

 import argparse
 from pathlib import Path
 from codecritic.utils.json import load_json
-from codecritic.data.utils import save_jsonl_dataset
+from codecritic.dataset.utils import save_jsonl_dataset
 
-from codecritic.data.edit_distance import (
+from codecritic.dataset.edit_distance import (
     mk_problem_groups,
     calculate_edit_distances,
     mk_edit_distance_dataset,

codecritic/cli/postprocess_critic_dataset.py

-import argparse
-from pathlib import Path
-import random
-from codecritic.utils.json import load_jsonl, save_jsonl
-
-
-def add_mask_and_score(messages):
-    for idx, turn in enumerate(messages):
-        if idx != 3:
-            turn["mask"] = True
-        else:
-            turn["mask"] = False
-
-        if idx == 5:
-            turn["score"] = True
-    return messages
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--path", type=str)
-    parser.add_argument("--split", action="store_true")
-    args = parser.parse_args()
-
-    dataset = load_jsonl(args.path)
-
-    for item in dataset:
-        item["messages"] = add_mask_and_score(item["messages"])
-
-    if args.split:
-        random.shuffle(dataset)
-        split_len = int(len(dataset) * 0.01)
-        test = dataset[:split_len]
-        train = dataset[split_len:]
-
-        dataset_path = Path(args.path).parent
-        save_jsonl(train, dataset_path / "train.jsonl")
-        save_jsonl(test, dataset_path / "test.jsonl")
-    else:
-        save_jsonl(dataset, args.path)

codecritic/cli/run_qwq.py

-import argparse
-from itertools import chain
-
-from codecritic.data.cov_with_diff import (
-    transform_preference_to_qwq_prompt,
-    transform_qwqout_to_trainset
-)
-
-from codecritic.utils.json import load_jsonl, save_jsonl
-from codecritic.utils.vllm import vllm_chatcomplete
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--model", type=str)
-    parser.add_argument("--preference_dataset", type=str)
-    parser.add_argument("--out", type=str)
-    args = parser.parse_args()
-
-    preference_dataset = load_jsonl(args.preference_dataset)
-    cov_prompts = [transform_preference_to_qwq_prompt(x) for x in preference_dataset]
-    cov_prompts = list(chain(*cov_prompts))
-
-    sampling_params = dict(n=1, temperature=0, max_tokens=6144)
-    covs = vllm_chatcomplete(args.model, cov_prompts, sampling_params, 2)
-    # save_jsonl(covs, args.out + ".raw")
-    dataset = list(map(transform_qwqout_to_trainset, covs))
-
-    save_jsonl(dataset, args.out)

codecritic/cli/test_genrm.py

@@ -5,7 +5,7 @@ import os
 from transformers import AutoTokenizer
 from vllm import SamplingParams
 
-from codecritic.data.genrm_prompt import THINK_PROMPT, JUDGE_PROMPT, JUDGE_TOEKNS
+from codecritic.dataset.genrm_prompt import THINK_PROMPT, JUDGE_PROMPT, JUDGE_TOEKNS
 from codecritic.evaluation.inference import generate_worker, score_worker
 from codecritic.utils.parallel import model_map
 from codecritic.utils.json import load_jsonl, save_jsonl

codecritic/dataset/apps.py

+import json
+
+
+def mk_prompt(doc):
+    prompt = "Write Python code to solve competitive programming problems in a markdown code block."
+
+    starter_code = None if len(doc["starter_code"]) == 0 else doc["starter_code"]
+    try:
+        input_outpout = json.loads(doc["input_output"])
+        fn_name = None if not input_outpout.get("fn_name") else input_outpout["fn_name"]
+    except ValueError:
+        fn_name = None
+    prompt += "\nQUESTION:\n"
+    prompt += doc["question"]
+    if starter_code:
+        prompt += starter_code
+    if not fn_name:
+        prompt += "\nUse Standard Input format"
+    else:
+        prompt += "\nUse Call-Based format"
+
+    prompt += "\nPlease generate the code in a ```python markdown block, ensuring to include the closing ``` at the end."
+
+    conversation = [{"role": "user", "content": prompt}]
+    return conversation

codecritic/data/cov.py → codecritic/dataset/cov.py

 # Additional Experiment:
 # Is reasoning really work? Let's verify step by step.
-from codecritic.data.code import extract_code, code_template
+from codecritic.dataset.code import extract_code, code_template
 
-from codecritic.data.utils import SPLITTER, mk_message
-from codecritic.data.verify import mk_critic_verify
+from codecritic.dataset.utils import SPLITTER, mk_message
+from codecritic.dataset.verify import mk_critic_verify
 
 COV_PROMPT = "Please verify your code step by step using Markdown code blocks. After each step, explain whether it's correct or not, and if not, explain the issue."
 

codecritic/data/cov_with_diff.py → codecritic/dataset/cov_with_diff.py

 from difflib import unified_diff
 import re
 
-from codecritic.data.code import extract_code
-from codecritic.data.verify import mk_critic_verify
+from codecritic.dataset.code import extract_code
+from codecritic.dataset.verify import mk_critic_verify
 
 # QwQ doesn't follow my instruction, but output *really* reasonable explanation
 

codecritic/data/edit_distance.py → codecritic/dataset/edit_distance.py

 from codecritic.utils.json import load_jsonl
-from codecritic.data.code import extract_code, code_template
+from codecritic.dataset.code import extract_code, code_template
 from nltk.metrics.distance import edit_distance
 from collections import defaultdict
 from itertools import product, chain

codecritic/evaluation/apps_eval.py

+# copy from codeparrot/apps_metric/utils.py
+# https://huggingface.co/spaces/codeparrot/apps_metric/blob/main/utils.py
+
+import json
+import multiprocessing
+import numpy as np
+from tqdm.contrib.concurrent import process_map
+
+from codecritic.evaluation.apps_exec import run_test
+from codecritic.utils.json import save_jsonl
+from codecritic.dataset.code import extract_code
+
+TIMEOUT = 10
+
+
+def check_correctness(sample, generation, timeout, debug=False):
+    """Check correctness of code generation with a global timeout.
+    The global timeout is to catch some extreme/rare cases not handled by the timeouts
+    inside `run_test`"""
+
+    def _temp_run(sample, generation, debug, result):
+        result.append(run_test(sample, test=generation, debug=debug))
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+    p = multiprocessing.Process(
+        target=_temp_run, args=(sample, generation, debug, result)
+    )
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+    if not result:
+        in_outs = json.loads(sample["input_output"])
+        # consider that all tests failed
+        result = [[-1 for i in range(len(in_outs["inputs"]))]]
+        if debug:
+            print(f"global timeout")
+    return result[0]
+
+
+def test_generation(args, debug=False):
+    apps_item, sample = args
+    code = extract_code(sample["response"][0]["content"])
+
+    curr_res = [-2]
+    try:
+        curr_res = check_correctness(apps_item, code, timeout=TIMEOUT, debug=debug)
+        if debug:
+            print(f"\nSuccessful compilation of task {code}!")
+        fixed = []
+        for e in curr_res:
+            if isinstance(e, np.ndarray):
+                e = e.item(0)
+            if isinstance(e, np.bool_):
+                e = bool(e)
+            fixed.append(e)
+        curr_res = fixed
+        if not np.all(curr_res):
+            if debug:
+                print(curr_res)
+                print(f"Results were not True for all test cases")
+    except Exception as e:
+        if debug:
+            print(f"Compilation failed, test framework exception = {repr(e)}{e}\n")
+    finally:
+        assert isinstance(curr_res, list)
+        problem_result = np.asarray(curr_res)
+
+    return {
+        **sample,
+        "code": code,
+        "eval_result": bool(np.all(problem_result > 0)),
+        "testcase": curr_res
+    }
+
+
+def evaluate_code_samples(code_samples, apps):
+    args = []
+    for sample in code_samples:
+        problem_id = sample["problem_id"]
+        args.append((apps["test"][int(problem_id)], sample))
+
+    cpu_num = multiprocessing.cpu_count()
+    chunksize = max(len(code_samples) // (cpu_num * 5), 1)
+    # TODO performance?
+    results = process_map(
+        test_generation, args, max_workers=cpu_num, chunksize=chunksize
+    )
+    return results
+
+
+def evaluate_incorrect_code_samples_again(results, apps, loop_num):
+    """
+    There are some strange bugs in apps evaluation that cannot be reproduced.
+    The observable issue is that the same code will yield different 'eval_result' values.
+    Typically, the test framework may encounter an exception or decide that the code has timed out unreasonably.
+
+    This function is an ugly workaround to address this problem:
+    Run twice to verify if the result is consistent.
+    The 'loop_num' parameter controls the number of times the function will be retried until the test framework obtains a consistent result.
+    """
+    maybe_incorrect_lst, correct_lst = [], []
+    for item in results:
+        if any(x in item["testcase"] for x in (-1, -2)):
+            maybe_incorrect_lst.append(item)
+        else:
+            correct_lst.append(item)
+
+    for _ in range(loop_num):
+        if len(maybe_incorrect_lst) == 0:
+            break
+
+        new_results = evaluate_code_samples(maybe_incorrect_lst, apps)
+        print(f"maybe incorrect lst size: {len(maybe_incorrect_lst)}")
+        check_lst = []
+        for i in range(len(new_results)):
+            old_item, new_item = maybe_incorrect_lst[i], new_results[i]
+            old_eval, new_eval = old_item["eval_result"], new_item["eval_result"]
+            if old_eval == new_eval:
+                correct_lst.append(old_item)
+            else:
+                check_lst.append(new_item)
+                print(old_item["problem_id"], old_eval, new_item["problem_id"], new_eval)
+
+        maybe_incorrect_lst = check_lst
+
+    if len(results) != len(correct_lst):
+        save_jsonl(maybe_incorrect_lst, "debug.jsonl")
+        # raise ValueError("cannot correctly evaluate codes")
+        print("cannot correctly evalute code. see debug.jsonl")
+        if len(maybe_incorrect_lst) < 5:
+            correct_lst.extend(maybe_incorrect_lst)
+
+    return correct_lst
+
+
+def evaluate(code_samples, apps):
+    results = evaluate_code_samples(code_samples, apps)
+    results = evaluate_incorrect_code_samples_again(results, apps, 10)
+    return results

codecritic/evaluation/apps_exec.py

+# copy from codeparrot/apps_metric/testing_util.py
+# https://huggingface.co/spaces/codeparrot/apps_metric/blob/main/testing_util.py
+
+# Log: Replace pyext with importlib 
+
+import json
+import sys
+import faulthandler
+import platform
+
+# used for debugging to time steps
+from datetime import datetime
+
+# to run the solution files we're using a timing based approach
+import signal
+
+import numpy as np
+# for capturing the stdout
+from io import StringIO
+# used for testing the code that reads from input
+from unittest.mock import patch, mock_open
+
+import importlib
+
+def import_module_from_string(module_name: str, source: str):
+    spec = importlib.util.spec_from_loader(module_name, loader=None)
+    module = importlib.util.module_from_spec(spec)
+    exec(source, module.__dict__)
+    return module
+
+from enum import Enum
+class CODE_TYPE(Enum):
+    call_based = 0
+    standard_input = 1
+
+# stuff for setting up signal timer
+class TimeoutException(Exception):
+    pass
+def timeout_handler(signum, frame):
+    print("alarm went off")
+    #return
+    raise TimeoutException
+signal.signal(signal.SIGALRM, timeout_handler)
+timeout = 4  # seconds
+
+# used to capture stdout as a list
+# from https://stackoverflow.com/a/16571630/6416660
+# alternative use redirect_stdout() from contextlib
+class Capturing(list):
+    def __enter__(self):
+        self._stdout = sys.stdout
+        sys.stdout = self._stringio = StringIO()
+        # Make closing the StringIO a no-op
+        self._stringio.close = lambda x: 1
+        return self
+    def __exit__(self, *args):
+        self.extend(self._stringio.getvalue().splitlines())
+        del self._stringio    # free up some memory
+        sys.stdout = self._stdout
+
+
+def run_test(sample, test=None, debug=False):
+    """
+    if test(generated_code) is not None it'll try to run the code.
+    otherwise it'll just return an input and output pair.
+    """
+    # Disable functionalities that can make destructive changes to the test.
+    reliability_guard()
+    
+    if debug:
+        print(f"start = {datetime.now().time()}")
+
+    try:
+        in_outs = json.loads(sample["input_output"])
+    except ValueError:
+        in_outs = None
+    if in_outs:
+        if in_outs.get("fn_name") is None:
+            which_type = CODE_TYPE.standard_input  # Standard input
+            method_name = None
+        else:
+            which_type = CODE_TYPE.call_based  # Call-based
+            method_name = in_outs["fn_name"]
+
+    if debug:
+        print(f"loaded input_output = {datetime.now().time()}")
+ 
+    if test is None:
+        return in_outs
+    elif test is not None:
+        results = []
+        sol = "import sys\nimport time\nimport itertools\nfrom itertools import accumulate, product, permutations, combinations\nimport collections\nfrom collections import Counter, OrderedDict, deque, defaultdict, ChainMap\nfrom functools import lru_cache\nimport math\nfrom math import sqrt, sin, cos, tan, ceil, fabs, floor, gcd, exp, log, log2\nimport fractions\nfrom typing import List, Tuple\nimport numpy as np\nimport random\nimport heapq\nfrom heapq import *\n"
+        if debug:
+            print(f"loading test code = {datetime.now().time()}")
+ 
+        if which_type == CODE_TYPE.call_based:
+            sol += test
+            if debug:
+                print(f"sol = {sol}")
+            signal.alarm(timeout)
+            try:
+                tmp_sol = import_module_from_string("tmp_sol", sol)
+                if "class Solution" not in test:
+                    tmp = tmp_sol
+                else:
+                    tmp = tmp_sol.Solution()
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                     print(f"type 0 compilation error = {e}")
+                results.append(-2)
+                return results
+            signal.alarm(0)
+
+        elif which_type == CODE_TYPE.standard_input:
+            # sol
+            tmp_test = test.split("\n")
+
+            new_test = []
+            for x in tmp_test:
+                if (not x.startswith("from ")) and (not x.startswith("import ")):
+                    new_test.append("\t" + x + "\n")
+                else:
+                    new_test.append(x + "\n")
+            tmp_test = new_test
+            
+            new_test = ""
+            started = False
+            for i in tmp_test:
+                if i.startswith("\t") and not started:
+                    new_test += "stdin = sys.stdin\nstdout = sys.stdout\n"
+                    new_test += "def code():\n"
+                    new_test += i
+                    started = True
+                elif started and ((i.startswith("from ")) or (i.startswith("import "))): 
+                    new_test += "\t" + i
+                else:
+                    new_test += i
+            tmp_test = new_test
+
+            sol += tmp_test
+            if debug:
+                print(f"sol = {sol}")
+            method_name = "code"
+            signal.alarm(timeout)
+            try:
+                tmp_sol = import_module_from_string("tmp_sol", sol)
+                tmp = tmp_sol
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                    print(f"type 1 compilation error = {e}")
+                results.append(-2)
+                return results
+            signal.alarm(0)
+        if debug:
+            print(f"get method = {datetime.now().time()}")
+ 
+        try:
+            method = getattr(tmp, method_name)  # get_attr second arg must be str
+        except:
+            signal.alarm(0)
+            e = sys.exc_info()
+            print(f"unable to get function error = {e}")
+            results.append(-2)
+            return results
+
+        for index, inputs in enumerate(in_outs["inputs"]):
+            # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list)
+            try:
+                if isinstance(inputs[0], dict):
+                    inputs = [{int(k): v for k,v in inputs[0].items()}]
+            except:
+                True
+            try:
+                if isinstance(in_outs["outputs"][index], dict):
+                    in_outs["outputs"][index] = [{int(k): v for k,v in in_outs["outputs"][index].items()}]
+            except:
+                True
+            try:
+                if isinstance(in_outs["outputs"][index][0], dict):
+                    in_outs["outputs"][index] = [{int(k): v for k,v in in_outs["outputs"][index][0].items()}]
+            except:
+                True
+
+            if debug:
+                print(f"time: {datetime.now().time()} testing index = {index}  inputs = {inputs}, {type(inputs)}. type = {which_type}")
+            if which_type == CODE_TYPE.call_based:  # Call-based
+                signal.alarm(timeout)
+                faulthandler.enable()
+                try:
+                    output = method(*inputs)
+
+                    # ground truth sequences are not tuples
+                    if isinstance(output, tuple):
+                        output = list(output)
+                    
+                    tmp_result = output == in_outs["outputs"][index]
+                    if isinstance(in_outs["outputs"][index], list) and in_outs["outputs"][index]:
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index][0])
+
+                    # ground truth sequences are not tuples
+                    try:
+                        if isinstance(output[0], tuple):
+                            tmp_result = tmp_result or ([list(x) for x in output] == in_outs["outputs"][index][0])
+                    except:
+                        True
+                    results.append(tmp_result)
+
+                    # reset the alarm
+                    signal.alarm(0)
+                except Exception as e:
+                    signal.alarm(0)
+                    faulthandler.disable()
+                    if debug:
+                        print(f"Standard input runtime error or time limit exceeded error = {e}")
+                    results.append(-1)
+                    continue
+                faulthandler.disable()
+                signal.alarm(0)
+                if debug:
+                    print(f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+            elif which_type == CODE_TYPE.standard_input:  # Standard input
+                faulthandler.enable()
+                signal.alarm(timeout)
+                passed = False
+
+                if isinstance(inputs, list):
+                    inputs = "\n".join(inputs)
+                if isinstance(in_outs['outputs'][index], list):
+                    in_outs['outputs'][index] = "\n".join(in_outs['outputs'][index])
+
+                with Capturing() as output:
+                    try:
+                        call_method(method, inputs)
+                        # reset the alarm
+                        signal.alarm(0)
+                        passed = True
+                    except Exception as e:
+                        # runtime error or took too long
+                        signal.alarm(0)
+                        print(f"Call-based runtime error or time limit exceeded error = {repr(e)}{e}")
+                        results.append(-1)
+                    signal.alarm(0)
+
+                if not passed:
+                    if debug:
+                        nl = "\n"
+                        if not isinstance(inputs, list):
+                            print(f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+                        else:
+                            print(f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+                    continue
+
+                if passed and debug:
+                    print(f"==> output = {output}, test outputs = {in_outs['outputs'][index]}")
+
+                if custom_compare_(output, in_outs['outputs'][index]):
+                    tmp_result = True
+                    results.append(tmp_result)
+                    continue
+
+                # ground truth sequences are expressed as lists not tuples
+                if isinstance(output, tuple):
+                    output = list(output)
+
+                tmp_result = False
+                try:
+                    tmp_result = (output == [in_outs["outputs"][index]])
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                        if isinstance(output[0], str):
+                            tmp_result = tmp_result or ([e.strip() for e in output] == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check1 exception = {e}")
+                    pass
+
+                if tmp_result == True:  
+                    results.append(tmp_result)
+                    continue
+
+                # try one more time without \n
+                if isinstance(in_outs["outputs"][index], list):
+                    for tmp_index, i in enumerate(in_outs["outputs"][index]):
+                        in_outs["outputs"][index][tmp_index] = i.split("\n")
+                        in_outs["outputs"][index][tmp_index] = [x.strip() for x in in_outs["outputs"][index][tmp_index] if x]
+                else:
+                    in_outs["outputs"][index] = in_outs["outputs"][index].split("\n")
+                    in_outs["outputs"][index] = list(filter(len, in_outs["outputs"][index]))
+                    in_outs["outputs"][index] = list(map(lambda x:x.strip(), in_outs["outputs"][index]))
+
+                try:
+                    tmp_result = (output == [in_outs["outputs"][index]])
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check2 exception = {e}")
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    output = list(filter(len, output))
+
+                if debug:
+                    nl = "\n"
+                    if not isinstance(inputs, list):
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") 
+                    else:
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") 
+                
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                try:
+                    tmp_result = (output == [in_outs["outputs"][index]])
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check3 exception = {e}")
+                    pass
+
+                try:
+                    output_float = [float(e) for e in output]
+                    gt_float = [float(e) for e in in_outs['outputs'][index]]
+                    tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float))
+                except Exception as e:
+                    pass
+                try:
+                    if isinstance(output[0], list):
+                        output_float = [float(e) for e in output[0]]
+                        gt_float = [float(e) for e in in_outs['outputs'][index][0]]
+                        tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float))
+                except Exception as e:
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the stuff into split up list
+                if isinstance(in_outs["outputs"][index], list):
+                    for tmp_index, i in enumerate(in_outs["outputs"][index]):
+                        in_outs["outputs"][index][tmp_index] = set(i.split())
+                else:
+                    in_outs["outputs"][index] = set(in_outs["outputs"][index].split())
+
+                try:
+                    tmp_result = (output == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check4 exception = {e}")
+                    continue
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue 
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = i.split()
+                    output = list(filter(len, output))
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = set(i)    
+                else:
+                    output = output.split()
+                    output = list(filter(len, output))
+                    output = set(output)
+
+                try:
+                    tmp_result = (set(frozenset(s) for s in output) == set(frozenset(s) for s in in_outs["outputs"][index]))
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check5 exception = {e}")
+
+
+                # if they are all numbers, round so that similar numbers are treated as identical
+                try:
+                    tmp_result = tmp_result or (set(frozenset(round(float(t),3) for t in s) for s in output) ==\
+                        set(frozenset(round(float(t),3) for t in s) for s in in_outs["outputs"][index]))
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check6 exception = {e}")
+                
+                if tmp_result == True and debug:
+                    print("PASSED")
+ 
+                results.append(tmp_result)
+                
+                if debug:
+                    nl = "\n"
+                    if not isinstance(inputs, list):
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
+                    else:
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") 
+
+
+    return results
+
+
+def custom_compare_(output, ground_truth):
+    
+    if isinstance(output, list):
+        output_1 = "\n".join(output)
+        if stripped_string_compare(output_1, ground_truth):
+            return True
+
+    if isinstance(output, list):
+        output_2 = [o.lstrip().rstrip() for o in output]
+        output_2 = "\n".join(output_2)
+        if stripped_string_compare(output_2, ground_truth):
+            return True
+
+    return False
+
+def stripped_string_compare(s1, s2):
+    s1 = s1.lstrip().rstrip()
+    s2 = s2.lstrip().rstrip()
+    return s1 == s2
+
+def call_method(method, inputs):
+
+    if isinstance(inputs, list):
+        inputs = "\n".join(inputs)
+
+    inputs_line_iterator = iter(inputs.split("\n"))
+
+    # sys.setrecursionlimit(10000)
+
+    # @patch('builtins.input', side_effect=inputs.split("\n"))
+    @patch('builtins.open', mock_open(read_data=inputs))
+    @patch('sys.stdin', StringIO(inputs))
+    @patch('sys.stdin.readline', lambda *args: next(inputs_line_iterator))
+    @patch('sys.stdin.readlines', lambda *args: inputs.split("\n"))
+    @patch('sys.stdin.read', lambda *args: inputs)
+    # @patch('sys.stdout.write', print)
+    def _inner_call_method(_method):
+        try:
+            return _method()
+        except SystemExit as e:
+            pass
+        finally:
+            pass
+    return _inner_call_method(method) 
+
+
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None

codecritic/evaluation/metric.py

+import numpy as np
+from datasets import load_dataset
+from collections import defaultdict
+
+
+def estimate_pass_at_k(
+    num_samples: list[int], num_correct: list[int], k: int
+) -> np.ndarray:
+    """
+    Estimates pass@k of each problem and returns them in an array.
+    """
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """
+        Calculates 1 - comb(n - c, k) / comb(n, k).
+        """
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    return np.array(
+        [estimator(int(n), int(c), k) for n, c in zip(num_samples, num_correct)]
+    )
+
+
+def group_results(results, apps_path):
+    """
+    Output
+    {
+        "interview": [
+            problem_id: [
+                {"problem_id", problem_id, "eval_result": True, ... },
+                ...
+            ],
+            ...
+        ],
+        ...
+    }
+    """
+    dataset = load_dataset(apps_path)
+
+    groups = defaultdict(lambda: defaultdict(list))
+    for item in results:
+        problem_id = item["problem_id"]
+        split, idx = problem_id.split("_")
+        difficulty = dataset[split][int(idx)]["difficulty"]
+        groups[difficulty][problem_id].append(item)
+
+    if "score" in results[0]:
+        for difficulty, problem in groups.items():
+            for problem_id, lst in problem.items():
+                sorted_lst = sorted(lst, key=lambda x: x["score"], reverse=True)
+                problem[problem_id] = sorted_lst
+
+    return groups
+
+
+def pass_at_k(groups, k):
+    result = {"strategy": "pass@k", "k": k}
+
+    for difficulty, problems in groups.items():
+        num_samples, num_correct = [], []
+        for lst in problems.values():
+            num_samples.append(len(lst))
+            num_correct.append(sum(item["eval_result"] for item in lst))
+        pass_at_k = np.mean(estimate_pass_at_k(num_samples, num_correct, k))
+        result[difficulty] = pass_at_k
+
+    return result
+
+
+def score_pass_at_k(groups, k, strategy):
+    result = {"strategy": strategy, "k": k}
+
+    for difficulty, problems in groups.items():
+        num_samples, num_correct = 0, 0
+        for lst in problems.values():
+            num_samples += 1
+            num_correct += any(item["eval_result"] for item in lst[:k])
+
+        pass_at_k = num_correct / num_samples
+        result[difficulty] = pass_at_k
+
+    return result

doc/readme.qmd

@@ -4,6 +4,8 @@ format: typst
 bibliography: refs.bib
 ---
 
+TODO Rewrite Readme
+
 ## Abstract
 
 LLM-based code verifiers are essential for improving the accuracy of large language models (LLMs) in code generation at test time by filtering out incorrect solutions. While humans can pinpoint bugs and judge the correctness by reasoning through the code, current code verifiers can only make end-to-end binary judgments, which limits their capability to properly verify the code.