step1: evaluate_code Change imap_unordered back to tqmd's process_map. Pool…

 step1: evaluate_code Change imap_unordered back to tqmd's process_map. Pool creates daemon processes, non-daemon processes can create new processes.

step1_evaluate_code.py

@@ -3,10 +3,9 @@
 
 import json
 import multiprocessing
-from multiprocessing import Pool
 import numpy as np
-from typing import Dict
 from datasets import load_dataset
+from tqdm.contrib.concurrent import process_map
 
 from step1_apps_test import run_test
 from utils import extract_code, read_config, load_jsonl, save_jsonl
@@ -66,7 +65,8 @@ def test_generation(args, debug=False):
     finally:
         assert isinstance(curr_res, list)
         problem_results = np.asarray(curr_res)
-        code_sample["eval_result"] = bool(np.all(problem_results > 0))
+
+    code_sample["eval_result"] = bool(np.all(problem_results > 0))
 
     return code_sample
 
@@ -83,8 +83,8 @@ def evaluate_code_samples(code_samples: list, dataset_path: str):
     args = [(get_apps_item(sample), sample) for sample in code_samples]
 
     cpu_num = multiprocessing.cpu_count()
-    with Pool(processes=cpu_num) as pool:
-        results = list(pool.imap_unordered(test_generation, args, chunksize=1000))
+    # TODO `chunksize` affects performance a lot
+    results = process_map(test_generation, args, max_workers=cpu_num, chunksize=1000)
 
     return results