inference 32b model

codecritic/utils/inference.py

@@ -13,7 +13,6 @@ def generate_worker(cuda_device, prompts, model_path, sampling_params):
         model=model_path,
         seed=42,
         max_model_len=8 * 1024,
-        swap_space=16,
         tensor_parallel_size=len(cuda_device),
     )
 
@@ -57,7 +56,6 @@ def score_worker(cuda_device, prompts, model_path, positive_token, negative_toke
         model=model_path,
         seed=42,
         max_model_len=8 * 1024,
-        swap_space=16,
         tensor_parallel_size=len(cuda_device),
     )
 

scripts/gen_dataset_32b.sh

@@ -3,6 +3,7 @@ set -xe
 model="/lustre/S/huangdi/open_for_out/models/Qwen2.5-Coder-32B-Instruct"
 project="/lustre/S/nanziyuan/projects/ccc"
 modelname="qwen25_coder_inst_32b"
+tp=4
 
 # APPS
 # CUDA_VISIBLE_DEVICES=0,1,2,3 \
@@ -11,7 +12,7 @@ python -m codecritic.cli.gen_dataset \
     --apps /lustre/S/nanziyuan/datasets/apps/ \
     --train "${project}/data/train/${modelname}-apps-train.jsonl" \
     --test "${project}/data/test/${modelname}-apps-test.jsonl" \
-    --tp 2
+    --tp ${tp}
 
 # HumanEval & MBPP
 evalplus.evaluate \
@@ -21,7 +22,7 @@ evalplus.evaluate \
     --dataset humaneval \
     --root "${project}/data/test/${modelname}-humaneval" \
     --backend vllm \
-    --tp 2
+    --tp ${tp}
 
 evalplus.evaluate \
     --model ${model} \
@@ -30,7 +31,7 @@ evalplus.evaluate \
     --dataset mbpp \
     --root "${project}/data/test/${modelname}-mbpp" \
     --backend vllm \
-    --tp 2
+    --tp ${tp}
 
 # HumanEvalPack