step3 & 4: fix llamafactory related issues

example_config.toml

 model = "/path/to/model"
 llamafactory_model_template = ""
+llamafactory_path = ""
 apps = "/path/to/apps_dataset"
 
 [sample]

step3_train_outcome_reward_model.py

@@ -14,7 +14,7 @@ deepspeed: {deepspeed_config_path}
 dataset: {dataset_name}
 template: deepseekcoder
 cutoff_len: 4096
-max_samples: 1000
+max_samples: 10000
 overwrite_cache: true
 preprocessing_num_workers: 16
 
@@ -36,7 +36,7 @@ bf16: true
 ddp_timeout: 180000000
 
 ### eval
-val_size: 0.1
+val_size: 0.01
 per_device_eval_batch_size: 1
 eval_strategy: steps
 eval_steps: 500

step4_test_reward_model.py

@@ -11,7 +11,7 @@ from utils_metric import group_results, score_pass_at_k
 from transformers import AutoTokenizer
 
 
-def run_server(api_port, cuda_device, rm_inference_yaml_path):
+def run_server(api_port, cuda_device, rm_inference_yaml_path, llamafactory_path):
     env = os.environ.copy()
     env["API_PORT"] = api_port
     env["CUDA_VISIBLE_DEVICES"] = cuda_device
@@ -20,6 +20,8 @@ def run_server(api_port, cuda_device, rm_inference_yaml_path):
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         env=env,
+        cwd=llamafactory_path,
+        shell=True
     )
     print(
         f"Started server with PID {server_process.pid} on port {api_port} and CUDA device {cuda_device}"
@@ -73,13 +75,13 @@ def reward_model_inference(args):
 
 
 def mutli_process_reward_model_inference(
-    test_path, model_path, inference_cfg_path, result_path
+    test_path, model_path, inference_cfg_path, result_path, llamafactory_path
 ):
     cuda_devices = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
     gpu_num = len(cuda_devices)
     test_dataset = preprocess_dataset(model_path, load_jsonl(test_path), gpu_num)
     server_processes = [
-        run_server(8000 + i, cuda_devices[i], inference_cfg_path)
+        run_server(8000 + i, cuda_devices[i], inference_cfg_path, llamafactory_path)
         for i in range(gpu_num)
     ]
     time.sleep(300)  # Wait for the servers to start (adjust the sleep time as needed)
@@ -97,6 +99,7 @@ if __name__ == "__main__":
         cfg["orm"][orm_test_model]["model_path"],
         cfg["orm"][orm_test_model]["inference_yaml_path"],
         cfg["orm"][orm_test_model]["minimal_test_score_path"],
+        cfg["llamafactory_path"]
     )
 
     groups = group_results(results)