gen

verl/trainer/main_make_datasets.py

@@ -156,7 +156,7 @@ def process_responses_for_vote(args):
 def main(config):
     local_path = copy_to_local(config.data.path)
     output_path = config.data.output_path
-    dataset = pd.read_parquet(local_path)[:10]
+    dataset = pd.read_parquet(local_path)
     print(len(dataset))
     prompts = dataset[config.data.prompt_key]
     responses = dataset[config.data.response_key]
@@ -250,7 +250,7 @@ def main(config):
     print(new_dataset[0])
 
     # 需不需要对new_dataset再做一次处理，保证code可综合？
-    # new_dataset.to_parquet(output_path)
+    new_dataset.to_parquet(output_path)
 
 if __name__ == '__main__':
     main()