Skip to content

V
verl
Commit ff6fcbd8 by 苏舞仙
Options

gen

parent 86867b2b
Showing with 2 additions and 2 deletions
verl/trainer/main_make_datasets.py
...@@ -156,7 +156,7 @@ def process_responses_for_vote(args): ...@@ -156,7 +156,7 @@ def process_responses_for_vote(args):
def main(config): def main(config):
local_path = copy_to_local(config.data.path) local_path = copy_to_local(config.data.path)
output_path = config.data.output_path output_path = config.data.output_path
dataset = pd.read_parquet(local_path)[:10] dataset = pd.read_parquet(local_path)
print(len(dataset)) print(len(dataset))
prompts = dataset[config.data.prompt_key] prompts = dataset[config.data.prompt_key]
responses = dataset[config.data.response_key] responses = dataset[config.data.response_key]
...@@ -250,7 +250,7 @@ def main(config): ...@@ -250,7 +250,7 @@ def main(config):
print(new_dataset[0]) print(new_dataset[0])
# 需不需要对new_dataset再做一次处理，保证code可综合？ # 需不需要对new_dataset再做一次处理，保证code可综合？
# new_dataset.to_parquet(output_path) new_dataset.to_parquet(output_path)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment