Update generate_data.py to fix caching (#17)

* Add index to the cache to point to the right leaf node. * Fix the save paths to make it concise Signed-off-by: Aakanksha Duggal <[email protected]>
aakankshaduggal · Jul 18, 2024 · 89e3b44 · 89e3b44
1 parent 94b49ca
commit 89e3b44
Showing 1 changed file with 4 additions and 5 deletions.
diff --git a/src/instructlab/sdg/generate_data.py b/src/instructlab/sdg/generate_data.py
@@ -8,7 +8,6 @@
 import json
 import os
 import time
-
 # Third Party
 # instructlab - All of these need to go away (other than sdg) - issue #6
 from datasets import Dataset, concatenate_datasets
@@ -241,7 +240,7 @@ def generate_data(
             # add to 1.0 recipe
 
 
-        generated_data = sdg.generate(ds, cache_dataset_path='~/tmp/cache.jsonl')
+        generated_data = sdg.generate(ds, cache_dataset_path=f"~/tmp/cache_{leaf_node_path}.jsonl")
 
         if is_knowledge:
             knowledge_phase_data = create_phase07_ds(generated_data)
@@ -257,10 +256,10 @@ def generate_data(
 
             # generate mmlubench data for the current leaf node 
             mmlubench_data = create_mmlu_evaluation_dataset(sdg_mmlubench.generate(ds))
-            eval_data_file_path=f"{output_dir}/node_datasets_{date_suffix}/mmlubench_{date_suffix}_{leaf_node_path}.jsonl"
+            eval_data_file_path=f"{output_dir}/node_datasets_{date_suffix}/mmlubench_{leaf_node_path}.jsonl"
             logger.info(f"Saving MMLU Dataset {eval_data_file_path}")
             mmlubench_data.to_json(eval_data_file_path, orient='records', lines=True)
-            yaml_file_path=f"{output_dir}/node_datasets_{date_suffix}/{leaf_node_path}_{date_suffix}_{leaf_node_path}_task.yaml"
+            yaml_file_path=f"{output_dir}/node_datasets_{date_suffix}/{leaf_node_path}_task.yaml"
             logger.info(f"Saving MMLU Task yaml {yaml_file_path}")
             create_mmlu_evaluation_yaml(task_name=leaf_node_path, 
                                                     eval_data_file_path=eval_data_file_path,
@@ -275,7 +274,7 @@ def generate_data(
                 num_proc=8,
             )
 
-            fpath = os.path.join(output_dir, f"node_datasets_{date_suffix}/node_{i}.jsonl")
+            fpath = os.path.join(output_dir, f"node_datasets_{date_suffix}/{leaf_node_path}.jsonl")
             messages.to_json(fpath, orient="records", lines=True)
             skills_recipe.add_dataset(fpath, NUM_SYNTH_SKILLS)