readme and run fixes

google-research · Apr 10, 2023 · 7b33ac0 · 7b33ac0
1 parent 4fde744
commit 7b33ac0
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 4 deletions.
diff --git a/flan/v2/README.md b/flan/v2/README.md
@@ -6,8 +6,7 @@ The Flan Collection of datasets and data augmentation methods for instruction tu
 
 ## Setup
 ```
-pip install --upgrade pip
-pip install -r flan/v2/requirements.txt
+bash setup.sh
 ```
 
 ## List of Mixtures

diff --git a/flan/v2/run_example.py b/flan/v2/run_example.py
@@ -109,10 +109,11 @@
 # If you would like to take min(1 epoch, NUM_SAMPLES) then use dataset.take(NUM_SAMPLES)
 # Or if you would like to gather a full epoch, simply `enumerate(dataset)` until completion.
 for i, ex in enumerate(dataset.take(NUM_SAMPLES)):
-    source_counter[ex["_task_source"].numpy()] += 1
+    source_counter[ex["_task_name"].numpy()] += 1
     save_data.append((ex["inputs_pretokenized"].numpy().decode(),
                       ex["targets_pretokenized"].numpy().decode()))
+    if i % 100000 == 0:
+        print(i)
 
 print(f"Data Submixture Counts: {source_counter}")
-
 print(save_data[0])
diff --git a/flan/v2/setup.sh b/flan/v2/setup.sh
@@ -0,0 +1,13 @@
+pip install --upgrade pip
+pip install -r flan/v2/requirements.txt
+
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00000-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00001-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00002-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00003-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00004-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00005-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00006-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00007-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00008-of-00010.zip -d flan/v2/niv2_few_shot_data/
+unzip flan/v2/niv2_few_shot_data/niv2_exemplars.jsonl-00009-of-00010.zip -d flan/v2/niv2_few_shot_data/