added a classification via eval thresholds to the finetuning predictions

Xeadriel · Jun 9, 2024 · 23ba83d · 23ba83d
1 parent 8a02d86
commit 23ba83d
Show file tree

Hide file tree

Showing 41 changed files with 235 additions and 5,314 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,9 @@ __pycache__/
 *.so
 
 checkpoints/
+results/
+*.tsv
+*pyc
 
 # Distribution / packaging
 .Python

diff --git a/firelang/__pycache__/__init__.cpython-310.pyc b/firelang/__pycache__/__init__.cpython-310.pyc
diff --git a/firelang/__pycache__/stack.cpython-310.pyc b/firelang/__pycache__/stack.cpython-310.pyc
diff --git a/firelang/function/__pycache__/__init__.cpython-310.pyc b/firelang/function/__pycache__/__init__.cpython-310.pyc
diff --git a/firelang/function/__pycache__/base.cpython-310.pyc b/firelang/function/__pycache__/base.cpython-310.pyc
diff --git a/firelang/function/__pycache__/functional.cpython-310.pyc b/firelang/function/__pycache__/functional.cpython-310.pyc
diff --git a/firelang/function/__pycache__/operators.cpython-310.pyc b/firelang/function/__pycache__/operators.cpython-310.pyc
diff --git a/firelang/function/components/__pycache__/__init__.cpython-310.pyc b/firelang/function/components/__pycache__/__init__.cpython-310.pyc
diff --git a/firelang/function/components/__pycache__/common.cpython-310.pyc b/firelang/function/components/__pycache__/common.cpython-310.pyc
diff --git a/firelang/function/components/__pycache__/dense.cpython-310.pyc b/firelang/function/components/__pycache__/dense.cpython-310.pyc
diff --git a/firelang/function/components/__pycache__/multilayer.cpython-310.pyc b/firelang/function/components/__pycache__/multilayer.cpython-310.pyc
diff --git a/firelang/function/components/__pycache__/planar.cpython-310.pyc b/firelang/function/components/__pycache__/planar.cpython-310.pyc
diff --git a/firelang/measure/__pycache__/__init__.cpython-310.pyc b/firelang/measure/__pycache__/__init__.cpython-310.pyc
diff --git a/firelang/measure/__pycache__/base.cpython-310.pyc b/firelang/measure/__pycache__/base.cpython-310.pyc
diff --git a/firelang/measure/__pycache__/dirac.cpython-310.pyc b/firelang/measure/__pycache__/dirac.cpython-310.pyc
diff --git a/firelang/measure/metrics/__pycache__/__init__.cpython-310.pyc b/firelang/measure/metrics/__pycache__/__init__.cpython-310.pyc
diff --git a/firelang/measure/metrics/__pycache__/sinkhorn.cpython-310.pyc b/firelang/measure/metrics/__pycache__/sinkhorn.cpython-310.pyc
diff --git a/firelang/models/__pycache__/__init__.cpython-310.pyc b/firelang/models/__pycache__/__init__.cpython-310.pyc
diff --git a/firelang/models/__pycache__/_firetensor.cpython-310.pyc b/firelang/models/__pycache__/_firetensor.cpython-310.pyc
diff --git a/firelang/models/__pycache__/_fireword.cpython-310.pyc b/firelang/models/__pycache__/_fireword.cpython-310.pyc
diff --git a/firelang/models/__pycache__/tensor.cpython-310.pyc b/firelang/models/__pycache__/tensor.cpython-310.pyc
diff --git a/firelang/models/__pycache__/word.cpython-310.pyc b/firelang/models/__pycache__/word.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/__init__.cpython-310.pyc b/firelang/utils/__pycache__/__init__.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/index.cpython-310.pyc b/firelang/utils/__pycache__/index.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/limits.cpython-310.pyc b/firelang/utils/__pycache__/limits.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/log.cpython-310.pyc b/firelang/utils/__pycache__/log.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/optim.cpython-310.pyc b/firelang/utils/__pycache__/optim.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/parse.cpython-310.pyc b/firelang/utils/__pycache__/parse.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/shape.cpython-310.pyc b/firelang/utils/__pycache__/shape.cpython-310.pyc
diff --git a/firelang/utils/__pycache__/timer.cpython-310.pyc b/firelang/utils/__pycache__/timer.cpython-310.pyc
diff --git a/requirements-cu11.txt b/requirements-cu11.txt
@@ -1,15 +1,15 @@
 -f https://download.pytorch.org/whl/cu111/torch_stable.html
-corpusit==0.1.3
-faster_fifo<=1.2.0
-nltk<=3.7
-numpy==1.20
-scipy==1.7.1
-numba==0.54
-pandas==1.3.5
-scikit_learn<=1.1.2
-torch==1.10.1+cu111
-tqdm<=4.64.1
-rich<=12.6.0
-typing_extensions==4.4.0
-matplotlib==3.5.3
-wandb==0.13.4
+corpusit
+faster_fifo
+nltk
+numpy
+scipy
+numba
+pandas
+scikit_learn
+torch
+tqdm
+rich
+typing_extensions
+matplotlib
+wandb
diff --git a/scripts/__pycache__/benchmark.cpython-310.pyc b/scripts/__pycache__/benchmark.cpython-310.pyc
diff --git a/scripts/__pycache__/sentsim.cpython-310.pyc b/scripts/__pycache__/sentsim.cpython-310.pyc
diff --git a/scripts/additionalBenchmark.py b/scripts/additionalBenchmark.py
@@ -5,6 +5,7 @@
 from collections import defaultdict, Counter
 from corpusit import Vocab
 import nltk
+import random
 import numpy as np
 import pandas as pd
 import torch
@@ -26,11 +27,19 @@
 
 @torch.no_grad()
 def main():
+	seed = 0
+	random.seed(seed)
+	np.random.seed(seed)
+	torch.manual_seed(seed)
+	torch.cuda.manual_seed(seed)
+	torch.backends.cudnn.deterministic = True
+
 	parser = argparse.ArgumentParser()
 	parser.add_argument(
 		"--checkpointsMRPC",
 		nargs="+",
 		default=[
+			"results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20finetune",
 			"checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy",
 			"checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10",
 			"checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20",
@@ -70,7 +79,6 @@ def main():
 	args = parser.parse_args()
 
 	device = "cuda"
-	torch.set_default_device(device)
 
 	sifA = 0.001
 	print("--------------------------------------------------------------------------------------------------------------------------------")
@@ -105,19 +113,19 @@ def main():
 
 		print(f"\t\taccuracy: {accuracy}\n\t\tf1: {f1}\n")
 
-		with open(f'scripts/taskResults/MRPC/median/MRPC-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile:
+		with open(f'scripts/taskResults/MRPC/median/MRPC-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile:
 			writer = csv.writer(csvfile, delimiter='\t', quotechar='ß')
 			writer.writerow(["index", "prediction"])
 			for index, pred in zip(range(len(predsMedianMRPC)), predsMedianMRPC):
 				writer.writerow([index, pred])
 
-		with open(f'scripts/taskResults/MRPC/threshold/MRPC-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile:
+		with open(f'scripts/taskResults/MRPC/threshold/MRPC-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile:
 			writer = csv.writer(csvfile, delimiter='\t', quotechar='ß')
 			writer.writerow(["index", "prediction"])
 			for index, pred in zip(range(len(predsThresholdMRPC)), predsThresholdMRPC):
 				writer.writerow([index, pred])
 
-		with open(f'scripts/taskResults/MRPC/f1Threshold/MRPC-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile:
+		with open(f'scripts/taskResults/MRPC/f1Threshold/MRPC-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile:
 			writer = csv.writer(csvfile, delimiter='\t', quotechar='ß')
 			writer.writerow(["index", "prediction"])
 			for index, pred in zip(range(len(predsF1ThresholdMRPC)), predsF1ThresholdMRPC):
@@ -133,13 +141,13 @@ def main():
 		model = FireWord.from_pretrained(checkpoint).to(device)
 		predsMedianSSTGlue, predsThresholdSSTGlue = predictSSTGlue(model, testPairsSSTGlue, devPairsSSTGlue, devLabelsSSTGlue, sifA)
 
-		with open(f'scripts/taskResults/SSTGLUE/median/SST-2-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile:
+		with open(f'scripts/taskResults/SSTGLUE/median/SST-2-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile:
 			writer = csv.writer(csvfile, delimiter='\t', quotechar='ß')
 			writer.writerow(["index", "prediction"])
 			for index, pred in zip(range(len(predsMedianSSTGlue)), predsMedianSSTGlue):
 				writer.writerow([index, pred])
 
-		with open(f'scripts/taskResults/SSTGLUE/threshold/SST-2-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile:
+		with open(f'scripts/taskResults/SSTGLUE/threshold/SST-2-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile:
 			writer = csv.writer(csvfile, delimiter='\t', quotechar='ß')
 			writer.writerow(["index", "prediction"])
 			for index, pred in zip(testIndicesSSTGlue, predsThresholdSSTGlue):
@@ -154,13 +162,13 @@ def main():
 		model = FireWord.from_pretrained(checkpoint).to(device)
 		predsMedianRTE, predsThresholdRTE = predictRTE(model, testPairsRTE, devPairsRTE, devLabelsRTE, sifA)
 
-		with open(f'scripts/taskResults/RTE/median/RTE-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile:
+		with open(f'scripts/taskResults/RTE/median/RTE-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile:
 				writer = csv.writer(csvfile, delimiter='\t', quotechar='ß')
 				writer.writerow(["index", "prediction"])
 				for index, pred in zip(range(len(predsMedianRTE)), predsMedianRTE):
 					writer.writerow([index, pred])
 
-		with open(f'scripts/taskResults/RTE/threshold/RTE-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile:
+		with open(f'scripts/taskResults/RTE/threshold/RTE-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile:
 			writer = csv.writer(csvfile, delimiter='\t', quotechar='ß')
 			writer.writerow(["index", "prediction"])
 			for index, pred in zip(testIndicesRTE, predsThresholdRTE):
@@ -231,7 +239,7 @@ def benchmarkMRPC(
 	low = min(preds)
 	high = max(preds)
 	steps = math.ceil((high - low) / 2)*100
-
+	
 	for threshold in np.linspace(low, high, steps):
 		truePosCount = sum([int(preds[i] >= threshold) == 1 and labels[i] == 1 for i in range(len(preds))])
 		falsePosCount = sum([int(preds[i] >= threshold) == 1 and labels[i] == 0 for i in range(len(preds))])

diff --git a/scripts/fine-tuning/1_fine_tune_MRPC_all_checkpoints.bat b/scripts/fine-tuning/1_fine_tune_MRPC_all_checkpoints.bat
@@ -1,5 +1,5 @@
-python -m scripts.fineTune --sz_batch=200 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=600000 --eval_interval=1000 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=MRPC
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=MRPC
 
-python -m scripts.fineTune --sz_batch=200 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=600000 --eval_interval=1000 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=MRPC
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=MRPC
 
-python -m scripts.fineTune --sz_batch=200 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=600000 --eval_interval=1000 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=MRPC
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=MRPC
diff --git a/scripts/fine-tuning/2_fine_tune_SST-2_all_checkpoints.bat b/scripts/fine-tuning/2_fine_tune_SST-2_all_checkpoints.bat
@@ -1,41 +1,5 @@
-python -m scripts.train \
-    --sz_batch=32768 \
-    --lr=0.005 \
-    --lr_scheduler=OneCycleLR \
-    --n_iters=600000 \
-    --eval_interval=1000 \
-    --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 \
-    --optimizer=adamw \
-    --seed=0 \
-    --accum_steps=10 \
-    --weight_decay=1e-6 \
-	--pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20
-	--task=MRPC
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/SST-2_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=SST-2  
 
-python -m scripts.train \
-    --sz_batch=32768 \
-    --lr=0.005 \
-    --lr_scheduler=OneCycleLR \
-    --n_iters=600000 \
-    --eval_interval=1000 \
-    --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10
-    --optimizer=adamw \
-    --seed=0 \
-    --accum_steps=10 \
-    --weight_decay=1e-6 \
-	--pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10
-	--task=MRPC
-
-python -m scripts.train \
-    --sz_batch=32768 \
-    --lr=0.005 \
-    --lr_scheduler=OneCycleLR \
-    --n_iters=600000 \
-    --eval_interval=1000 \
-    --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy
-    --optimizer=adamw \
-    --seed=0 \
-    --accum_steps=10 \
-    --weight_decay=1e-6 \
-	--pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy
-	--task=MRPC
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/SST-2_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=SST-2  
+
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/SST-2_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=SST-2
diff --git a/scripts/fine-tuning/3_fine_tune_RTE_all_checkpoints.bat b/scripts/fine-tuning/3_fine_tune_RTE_all_checkpoints.bat
@@ -1,41 +1,5 @@
-python -m scripts.train \
-    --sz_batch=32768 \
-    --lr=0.005 \
-    --lr_scheduler=OneCycleLR \
-    --n_iters=600000 \
-    --eval_interval=1000 \
-    --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 \
-    --optimizer=adamw \
-    --seed=0 \
-    --accum_steps=10 \
-    --weight_decay=1e-6 \
-	--pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20
-	--task=MRPC
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/RTE_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=RTE  
 
-python -m scripts.train \
-    --sz_batch=32768 \
-    --lr=0.005 \
-    --lr_scheduler=OneCycleLR \
-    --n_iters=600000 \
-    --eval_interval=1000 \
-    --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10
-    --optimizer=adamw \
-    --seed=0 \
-    --accum_steps=10 \
-    --weight_decay=1e-6 \
-	--pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10
-	--task=MRPC
-
-python -m scripts.train \
-    --sz_batch=32768 \
-    --lr=0.005 \
-    --lr_scheduler=OneCycleLR \
-    --n_iters=600000 \
-    --eval_interval=1000 \
-    --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy
-    --optimizer=adamw \
-    --seed=0 \
-    --accum_steps=10 \
-    --weight_decay=1e-6 \
-	--pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy
-	--task=MRPC
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/RTE_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=RTE  
+
+python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/RTE_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=RTE
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,6 +7,9 @@ __pycache__/ @@
     *.so
     checkpoints/
+    results/
+    *.tsv
+    *pyc
     # Distribution / packaging
     .Python
@@ Expand Down @@