From 23ba83d771b6926a7bed58d2bf7d35ddf5fc1eb8 Mon Sep 17 00:00:00 2001 From: Xeadriel Date: Sun, 9 Jun 2024 19:48:45 +0200 Subject: [PATCH] added a classification via eval thresholds to the finetuning predictions --- .gitignore | 3 + firelang/__pycache__/__init__.cpython-310.pyc | Bin 315 -> 318 bytes firelang/__pycache__/stack.cpython-310.pyc | Bin 5997 -> 6017 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 223 -> 226 bytes .../function/__pycache__/base.cpython-310.pyc | Bin 7065 -> 7068 bytes .../__pycache__/functional.cpython-310.pyc | Bin 1214 -> 1217 bytes .../__pycache__/operators.cpython-310.pyc | Bin 5733 -> 5736 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 250 -> 253 bytes .../__pycache__/common.cpython-310.pyc | Bin 706 -> 709 bytes .../__pycache__/dense.cpython-310.pyc | Bin 2823 -> 2826 bytes .../__pycache__/multilayer.cpython-310.pyc | Bin 6009 -> 6012 bytes .../__pycache__/planar.cpython-310.pyc | Bin 4483 -> 4486 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 196 -> 199 bytes .../measure/__pycache__/base.cpython-310.pyc | Bin 1742 -> 1745 bytes .../measure/__pycache__/dirac.cpython-310.pyc | Bin 10001 -> 10004 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 195 -> 198 bytes .../__pycache__/sinkhorn.cpython-310.pyc | Bin 6803 -> 6806 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 196 -> 199 bytes .../__pycache__/_firetensor.cpython-310.pyc | Bin 3807 -> 3810 bytes .../__pycache__/_fireword.cpython-310.pyc | Bin 9514 -> 9517 bytes .../models/__pycache__/tensor.cpython-310.pyc | Bin 183 -> 186 bytes .../models/__pycache__/word.cpython-310.pyc | Bin 179 -> 182 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 156 -> 159 bytes .../utils/__pycache__/index.cpython-310.pyc | Bin 3114 -> 3117 bytes .../utils/__pycache__/limits.cpython-310.pyc | Bin 942 -> 945 bytes .../utils/__pycache__/log.cpython-310.pyc | Bin 507 -> 510 bytes .../utils/__pycache__/optim.cpython-310.pyc | Bin 3206 -> 3209 bytes .../utils/__pycache__/parse.cpython-310.pyc | Bin 780 -> 783 bytes .../utils/__pycache__/shape.cpython-310.pyc | Bin 1828 -> 1831 bytes .../utils/__pycache__/timer.cpython-310.pyc | Bin 4522 -> 4525 bytes requirements-cu11.txt | 28 +- scripts/__pycache__/benchmark.cpython-310.pyc | Bin 21013 -> 21042 bytes scripts/__pycache__/sentsim.cpython-310.pyc | Bin 4360 -> 4363 bytes scripts/additionalBenchmark.py | 26 +- .../1_fine_tune_MRPC_all_checkpoints.bat | 6 +- .../2_fine_tune_SST-2_all_checkpoints.bat | 44 +- .../3_fine_tune_RTE_all_checkpoints.bat | 44 +- scripts/fineTune.py | 220 ++- .../MRPC-wacky_mlplanardiv_d2_l4_k10.tsv | 1726 ----------------- ...PC-wacky_mlplanardiv_d2_l4_k1_polysemy.tsv | 1726 ----------------- .../MRPC-wacky_mlplanardiv_d2_l8_k20.tsv | 1726 ----------------- 41 files changed, 235 insertions(+), 5314 deletions(-) delete mode 100644 scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k10.tsv delete mode 100644 scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k1_polysemy.tsv delete mode 100644 scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l8_k20.tsv diff --git a/.gitignore b/.gitignore index 8f7c68f..90eb78b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ __pycache__/ *.so checkpoints/ +results/ +*.tsv +*pyc # Distribution / packaging .Python diff --git a/firelang/__pycache__/__init__.cpython-310.pyc b/firelang/__pycache__/__init__.cpython-310.pyc index 741644873d157e7c0c4747cc04f589fa5496b036..5b54506514735ec6a0397d44401fa07a29298f69 100644 GIT binary patch delta 51 zcmdnZw2z55pO=@50SFAbKc=pk$a`JNMn5;NL_bkKpeR4RC^0uTGcR4gpt2+*KX2l1 F2LPBm5(5AL delta 48 zcmdnTw3~@HpO=@50SLDBe@tCAk@vc|rG7?!ZmNDoYGO)JW@?VUOKNd;Nq)h^e+~e6 C`Vmn8 diff --git a/firelang/__pycache__/stack.cpython-310.pyc b/firelang/__pycache__/stack.cpython-310.pyc index 373ed12584de5d599ea7480d03c1119e7e14b181..c32ed4e14bda757262b8877307004ea8a2182e13 100644 GIT binary patch delta 753 zcmX|-U2D@&7{|{^+9q9J*L8L6+!UnK$Y>Bnr?YO<32sPn>#T2++BJRM+N7sxwIc;* z2li$No=Y$6M1-L?67~u7&MzR%)=O&5T<5Mhf+38Mj&xk%3{>G2tZ@4reJh5S zdBwJmm(Y{b8(hcMu^W8fVb~6E$6+}7g2EqQj0_vw5nG5{V9sgS!PO^*bq|Hnu&4jT zQ>#{9cN>E4>2f4(2z_sl%{$K8jAeby7Ioh%RKSW|aqNkPu!Op}3yE;)HhmvoVb9wWFa|)lh#_mJn$rL-tG~hZQPJfL2 z!t?aVwKow5F$q{^SY&SlG7)pteHq)y9INSTPZ+1@q35G_jkOY=Bq?_8Owyz$l+fE+ zJ|?-yI#c8xAk8qaD=;AUK|BD!3ONaQ2zUf|4A`JY-Y9-be|o>-8}yxT#eI|MfvuEn zl*j`8X*n>56XyHCIgHcv`&iob9DEbdqO-v$uA6H?4qvJq?`&C%fNcheBWIyLGpu)y zeSm1RH!d%?!AJlZa1n4G09S1NAK8Se07x(l>sqsxSB>5>9^3~f8d+7NB{r; delta 734 zcmXYtOKa3n6oBtdW+sz)q+{$%JG6CSsX9`cg;M)wgi5uGVr^-Cwv0{cOdpeY=XPx2 z4q95UI~RAUi_At4R}$z(6v36yA0R>S7fA6Rcur~{-#zEM=bVJIZ~YlE+-x>Q;5mNt z*O@AQGAyNQ*s8kQjnxMY(sOC)?<#L+!3oW(fde@^^Vn@p1@w;wC5SW6SC zpB6Egm=sB0v!h8oBQ%~pP}ri-eB%f@TYCLcWIMn*yyG=lr?zUx4wFKgCcOs~eon9F zqmmqy?DAYx61NkQ>m_4cC_-ITg}OjW+FbBi@1tEKu#u9=`ru#}5MzjOd7An~N99TS zuzMUu4`K#ThPN>RCR0+{etbNIZ4jEmjS_$FHKEITHfu19FWVv&Idck_9La*1*L_Hf(BVrR!QrKmT`_JN67)Rb@u6&ob z=&n4;cVC-9zk;YB97Guq#v8n{w#ht6&o!gE`p&ku7b`aT5gz~m delta 52 zcmbPZKGU2ppO=@50SLDBe@s0nzL8IZN!(69BR@A)zalj;r6@BsN8cs2IJ+djU~>SI Gk0b!2rVzdW diff --git a/firelang/function/__pycache__/functional.cpython-310.pyc b/firelang/function/__pycache__/functional.cpython-310.pyc index a4e4d93eadcec262e48f05d5551686590623c121..81e1b58f9ec297663f2acf13cfb1c6ac470b1309 100644 GIT binary patch delta 54 zcmdnTd61JgpO=@50SFAbKc+@*^ky5BvZC diff --git a/firelang/function/__pycache__/operators.cpython-310.pyc b/firelang/function/__pycache__/operators.cpython-310.pyc index 1f4b1064579dead8b2434947292561d467fae25e..132a0d59152f81eff703838ebf4a5bc701ae5db4 100644 GIT binary patch delta 55 zcmaE=^FoI&pO=@50SFAbKc;S!+{nkxEaj%3n^&Trs2@<2pI(%ho12-Ju3u1Dl98Xc J*?`$p7y#FP5sm-= delta 52 zcmaE%^Hhf~pO=@50SLDBe@wkEv5}9PS=?DaBR@A)zalj;r6@BsN8cs2IJ+djV6!o^ GsW1S(U=a@h diff --git a/firelang/function/components/__pycache__/__init__.cpython-310.pyc b/firelang/function/components/__pycache__/__init__.cpython-310.pyc index 788bb3dde678824943f79b67c9bddb78b944ed2a..21af6e770e20eaef921e77e3dfb256eb28246b40 100644 GIT binary patch delta 51 zcmeyx_?MA4pO=@50SFAbKc>b`=6L~ diff --git a/firelang/function/components/__pycache__/common.cpython-310.pyc b/firelang/function/components/__pycache__/common.cpython-310.pyc index ac8e61533b8548f927f0379fc2bb5cbf90ddb724..08c952f54727a5ddd6997cffee7874a46d10fd76 100644 GIT binary patch delta 52 zcmX@adX$wnpO=@50SFAbKc@bf$h%p}Pd_)WL_bkKpeR4RC^0uTGcR4gpt2+*KX2pp G*NgzTD-!(x delta 49 zcmX@gdWe-bpO=@50SLDBe@uNfk$1DWw|+)`ZmNDoYGO)JW@?VUOKNd;Nq)h`Tdx@b DmBJD~ diff --git a/firelang/function/components/__pycache__/dense.cpython-310.pyc b/firelang/function/components/__pycache__/dense.cpython-310.pyc index 3b78e6a8b8b97ff436a073c523ecf463a350e9da..ce0976084e3df20c4f7a581bd20d49c4d129a9ef 100644 GIT binary patch delta 54 zcmZn{>k{M5=jG*M00M*VkEt0OdDB^>eD!nlO7s);1B&v~ixP8lGxO5*3o1)8^7A%# Iv(&Hw0HVARJOBUy delta 51 zcmeAYYZv3q=jG*M0D^7(A5$4N@}{$hd+BH7=cejcq$Z{mWv1rnyQCIpm*f|0?qjK8 F0|0l_5GViu diff --git a/firelang/function/components/__pycache__/multilayer.cpython-310.pyc b/firelang/function/components/__pycache__/multilayer.cpython-310.pyc index 9b5efa8270428d87c6eaf319b7a215ddb7b04097..b6fe0bf6d323866aedbef9f453b51dcaf2a7a036 100644 GIT binary patch delta 55 zcmeyV_eYN}pO=@50SFAbKc=!uZsaRwkqXk!%`4GQ)DI}iPcKT$&CSe9*Dt6n$;i*! JJcH$-5CGYk5|;n~ delta 52 zcmeyP_fwBApO=@50SLDBe@xYp*vMDRBJQuBk)NBYUy+)aQk0pRqwkVhoL!P%uz5Di GLm>darxFnW diff --git a/firelang/function/components/__pycache__/planar.cpython-310.pyc b/firelang/function/components/__pycache__/planar.cpython-310.pyc index 9056da89b4cd0d105930ffdfa9d9541fa6449790..c933d6088f4b0ab8c18b6a613f6f071007a96a9b 100644 GIT binary patch delta 54 zcmZoxZd2yX=jG*M00M*VkE!=J^4?^Y^3%`FE74EX4=Bn{FG|eK&CE;JFQ_cZ$j{sS Im${7>0LYONxc~qF delta 50 zcmZouZdT^a=jG*M0D^7(A5+~n^4???_twwI&rQ{@NKH&B%1q7CcS$YIF3B(0#L~tK E0DxBzbN~PV diff --git a/firelang/measure/__pycache__/__init__.cpython-310.pyc b/firelang/measure/__pycache__/__init__.cpython-310.pyc index 3eeb0963b569eec87a6f0fef2a9970d49c18042f..d89d5c25e7f01462b2a009d10332b5b4303fbc77 100644 GIT binary patch delta 51 zcmX@Yc$|?ppO=@50SFAbKc;F<J! CP7nJ4 diff --git a/firelang/measure/__pycache__/base.cpython-310.pyc b/firelang/measure/__pycache__/base.cpython-310.pyc index 713837f7897496b15ecd8b88682d63fc2a5e51fa..a31aa963d7fd7f108949447cc5a4a54224f84741 100644 GIT binary patch delta 55 zcmX@ddy$tfpO=@50SFAbKc-5vZsaRsmU7U~%`4GQ)DI}iPcKT$&CSe9*Dt6n$;i*! JJe}E`8349W5i$S( delta 52 zcmcb}dybbcpO=@50SLDBe@y+wvXQTdS=?4XBR@A)zalj;r6@BsN8cs2IJ+djVDl_y Gb7laygb`!_ diff --git a/firelang/measure/__pycache__/dirac.cpython-310.pyc b/firelang/measure/__pycache__/dirac.cpython-310.pyc index 294671e62b3d2dbfd4618500f7dcc751be468c28..b6d0ad489138840f92b55ef74090332780115485 100644 GIT binary patch delta 55 zcmbQ}H^q-HpO=@50SFAbKc+6Q+{ow4D&?r3n^&Trs2@<2pI(%ho12-Ju3u1Dl98Xc JxrlYXA^^^p5;On+ delta 52 zcmbQ@H_?wTpO=@50SLDBe@uODzLC$DRoqTLBR@A)zalj;r6@BsN8cs2IJ+djU~?(! Gd_@4c1QE>u diff --git a/firelang/measure/metrics/__pycache__/__init__.cpython-310.pyc b/firelang/measure/metrics/__pycache__/__init__.cpython-310.pyc index b773b3c7437e42c8f5e7c170be6e5c7a46e1df2a..3f69dd1441ffd0e962cdefd3d0efb09d3bf73cb1 100644 GIT binary patch delta 48 zcmX@ic#M%NpO=@50SFAbCvrte`RM27mFOqx2NdO}7bWKAX6B{q7gUyH`UBR@A)zalj;r6@BsN8cs2IJ+djV6zo- Gj5q+N5)jz{ diff --git a/firelang/models/__pycache__/__init__.cpython-310.pyc b/firelang/models/__pycache__/__init__.cpython-310.pyc index 7c24855dccbbd7df01d9507f5d088926f1a8e4ef..c57d5fff957a8231e0cfd34b8fc0c9baf9b5f11c 100644 GIT binary patch delta 51 zcmX@Yc$|?ppO=@50SFAbKc;F;dpi0|3f<5w-vT delta 52 zcmaDPdta6>pO=@50SLDBe@y+#vyso6N!(dKBR@A)zalj;r6@BsN8cs2IJ+djU~@53 GJ`VuEmJy%; diff --git a/firelang/models/__pycache__/_fireword.cpython-310.pyc b/firelang/models/__pycache__/_fireword.cpython-310.pyc index 09b6df4d0c60b4d0a50936c84cf960bbf33e3504..80854df742194fce144ba647407e67c8e2fd976e 100644 GIT binary patch delta 55 zcmZ4GwbqL-pO=@50SLA}^++>S*~oW+S;|E}H?KrLQ9qz4KfNe1H#ajcUB95RBqKj> J^H*k5MF8PG60iUO delta 52 zcmZ4MwaSYxpO=@50SLDBe@qoq-pF@>S=>=SBR@A)zalj;r6@BsN8cs2IJ+djVDk@V GQ$+x?RuQcL diff --git a/firelang/models/__pycache__/tensor.cpython-310.pyc b/firelang/models/__pycache__/tensor.cpython-310.pyc index dc5f451ae4aefdcbf0e51361c2f2bfb6fa6fde0f..b7ff408154e5bccb10ba26cf2a0b3d22e42bdcb0 100644 GIT binary patch delta 48 zcmdnaxQmf1pO=@50SFAbCvrtfIqK);mFOqx2NdO}7bWKAX6B{q7gUyHR#pO=@50SFAbCvw?JIqB!-mFOqx2NdO}7bWKAX6B{q7gUyH*wZ`=qKt26y>KECFbU4=B4WwRF-7q=WXU> HtmOs(pqLQp delta 50 zcmeB_Y?I{8=jG*M0D^7(A5+;T@;()}(a*@wP1UbRO-w1uOwG}ENiEJU$uHQu2N5=jG*M00M*VkExLxdABi2+3V-#mFOqx2NdO}7bWKAX6B{q7gUyHtW-~=jG*M0D^7(A5)z+@@``kx6#kY&rQ{@NKH&B%1q7CcS$YIF3B(0e3#LP F5deg>5RU)= diff --git a/firelang/utils/__pycache__/shape.cpython-310.pyc b/firelang/utils/__pycache__/shape.cpython-310.pyc index 8fd3977d2f7ed6834ec1ec141f734035f1ac20f2..3f761605c477002b6b34b7c8c5693b85c705e0bf 100644 GIT binary patch delta 54 zcmZ3&x15hRpO=@50SFAbKc<#!OHNj=oE3adt_5!RB*J G_xS*=f)WJ) diff --git a/requirements-cu11.txt b/requirements-cu11.txt index 9a63c0f..3a1f13a 100644 --- a/requirements-cu11.txt +++ b/requirements-cu11.txt @@ -1,15 +1,15 @@ -f https://download.pytorch.org/whl/cu111/torch_stable.html -corpusit==0.1.3 -faster_fifo<=1.2.0 -nltk<=3.7 -numpy==1.20 -scipy==1.7.1 -numba==0.54 -pandas==1.3.5 -scikit_learn<=1.1.2 -torch==1.10.1+cu111 -tqdm<=4.64.1 -rich<=12.6.0 -typing_extensions==4.4.0 -matplotlib==3.5.3 -wandb==0.13.4 \ No newline at end of file +corpusit +faster_fifo +nltk +numpy +scipy +numba +pandas +scikit_learn +torch +tqdm +rich +typing_extensions +matplotlib +wandb \ No newline at end of file diff --git a/scripts/__pycache__/benchmark.cpython-310.pyc b/scripts/__pycache__/benchmark.cpython-310.pyc index 21005265447ae81a7814b608438085e4b171c17f..a6e141243a636bd05454f0d49ef8a80178a7835d 100644 GIT binary patch delta 1653 zcmZuxZERCz6z=Km?%LkQB8CM-c(PRWZ_o+>A8DOqupm=^6z zCeGvx%YDMkO5VKXd6PF^``WV;3Uo1cRE@K!7j!YoSezyB@%AFDtX+aT_=Qx6Kh_?Q ze&<885rTE!Zl7EHgUmm1*7uL(0t?9eYX_J1T3wli4|=U}0)pDZ#xy`VzS8tAJi^lE zEnvrv=E@#3@Kk|=ZVgXOC8BCF!MxLjFF7>J-FIh0UpiXHUw}N_qMyC7w6i8(kkg91grR6`v44L zUPpQJ5LK!+yieoAo=z0yrp3%%!=&vpVSb0J=zsI@N{3T2RNYkKK588=!z#Kvo1j?R z-}yK6ITU$bNarMTj^&*f6iHdP9o)|4cJ32feiGlXc`TKalAR_c4=;6>Lq2}k9mf@a znV*RpJF(tyHkr1J&)5-(n-$c54u*QY_(o5a-~bc86DTi|Gs;|IbfT;Pf9;vrY@AC$ zCY(819~F9r7YbN|>?}>`#Ou9IVJ|De+r3+hwq#O_4Sbf&O1$4&FV+yBFlT66*D31e7zchk6U37Q zFOWlE5#o~s3OPWx96zZf*j3&~kn7!;9`cLb6tBUXL)FDbu?sYV6rP!xnhk{$vucFJ zaBZmA`-r-)BXK@K-g%BPd?J3LR z8T>J5q8~GRixx2kHnU@Rd$bBh@TbwF_$n=Q2KVh=+uTF+J+9NU)p&@%`U)GSta5_C zaB?gt(W|unUV8B6n78aO*)amz6x|k?IT4D)PpC;Yj}OLHB$|6;mX+~uVH;lAbM9Yr Ca+{+7 delta 1625 zcmZ`(ZA?>V6z*wDd)wQVR>7hL5ek$R3utFb`7R<-J`|%{SgO>*t;#Gdye)#J9d4Rz zkq{G4HeHw*WbDVX#bh*D!jdIR_+hdi(! zCB3A%uF4oXRo5VEkOZu+t}<>JBm+v-zJT$4%_3wi8Owsems!$=j71W6s}_ulD$Y#X z%)H6el4VYNQsvIiJ*@5q3?A5nKXe=MnCBKe#z&sG_>{NBdI>1DW3r}9_dA7u;JY>d zh((|U3;#DlQSR=3hZd~3QXhs>xxxck00-j158(y+efvPbL0@^B3iwjsAPdoWJQYi% zQ_MxV5`r(O@G3zahmZ4E8D#|m8-XLY;!6Rz(R=74a|6%Fc^f5c_~W5xx}D_tf{rQv zMD#aa=LTlOsNb%8Lc1Km*ZpB@BYA1V4ampG{=M)7g@%BB2T4tU z>;6JK)8K9K@mSVMqa$hdoEErAuOvpNVK!p(9OS&wgcO^03v7l`0Y$|74Hd;|qO_S? z%@#;g8mlK(h5A4#ROMWO5WufE(O3zsc)PJnTqAXaU=7zAJK+d=n@S3gQl$zd`yV~h zWY>3-(xSrJrsDr`-D|RoimR2DuH(a|30T99=7X>+H`)9gf;!1Or^%Yqrq`@*Ya~(H zh|N6C=dooowruKKG1oGtjK!{!^6}Hw5x=;>wKpmEJg!!9)xntJ-C%NJrWX)(I?bfj#?vsV7g9$#{ zshAAs*e_(^PyMh5j^W_pQV8SpVHt+-WarC{LEdW0${kiP@wc%6ia-GMRV}ddxz)~L zfGB?3vl9bd`7n&3E(cu7#kvaAa22m~&%rbbp<>+|bX`x-6MCuT5}on_2R@(^#2LY9 zN@!S=_z=M`B|z3*oRU)P4&S|t`<*x+s?{}PK>bg zBeG@gdd04= jFJz%uyo-ihn~0^Q|})c^nh diff --git a/scripts/__pycache__/sentsim.cpython-310.pyc b/scripts/__pycache__/sentsim.cpython-310.pyc index e62f36d4529364d08f2954675a08a6fc79d3bdce..6f3f63ca72466f5616b1d82de01f36991dc227ea 100644 GIT binary patch delta 55 zcmeBB>Q>^*=jG*M00M*VkEuKY8~Oe)Nm=RV=9TCt>IW3%rxzvW=4R%l>laj(WaQ^< JR%Cw83jnd=5t9G_ delta 52 zcmeBH>QLg#=jG*M0D^7(A5)w7H}d^q5;xb+$j?pHuSiWyDauUE(RWEL&MwI>*sQ|* GoEHG3WD#5d diff --git a/scripts/additionalBenchmark.py b/scripts/additionalBenchmark.py index 6558439..638dd1c 100644 --- a/scripts/additionalBenchmark.py +++ b/scripts/additionalBenchmark.py @@ -5,6 +5,7 @@ from collections import defaultdict, Counter from corpusit import Vocab import nltk +import random import numpy as np import pandas as pd import torch @@ -26,11 +27,19 @@ @torch.no_grad() def main(): + seed = 0 + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.backends.cudnn.deterministic = True + parser = argparse.ArgumentParser() parser.add_argument( "--checkpointsMRPC", nargs="+", default=[ + "results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20finetune", "checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy", "checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10", "checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20", @@ -70,7 +79,6 @@ def main(): args = parser.parse_args() device = "cuda" - torch.set_default_device(device) sifA = 0.001 print("--------------------------------------------------------------------------------------------------------------------------------") @@ -105,19 +113,19 @@ def main(): print(f"\t\taccuracy: {accuracy}\n\t\tf1: {f1}\n") - with open(f'scripts/taskResults/MRPC/median/MRPC-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile: + with open(f'scripts/taskResults/MRPC/median/MRPC-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter='\t', quotechar='ß') writer.writerow(["index", "prediction"]) for index, pred in zip(range(len(predsMedianMRPC)), predsMedianMRPC): writer.writerow([index, pred]) - with open(f'scripts/taskResults/MRPC/threshold/MRPC-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile: + with open(f'scripts/taskResults/MRPC/threshold/MRPC-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter='\t', quotechar='ß') writer.writerow(["index", "prediction"]) for index, pred in zip(range(len(predsThresholdMRPC)), predsThresholdMRPC): writer.writerow([index, pred]) - with open(f'scripts/taskResults/MRPC/f1Threshold/MRPC-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile: + with open(f'scripts/taskResults/MRPC/f1Threshold/MRPC-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter='\t', quotechar='ß') writer.writerow(["index", "prediction"]) for index, pred in zip(range(len(predsF1ThresholdMRPC)), predsF1ThresholdMRPC): @@ -133,13 +141,13 @@ def main(): model = FireWord.from_pretrained(checkpoint).to(device) predsMedianSSTGlue, predsThresholdSSTGlue = predictSSTGlue(model, testPairsSSTGlue, devPairsSSTGlue, devLabelsSSTGlue, sifA) - with open(f'scripts/taskResults/SSTGLUE/median/SST-2-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile: + with open(f'scripts/taskResults/SSTGLUE/median/SST-2-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter='\t', quotechar='ß') writer.writerow(["index", "prediction"]) for index, pred in zip(range(len(predsMedianSSTGlue)), predsMedianSSTGlue): writer.writerow([index, pred]) - with open(f'scripts/taskResults/SSTGLUE/threshold/SST-2-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile: + with open(f'scripts/taskResults/SSTGLUE/threshold/SST-2-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter='\t', quotechar='ß') writer.writerow(["index", "prediction"]) for index, pred in zip(testIndicesSSTGlue, predsThresholdSSTGlue): @@ -154,13 +162,13 @@ def main(): model = FireWord.from_pretrained(checkpoint).to(device) predsMedianRTE, predsThresholdRTE = predictRTE(model, testPairsRTE, devPairsRTE, devLabelsRTE, sifA) - with open(f'scripts/taskResults/RTE/median/RTE-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile: + with open(f'scripts/taskResults/RTE/median/RTE-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter='\t', quotechar='ß') writer.writerow(["index", "prediction"]) for index, pred in zip(range(len(predsMedianRTE)), predsMedianRTE): writer.writerow([index, pred]) - with open(f'scripts/taskResults/RTE/threshold/RTE-{checkpoint[17:]}.tsv', 'w', newline='') as csvfile: + with open(f'scripts/taskResults/RTE/threshold/RTE-{str(model.config.dim) + model.config.func + model.config.measure}.tsv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter='\t', quotechar='ß') writer.writerow(["index", "prediction"]) for index, pred in zip(testIndicesRTE, predsThresholdRTE): @@ -231,7 +239,7 @@ def benchmarkMRPC( low = min(preds) high = max(preds) steps = math.ceil((high - low) / 2)*100 - + for threshold in np.linspace(low, high, steps): truePosCount = sum([int(preds[i] >= threshold) == 1 and labels[i] == 1 for i in range(len(preds))]) falsePosCount = sum([int(preds[i] >= threshold) == 1 and labels[i] == 0 for i in range(len(preds))]) diff --git a/scripts/fine-tuning/1_fine_tune_MRPC_all_checkpoints.bat b/scripts/fine-tuning/1_fine_tune_MRPC_all_checkpoints.bat index c091f4a..199ad3a 100644 --- a/scripts/fine-tuning/1_fine_tune_MRPC_all_checkpoints.bat +++ b/scripts/fine-tuning/1_fine_tune_MRPC_all_checkpoints.bat @@ -1,5 +1,5 @@ -python -m scripts.fineTune --sz_batch=200 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=600000 --eval_interval=1000 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=MRPC +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=MRPC -python -m scripts.fineTune --sz_batch=200 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=600000 --eval_interval=1000 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=MRPC +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=MRPC -python -m scripts.fineTune --sz_batch=200 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=600000 --eval_interval=1000 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=MRPC \ No newline at end of file +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=MRPC \ No newline at end of file diff --git a/scripts/fine-tuning/2_fine_tune_SST-2_all_checkpoints.bat b/scripts/fine-tuning/2_fine_tune_SST-2_all_checkpoints.bat index ffd4943..f279656 100644 --- a/scripts/fine-tuning/2_fine_tune_SST-2_all_checkpoints.bat +++ b/scripts/fine-tuning/2_fine_tune_SST-2_all_checkpoints.bat @@ -1,41 +1,5 @@ -python -m scripts.train \ - --sz_batch=32768 \ - --lr=0.005 \ - --lr_scheduler=OneCycleLR \ - --n_iters=600000 \ - --eval_interval=1000 \ - --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 \ - --optimizer=adamw \ - --seed=0 \ - --accum_steps=10 \ - --weight_decay=1e-6 \ - --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 - --task=MRPC +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/SST-2_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=SST-2 -python -m scripts.train \ - --sz_batch=32768 \ - --lr=0.005 \ - --lr_scheduler=OneCycleLR \ - --n_iters=600000 \ - --eval_interval=1000 \ - --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10 - --optimizer=adamw \ - --seed=0 \ - --accum_steps=10 \ - --weight_decay=1e-6 \ - --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 - --task=MRPC - -python -m scripts.train \ - --sz_batch=32768 \ - --lr=0.005 \ - --lr_scheduler=OneCycleLR \ - --n_iters=600000 \ - --eval_interval=1000 \ - --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy - --optimizer=adamw \ - --seed=0 \ - --accum_steps=10 \ - --weight_decay=1e-6 \ - --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy - --task=MRPC \ No newline at end of file +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/SST-2_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=SST-2 + +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/SST-2_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=SST-2 \ No newline at end of file diff --git a/scripts/fine-tuning/3_fine_tune_RTE_all_checkpoints.bat b/scripts/fine-tuning/3_fine_tune_RTE_all_checkpoints.bat index ffd4943..f58e6ed 100644 --- a/scripts/fine-tuning/3_fine_tune_RTE_all_checkpoints.bat +++ b/scripts/fine-tuning/3_fine_tune_RTE_all_checkpoints.bat @@ -1,41 +1,5 @@ -python -m scripts.train \ - --sz_batch=32768 \ - --lr=0.005 \ - --lr_scheduler=OneCycleLR \ - --n_iters=600000 \ - --eval_interval=1000 \ - --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l8_k20 \ - --optimizer=adamw \ - --seed=0 \ - --accum_steps=10 \ - --weight_decay=1e-6 \ - --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 - --task=MRPC +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/RTE_v1.1_wacky_mlplanardiv_d2_l8_k20 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l8_k20 --task=RTE -python -m scripts.train \ - --sz_batch=32768 \ - --lr=0.005 \ - --lr_scheduler=OneCycleLR \ - --n_iters=600000 \ - --eval_interval=1000 \ - --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k10 - --optimizer=adamw \ - --seed=0 \ - --accum_steps=10 \ - --weight_decay=1e-6 \ - --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 - --task=MRPC - -python -m scripts.train \ - --sz_batch=32768 \ - --lr=0.005 \ - --lr_scheduler=OneCycleLR \ - --n_iters=600000 \ - --eval_interval=1000 \ - --savedir=results/fineTuningResults/MRPC_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy - --optimizer=adamw \ - --seed=0 \ - --accum_steps=10 \ - --weight_decay=1e-6 \ - --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy - --task=MRPC \ No newline at end of file +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/RTE_v1.1_wacky_mlplanardiv_d2_l4_k10 --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k10 --task=RTE + +python -m scripts.fineTune --sz_batch=400 --lr=0.005 --lr_scheduler=OneCycleLR --n_iters=10000 --eval_interval=200 --savedir=results/fineTuningResults/RTE_v1.1_wacky_mlplanardiv_d2_l4_k1_polysemy --optimizer=adamw --seed=0 --accum_steps=10 --weight_decay=1e-6 --pretrainedModel=checkpoints/v1.1/wacky_mlplanardiv_d2_l4_k1_polysemy --task=RTE \ No newline at end of file diff --git a/scripts/fineTune.py b/scripts/fineTune.py index 251a5fb..25beee0 100644 --- a/scripts/fineTune.py +++ b/scripts/fineTune.py @@ -41,7 +41,8 @@ def fineTune(args): set_seed(args.seed) model = FireWord.from_pretrained(args.pretrainedModel) - + logger.info("model loaded") + sentencePairs = [] labels = [] if args.task == "MRPC": @@ -56,17 +57,44 @@ def fineTune(args): elif args.task == "RTE": sentencePairs, labels, evalPairs, evalLabels, _, _ = prepareRTEGlueData('scripts/tasks/RTE/train.tsv', 'scripts/tasks/RTE/dev.tsv', 'scripts/tasks/RTE/test.tsv') - + logger.info("data prepared") indices = list(range(len(sentencePairs[0]))) logger.info(model) model = model.to(device) - model.train() + model.eval() best_simscore = -9999 best_loss = 9999 + if args.task == "MRPC": + #F1 score + simscore = benchmarkMRPC(model, evalPairs, evalLabels)[4] + elif args.task == "SST-2": + #threshold search based predictions + predictions = predictSSTGlue(model, evalPairs, evalPairs, evalLabels)[1] + #accuracy + simscore = sum([predictions[x] == evalLabels[x] for x in range(len(predictions))]) / len(predictions) + elif args.task == "RTE": + #threshold search based predictions + predictions = predictRTE(model, evalPairs, evalPairs, evalLabels)[1] + #accuracy + simscore = sum([predictions[x] == evalLabels[x] for x in range(len(predictions))]) / len(predictions) + + logits = predictSentencePairs(model, evalPairs) + + lossSim = F.binary_cross_entropy_with_logits( + logits, + torch.tensor(evalLabels, dtype=torch.float, device=device, requires_grad=False), reduction="none" + ) + loss.add("sim", lossSim) + + total_loss = loss.reduced_total() + best_loss = total_loss.item() + + model.train() + if args.optimizer == "adamw": optimizer = AdamW( model.parameters(), lr=args.lr, weight_decay=args.weight_decay @@ -79,7 +107,11 @@ def fineTune(args): ) elif args.optimizer == "sgd": optimizer = SGD(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) + logger.info("optimizer set") + + if args.lr_scheduler == "OneCycleLR": + logger.info("pre onecycle") scheduler = OneCycleLR( optimizer, max_lr=args.lr, @@ -87,24 +119,33 @@ def fineTune(args): div_factor=1.0, final_div_factor=20.0, ) + logger.info("after onecycle") else: + logger.info("dummy") scheduler = DummyScheduler(args.lr) logger.info(f"Initialized optimizer and scheduler") logger.info(f" Optimizer: {optimizer}") logger.info(f" Scheduler: {scheduler}") + logger.info("scheduler set") + if args.amp: scaler = GradScaler() autocaster = autocast() else: autocaster = nullcontext() + logger.info("amp set") + if args.profile: prof = torch.autograd.profiler.profile(use_cuda=True) else: prof = nullcontext() - + logger.info("profiler set") + + logger.info("pre for loop ") for i in range(1, args.n_iters + 1): - + logger.info(f"iteration: {i}") + with Timer(elapsed, "prepare", sync_cuda=True): iterationIndices = random.sample(indices, args.sz_batch) iterationPairs = [[sentencePairs[0][x] for x in iterationIndices], [sentencePairs[1][x] for x in iterationIndices]] @@ -126,6 +167,7 @@ def fineTune(args): total_loss = loss.reduced_total() steploss = total_loss / args.accum_steps if args.profile: + pass logger.debug("----- forward -----") logger.debug(prof.key_averages().table(sort_by="self_cpu_time_total")) """ ----------------- backward pass -------------------""" @@ -135,29 +177,16 @@ def fineTune(args): else: steploss.backward() - # grad_norm = ( - # torch.cat([p.grad.data.reshape(-1) for p in model.parameters()]) - # .norm() - # .item() - # ) if args.profile: + pass logger.debug("----- backward -----") logger.debug(prof.key_averages().table(sort_by="self_cpu_time_total")) """ ----------------- optim -------------------""" if i % args.accum_steps == 0: with Timer(elapsed, "optim", sync_cuda=True): with Timer(elapsed, "step"): - for name, p in model.named_parameters(): - print(name, p, p.grad) - isnan = p.grad.isnan() - isinf = p.grad.isinf() - isinvalid = isnan | isinf - if isinvalid.any(): - p.grad.masked_fill_(isinvalid, 0) - p[isinvalid].normal_(0, 0.1) - print(f"Fixed nan/inf values in grad of {name}") - print(f" grad = {p.grad}") + logger.info(f"step iteration: {i}") if args.amp: scaler.step(optimizer) @@ -172,7 +201,7 @@ def fineTune(args): model.zero_grad() if i % args.eval_interval == 0: - + logger.info(f"eval iteration: {i}") os.makedirs(args.savedir, exist_ok=True) model.eval() @@ -241,13 +270,67 @@ def sentence_simmat(model, sents: List[List[str]], sif_weights: Mapping[str, flo ) return sentsim -def predictSentencePairs( +def predictSentencePairsWithDevThresholds( model: FireWord, pairs, + devPairs, + devLabels, sif_alpha=1e-3, ): vocab: Vocab = model.vocab + def computeThresholdFromDevData(): + counts = pd.Series(vocab.counts_dict()) + probs = counts / counts.sum() + sif_weights: Mapping[str, float] = { + w: sif_alpha / (sif_alpha + prob) for w, prob in probs.items() + } + + sents1 = devPairs[0] + sents2 = devPairs[1] + allsents = sents1 + sents2 + allsents = [ + [w for w in sent if w in sif_weights and w != vocab.unk] + for sent in allsents + ] + + """ similarity """ + with Timer(elapsed, "similarity", sync_cuda=True): + simmat = sentence_simmat(model, allsents, sif_weights) + + """ regularization: sim(i,j) <- sim(i,j) - 0.5 * (sim(i,i) + sim(j,j)) + halved bc (9)""" + with Timer(elapsed, "regularization"): + diag = np.diag(simmat) + simmat = simmat - 0.5 * (diag.reshape(-1, 1) + diag.reshape(1, -1)) + + with Timer(elapsed, "smooth"): + mean1 = np.mean(simmat, axis=1, keepdims=True) + std1 = np.std(simmat, axis=1, keepdims=True) + mean0 = np.mean(simmat, axis=0, keepdims=True) + std0 = np.std(simmat, axis=0, keepdims=True) + simmat = (simmat - (mean1 + mean0) / 2) / (std0 * std1) ** 0.5 + + N = len(devPairs[0]) + preds = [simmat[i, i + N] for i in range(N)] + preds = np.exp(preds) + preds = np.array(preds) + + bestThreshold = 0 + bestAccuracy = 0 + low = min(preds) + high = max(preds) + steps = math.ceil((high - low) / 2)*100 + + for threshold in np.linspace(low, high, steps): + accuracy = sum([int(preds[i] >= threshold) == (devLabels[i]) for i in range(len(preds))]) / len(preds) + if bestAccuracy < accuracy: + bestThreshold = threshold + bestAccuracy = accuracy + + + return bestThreshold + counts = pd.Series(vocab.counts_dict()) probs = counts / counts.sum() sif_weights: Mapping[str, float] = { @@ -257,18 +340,84 @@ def predictSentencePairs( sents1 = [ [w for w in sent if w in sif_weights and w != vocab.unk] for sent in pairs[0] ] sents2 = [ [w for w in sent if w in sif_weights and w != vocab.unk] for sent in pairs[1] ] + allsents = sents1 + sents2 + allsents = [ + [w for w in sent if w in sif_weights and w != vocab.unk] + for sent in allsents + ] + simmat = sentence_simmat(model, allsents, sif_weights) + + """ regularization: sim(i,j) <- sim(i,j) - 0.5 * (sim(i,i) + sim(j,j)) + halved bc (9)""" + + diag = np.diag(simmat) + simmat = simmat - 0.5 * (diag.reshape(-1, 1) + diag.reshape(1, -1)) + + """ smoothing by standardization """ + mean1 = np.mean(simmat, axis=1, keepdims=True) + std1 = np.std(simmat, axis=1, keepdims=True) + mean0 = np.mean(simmat, axis=0, keepdims=True) + std0 = np.std(simmat, axis=0, keepdims=True) + simmat = (simmat - (mean1 + mean0) / 2) / (std0 * std1) ** 0.5 + + simmat = [simmat[i, i + len(sents1)] for i in range(len(sents1))] + + simmat = [int(x) for x in simmat >= computeThresholdFromDevData()] + similarities = torch.zeros(len(sents1), requires_grad=True, device=device) similaritiesTmp = torch.zeros(len(sents1), requires_grad=False, device=device) + + for y in range(len(sents1)): + similaritiesTmp[y] = similaritiesTmp[y] + simmat[y] + + similarities = similarities + similaritiesTmp + + return similarities + +def predictSentencePairs( + model: FireWord, + pairs, + sif_alpha=1e-3, +): + vocab: Vocab = model.vocab + + counts = pd.Series(vocab.counts_dict()) + probs = counts / counts.sum() + sif_weights: Mapping[str, float] = { + w: sif_alpha / (sif_alpha + prob) for w, prob in probs.items() + } + + sents1 = [ [w for w in sent if w in sif_weights and w != vocab.unk] for sent in pairs[0] ] + sents2 = [ [w for w in sent if w in sif_weights and w != vocab.unk] for sent in pairs[1] ] + + allsents = sents1 + sents2 + allsents = [ + [w for w in sent if w in sif_weights and w != vocab.unk] + for sent in allsents + ] + simmat = sentence_simmat(model, allsents, sif_weights) + + """ regularization: sim(i,j) <- sim(i,j) - 0.5 * (sim(i,i) + sim(j,j)) + halved bc (9)""" + + diag = np.diag(simmat) + simmat = simmat - 0.5 * (diag.reshape(-1, 1) + diag.reshape(1, -1)) + + """ smoothing by standardization """ + mean1 = np.mean(simmat, axis=1, keepdims=True) + std1 = np.std(simmat, axis=1, keepdims=True) + mean0 = np.mean(simmat, axis=0, keepdims=True) + std0 = np.std(simmat, axis=0, keepdims=True) + simmat = (simmat - (mean1 + mean0) / 2) / (std0 * std1) ** 0.5 + + simmat = [simmat[i, i + len(sents1)] for i in range(len(sents1))] + similarities = torch.zeros(len(sents1), requires_grad=True, device=device) + similaritiesTmp = torch.zeros(len(sents1), requires_grad=False, device=device) + for y in range(len(sents1)): - sent1 = sents1[y] - sent2 = sents2[y] + similaritiesTmp[y] = similaritiesTmp[y] + simmat[y] - - for word1 in sent1: - for word2 in sent2: - similaritiesTmp[y] += (model[word1] * model[word2] * sif_weights[word1] * sif_weights[word2])[0] - similarities = similarities + similaritiesTmp return similarities @@ -379,6 +528,17 @@ def parse_arguments(): help = "Choose the benchmark task to fine-tune for." ) + def boolean_string(s): + if s not in {"False", "True"}: + raise ValueError("Not a valid boolean string") + return s == "True" + + parser.add_argument("--thresholdPrediction", + type=boolean_string, + default="False", + help="Whether the prediction uses the evaluation data to generate classification thresholds.", + ) + args = parser.parse_args() return args @@ -387,4 +547,4 @@ def parse_arguments(): if __name__ == "__main__": args = parse_arguments() - fineTune(args) + fineTune(args) \ No newline at end of file diff --git a/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k10.tsv b/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k10.tsv deleted file mode 100644 index f3c381c..0000000 --- a/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k10.tsv +++ /dev/null @@ -1,1726 +0,0 @@ -index prediction -0 1 -1 1 -2 1 -3 1 -4 0 -5 1 -6 1 -7 1 -8 1 -9 1 -10 1 -11 1 -12 1 -13 1 -14 1 -15 0 -16 1 -17 1 -18 1 -19 1 -20 1 -21 1 -22 1 -23 1 -24 1 -25 1 -26 1 -27 1 -28 1 -29 1 -30 1 -31 1 -32 1 -33 1 -34 1 -35 1 -36 1 -37 1 -38 1 -39 1 -40 1 -41 1 -42 0 -43 1 -44 1 -45 1 -46 1 -47 1 -48 1 -49 1 -50 1 -51 1 -52 1 -53 1 -54 1 -55 1 -56 1 -57 1 -58 1 -59 1 -60 1 -61 1 -62 1 -63 1 -64 1 -65 1 -66 1 -67 1 -68 1 -69 1 -70 1 -71 1 -72 1 -73 1 -74 1 -75 1 -76 1 -77 1 -78 1 -79 1 -80 1 -81 1 -82 1 -83 1 -84 1 -85 1 -86 1 -87 0 -88 1 -89 1 -90 1 -91 1 -92 1 -93 1 -94 1 -95 1 -96 1 -97 1 -98 0 -99 1 -100 1 -101 0 -102 1 -103 1 -104 1 -105 1 -106 1 -107 1 -108 1 -109 1 -110 1 -111 1 -112 1 -113 1 -114 1 -115 1 -116 1 -117 1 -118 1 -119 1 -120 1 -121 1 -122 1 -123 1 -124 1 -125 1 -126 1 -127 1 -128 1 -129 1 -130 1 -131 1 -132 1 -133 1 -134 1 -135 1 -136 1 -137 1 -138 1 -139 1 -140 1 -141 1 -142 1 -143 1 -144 1 -145 1 -146 1 -147 1 -148 1 -149 1 -150 1 -151 1 -152 1 -153 1 -154 1 -155 1 -156 1 -157 1 -158 1 -159 1 -160 1 -161 1 -162 1 -163 1 -164 1 -165 1 -166 1 -167 1 -168 1 -169 1 -170 1 -171 1 -172 1 -173 1 -174 1 -175 1 -176 1 -177 1 -178 1 -179 1 -180 1 -181 1 -182 1 -183 1 -184 1 -185 1 -186 1 -187 1 -188 1 -189 1 -190 1 -191 1 -192 1 -193 1 -194 1 -195 1 -196 1 -197 1 -198 1 -199 1 -200 1 -201 1 -202 1 -203 1 -204 1 -205 1 -206 1 -207 1 -208 0 -209 1 -210 1 -211 1 -212 1 -213 1 -214 0 -215 1 -216 1 -217 1 -218 1 -219 1 -220 1 -221 1 -222 1 -223 1 -224 1 -225 1 -226 1 -227 0 -228 1 -229 1 -230 1 -231 1 -232 1 -233 1 -234 1 -235 1 -236 1 -237 1 -238 1 -239 1 -240 1 -241 1 -242 1 -243 1 -244 1 -245 1 -246 1 -247 1 -248 1 -249 1 -250 1 -251 1 -252 1 -253 1 -254 1 -255 1 -256 1 -257 1 -258 1 -259 1 -260 1 -261 1 -262 1 -263 1 -264 1 -265 1 -266 1 -267 1 -268 1 -269 1 -270 1 -271 1 -272 1 -273 1 -274 1 -275 1 -276 1 -277 1 -278 1 -279 1 -280 1 -281 1 -282 1 -283 1 -284 1 -285 1 -286 1 -287 0 -288 1 -289 1 -290 1 -291 1 -292 1 -293 1 -294 1 -295 1 -296 1 -297 1 -298 1 -299 1 -300 1 -301 1 -302 1 -303 1 -304 1 -305 1 -306 1 -307 1 -308 1 -309 1 -310 1 -311 1 -312 1 -313 1 -314 1 -315 1 -316 1 -317 1 -318 1 -319 0 -320 1 -321 1 -322 1 -323 1 -324 1 -325 1 -326 1 -327 1 -328 1 -329 1 -330 1 -331 1 -332 1 -333 1 -334 1 -335 1 -336 1 -337 1 -338 1 -339 1 -340 1 -341 1 -342 1 -343 1 -344 1 -345 1 -346 1 -347 1 -348 1 -349 1 -350 1 -351 1 -352 1 -353 1 -354 1 -355 1 -356 1 -357 1 -358 1 -359 1 -360 1 -361 1 -362 1 -363 1 -364 1 -365 1 -366 1 -367 1 -368 1 -369 1 -370 1 -371 1 -372 1 -373 1 -374 1 -375 1 -376 1 -377 1 -378 1 -379 1 -380 1 -381 1 -382 1 -383 1 -384 1 -385 1 -386 1 -387 1 -388 1 -389 1 -390 1 -391 1 -392 1 -393 1 -394 1 -395 1 -396 1 -397 1 -398 1 -399 1 -400 1 -401 1 -402 1 -403 1 -404 1 -405 1 -406 1 -407 1 -408 1 -409 1 -410 1 -411 1 -412 1 -413 1 -414 1 -415 1 -416 1 -417 1 -418 1 -419 1 -420 1 -421 1 -422 1 -423 1 -424 1 -425 1 -426 1 -427 1 -428 1 -429 1 -430 1 -431 1 -432 1 -433 1 -434 1 -435 1 -436 1 -437 1 -438 1 -439 1 -440 0 -441 1 -442 1 -443 1 -444 1 -445 1 -446 1 -447 1 -448 1 -449 1 -450 0 -451 1 -452 1 -453 1 -454 1 -455 0 -456 1 -457 1 -458 1 -459 1 -460 1 -461 1 -462 1 -463 1 -464 1 -465 1 -466 1 -467 1 -468 1 -469 1 -470 1 -471 1 -472 1 -473 1 -474 1 -475 1 -476 1 -477 1 -478 1 -479 1 -480 1 -481 1 -482 1 -483 1 -484 0 -485 1 -486 1 -487 1 -488 1 -489 1 -490 1 -491 1 -492 1 -493 1 -494 0 -495 1 -496 1 -497 1 -498 1 -499 1 -500 1 -501 1 -502 1 -503 1 -504 1 -505 1 -506 1 -507 0 -508 1 -509 1 -510 1 -511 1 -512 1 -513 1 -514 1 -515 1 -516 1 -517 1 -518 1 -519 1 -520 1 -521 1 -522 1 -523 1 -524 1 -525 1 -526 1 -527 1 -528 1 -529 1 -530 1 -531 1 -532 1 -533 1 -534 1 -535 1 -536 1 -537 1 -538 1 -539 1 -540 1 -541 1 -542 1 -543 1 -544 1 -545 1 -546 1 -547 1 -548 1 -549 0 -550 1 -551 1 -552 1 -553 1 -554 1 -555 1 -556 1 -557 1 -558 1 -559 1 -560 1 -561 1 -562 1 -563 1 -564 1 -565 1 -566 1 -567 1 -568 1 -569 1 -570 1 -571 1 -572 1 -573 1 -574 1 -575 1 -576 1 -577 1 -578 1 -579 1 -580 1 -581 1 -582 1 -583 1 -584 1 -585 1 -586 1 -587 1 -588 1 -589 1 -590 1 -591 1 -592 1 -593 1 -594 1 -595 1 -596 1 -597 1 -598 1 -599 1 -600 1 -601 1 -602 1 -603 1 -604 1 -605 1 -606 1 -607 1 -608 1 -609 1 -610 1 -611 1 -612 1 -613 1 -614 1 -615 1 -616 1 -617 0 -618 1 -619 1 -620 1 -621 1 -622 1 -623 1 -624 1 -625 1 -626 1 -627 1 -628 1 -629 1 -630 1 -631 1 -632 1 -633 1 -634 1 -635 1 -636 1 -637 1 -638 1 -639 1 -640 1 -641 1 -642 1 -643 1 -644 1 -645 0 -646 1 -647 1 -648 1 -649 1 -650 1 -651 1 -652 1 -653 1 -654 1 -655 1 -656 1 -657 1 -658 1 -659 1 -660 1 -661 1 -662 1 -663 1 -664 1 -665 1 -666 1 -667 1 -668 1 -669 1 -670 1 -671 1 -672 1 -673 1 -674 1 -675 1 -676 1 -677 1 -678 1 -679 1 -680 1 -681 1 -682 1 -683 1 -684 1 -685 1 -686 1 -687 1 -688 1 -689 1 -690 1 -691 1 -692 1 -693 1 -694 1 -695 1 -696 1 -697 1 -698 1 -699 1 -700 1 -701 1 -702 1 -703 1 -704 1 -705 1 -706 1 -707 1 -708 1 -709 1 -710 1 -711 1 -712 1 -713 1 -714 1 -715 1 -716 1 -717 1 -718 1 -719 1 -720 1 -721 1 -722 1 -723 1 -724 1 -725 1 -726 1 -727 1 -728 1 -729 0 -730 1 -731 1 -732 1 -733 1 -734 1 -735 1 -736 1 -737 1 -738 1 -739 1 -740 1 -741 1 -742 1 -743 1 -744 1 -745 1 -746 1 -747 1 -748 1 -749 1 -750 1 -751 1 -752 1 -753 1 -754 1 -755 1 -756 1 -757 1 -758 1 -759 1 -760 1 -761 1 -762 1 -763 1 -764 1 -765 1 -766 1 -767 1 -768 1 -769 0 -770 1 -771 1 -772 1 -773 1 -774 1 -775 1 -776 1 -777 1 -778 1 -779 1 -780 1 -781 1 -782 1 -783 1 -784 1 -785 1 -786 1 -787 1 -788 1 -789 1 -790 1 -791 1 -792 1 -793 1 -794 1 -795 1 -796 1 -797 1 -798 1 -799 1 -800 1 -801 1 -802 1 -803 1 -804 1 -805 1 -806 1 -807 1 -808 1 -809 1 -810 1 -811 1 -812 1 -813 1 -814 1 -815 1 -816 1 -817 1 -818 1 -819 1 -820 1 -821 1 -822 0 -823 1 -824 1 -825 1 -826 1 -827 1 -828 0 -829 1 -830 1 -831 1 -832 1 -833 0 -834 1 -835 1 -836 1 -837 1 -838 1 -839 1 -840 1 -841 1 -842 1 -843 1 -844 1 -845 1 -846 1 -847 1 -848 1 -849 1 -850 1 -851 1 -852 1 -853 1 -854 1 -855 1 -856 1 -857 1 -858 1 -859 1 -860 1 -861 1 -862 1 -863 1 -864 1 -865 1 -866 1 -867 1 -868 1 -869 1 -870 1 -871 1 -872 1 -873 1 -874 1 -875 1 -876 0 -877 1 -878 0 -879 1 -880 1 -881 1 -882 1 -883 1 -884 1 -885 1 -886 1 -887 1 -888 1 -889 1 -890 1 -891 1 -892 1 -893 1 -894 1 -895 1 -896 1 -897 1 -898 1 -899 1 -900 1 -901 1 -902 1 -903 1 -904 1 -905 1 -906 1 -907 1 -908 1 -909 1 -910 1 -911 1 -912 1 -913 1 -914 1 -915 1 -916 1 -917 1 -918 1 -919 1 -920 1 -921 1 -922 1 -923 1 -924 1 -925 1 -926 1 -927 1 -928 1 -929 1 -930 1 -931 1 -932 1 -933 1 -934 1 -935 1 -936 1 -937 1 -938 1 -939 1 -940 1 -941 1 -942 1 -943 1 -944 1 -945 1 -946 1 -947 1 -948 1 -949 1 -950 1 -951 1 -952 1 -953 1 -954 1 -955 1 -956 1 -957 1 -958 1 -959 1 -960 1 -961 1 -962 1 -963 1 -964 1 -965 1 -966 1 -967 1 -968 1 -969 1 -970 1 -971 1 -972 0 -973 1 -974 1 -975 0 -976 1 -977 1 -978 1 -979 1 -980 1 -981 1 -982 1 -983 1 -984 1 -985 1 -986 1 -987 1 -988 1 -989 1 -990 1 -991 1 -992 1 -993 1 -994 1 -995 1 -996 1 -997 1 -998 1 -999 1 -1000 0 -1001 1 -1002 0 -1003 1 -1004 1 -1005 1 -1006 1 -1007 1 -1008 1 -1009 1 -1010 1 -1011 1 -1012 1 -1013 1 -1014 1 -1015 1 -1016 1 -1017 1 -1018 1 -1019 1 -1020 1 -1021 1 -1022 1 -1023 1 -1024 1 -1025 1 -1026 1 -1027 1 -1028 1 -1029 1 -1030 1 -1031 1 -1032 1 -1033 1 -1034 1 -1035 1 -1036 1 -1037 1 -1038 1 -1039 1 -1040 1 -1041 1 -1042 1 -1043 1 -1044 1 -1045 1 -1046 1 -1047 1 -1048 1 -1049 1 -1050 1 -1051 1 -1052 1 -1053 1 -1054 1 -1055 1 -1056 1 -1057 1 -1058 1 -1059 1 -1060 1 -1061 1 -1062 1 -1063 1 -1064 1 -1065 1 -1066 1 -1067 1 -1068 1 -1069 0 -1070 1 -1071 1 -1072 1 -1073 1 -1074 1 -1075 1 -1076 1 -1077 1 -1078 1 -1079 1 -1080 1 -1081 1 -1082 1 -1083 1 -1084 1 -1085 1 -1086 1 -1087 1 -1088 1 -1089 1 -1090 1 -1091 1 -1092 1 -1093 1 -1094 1 -1095 1 -1096 1 -1097 1 -1098 1 -1099 1 -1100 1 -1101 1 -1102 1 -1103 1 -1104 1 -1105 1 -1106 1 -1107 1 -1108 1 -1109 1 -1110 1 -1111 1 -1112 1 -1113 1 -1114 1 -1115 1 -1116 1 -1117 1 -1118 1 -1119 1 -1120 1 -1121 1 -1122 1 -1123 1 -1124 1 -1125 1 -1126 1 -1127 1 -1128 1 -1129 1 -1130 1 -1131 1 -1132 1 -1133 1 -1134 1 -1135 1 -1136 1 -1137 1 -1138 1 -1139 1 -1140 1 -1141 1 -1142 1 -1143 1 -1144 1 -1145 1 -1146 1 -1147 1 -1148 1 -1149 1 -1150 1 -1151 0 -1152 1 -1153 1 -1154 1 -1155 1 -1156 1 -1157 1 -1158 1 -1159 1 -1160 1 -1161 1 -1162 1 -1163 1 -1164 1 -1165 1 -1166 1 -1167 1 -1168 1 -1169 1 -1170 1 -1171 1 -1172 1 -1173 1 -1174 1 -1175 1 -1176 1 -1177 1 -1178 1 -1179 1 -1180 1 -1181 1 -1182 1 -1183 1 -1184 1 -1185 1 -1186 1 -1187 1 -1188 1 -1189 1 -1190 1 -1191 1 -1192 1 -1193 1 -1194 1 -1195 1 -1196 1 -1197 1 -1198 1 -1199 1 -1200 1 -1201 1 -1202 1 -1203 1 -1204 0 -1205 1 -1206 1 -1207 1 -1208 1 -1209 1 -1210 1 -1211 1 -1212 1 -1213 1 -1214 1 -1215 1 -1216 1 -1217 1 -1218 1 -1219 1 -1220 1 -1221 1 -1222 1 -1223 1 -1224 1 -1225 1 -1226 1 -1227 1 -1228 1 -1229 1 -1230 1 -1231 1 -1232 1 -1233 1 -1234 1 -1235 1 -1236 1 -1237 1 -1238 1 -1239 1 -1240 1 -1241 1 -1242 1 -1243 1 -1244 1 -1245 1 -1246 1 -1247 1 -1248 1 -1249 1 -1250 1 -1251 1 -1252 1 -1253 1 -1254 1 -1255 1 -1256 1 -1257 1 -1258 1 -1259 1 -1260 1 -1261 1 -1262 1 -1263 1 -1264 1 -1265 1 -1266 1 -1267 1 -1268 1 -1269 1 -1270 1 -1271 1 -1272 1 -1273 1 -1274 1 -1275 1 -1276 1 -1277 1 -1278 1 -1279 1 -1280 1 -1281 1 -1282 1 -1283 1 -1284 1 -1285 1 -1286 1 -1287 1 -1288 1 -1289 1 -1290 1 -1291 1 -1292 1 -1293 1 -1294 1 -1295 1 -1296 1 -1297 1 -1298 1 -1299 1 -1300 1 -1301 1 -1302 1 -1303 1 -1304 1 -1305 1 -1306 1 -1307 1 -1308 1 -1309 1 -1310 1 -1311 1 -1312 1 -1313 1 -1314 1 -1315 1 -1316 1 -1317 1 -1318 1 -1319 1 -1320 1 -1321 1 -1322 1 -1323 1 -1324 1 -1325 1 -1326 1 -1327 1 -1328 1 -1329 1 -1330 1 -1331 1 -1332 1 -1333 1 -1334 1 -1335 1 -1336 1 -1337 1 -1338 1 -1339 1 -1340 1 -1341 1 -1342 1 -1343 1 -1344 1 -1345 1 -1346 1 -1347 1 -1348 1 -1349 1 -1350 1 -1351 1 -1352 1 -1353 1 -1354 1 -1355 1 -1356 1 -1357 1 -1358 1 -1359 1 -1360 1 -1361 1 -1362 1 -1363 1 -1364 1 -1365 1 -1366 1 -1367 1 -1368 1 -1369 1 -1370 1 -1371 1 -1372 1 -1373 1 -1374 1 -1375 1 -1376 1 -1377 1 -1378 1 -1379 1 -1380 1 -1381 1 -1382 1 -1383 1 -1384 1 -1385 1 -1386 1 -1387 1 -1388 1 -1389 1 -1390 1 -1391 1 -1392 1 -1393 1 -1394 1 -1395 1 -1396 1 -1397 1 -1398 1 -1399 1 -1400 1 -1401 1 -1402 1 -1403 1 -1404 1 -1405 0 -1406 1 -1407 1 -1408 1 -1409 1 -1410 1 -1411 1 -1412 1 -1413 1 -1414 0 -1415 1 -1416 1 -1417 1 -1418 1 -1419 1 -1420 1 -1421 1 -1422 1 -1423 1 -1424 1 -1425 1 -1426 1 -1427 1 -1428 1 -1429 1 -1430 1 -1431 1 -1432 1 -1433 1 -1434 1 -1435 1 -1436 1 -1437 1 -1438 1 -1439 1 -1440 1 -1441 1 -1442 1 -1443 1 -1444 1 -1445 1 -1446 1 -1447 1 -1448 1 -1449 1 -1450 1 -1451 1 -1452 1 -1453 1 -1454 1 -1455 1 -1456 1 -1457 1 -1458 1 -1459 1 -1460 1 -1461 1 -1462 1 -1463 1 -1464 1 -1465 1 -1466 1 -1467 1 -1468 1 -1469 1 -1470 1 -1471 1 -1472 1 -1473 1 -1474 1 -1475 1 -1476 1 -1477 1 -1478 1 -1479 1 -1480 1 -1481 1 -1482 1 -1483 1 -1484 1 -1485 1 -1486 1 -1487 1 -1488 1 -1489 1 -1490 1 -1491 1 -1492 1 -1493 1 -1494 1 -1495 1 -1496 1 -1497 1 -1498 1 -1499 1 -1500 1 -1501 1 -1502 1 -1503 1 -1504 1 -1505 1 -1506 1 -1507 1 -1508 1 -1509 1 -1510 1 -1511 1 -1512 1 -1513 1 -1514 0 -1515 1 -1516 1 -1517 1 -1518 1 -1519 1 -1520 1 -1521 1 -1522 1 -1523 1 -1524 1 -1525 1 -1526 1 -1527 1 -1528 1 -1529 1 -1530 1 -1531 1 -1532 1 -1533 1 -1534 1 -1535 1 -1536 1 -1537 0 -1538 1 -1539 1 -1540 1 -1541 1 -1542 1 -1543 1 -1544 1 -1545 1 -1546 1 -1547 1 -1548 1 -1549 1 -1550 1 -1551 1 -1552 1 -1553 1 -1554 1 -1555 1 -1556 1 -1557 1 -1558 1 -1559 1 -1560 1 -1561 1 -1562 1 -1563 1 -1564 1 -1565 1 -1566 1 -1567 1 -1568 1 -1569 1 -1570 1 -1571 1 -1572 1 -1573 1 -1574 1 -1575 1 -1576 1 -1577 1 -1578 0 -1579 1 -1580 1 -1581 1 -1582 1 -1583 1 -1584 1 -1585 1 -1586 1 -1587 1 -1588 1 -1589 1 -1590 1 -1591 1 -1592 1 -1593 1 -1594 1 -1595 1 -1596 1 -1597 1 -1598 1 -1599 1 -1600 1 -1601 1 -1602 1 -1603 0 -1604 1 -1605 1 -1606 1 -1607 1 -1608 1 -1609 1 -1610 1 -1611 1 -1612 1 -1613 1 -1614 1 -1615 1 -1616 1 -1617 1 -1618 1 -1619 1 -1620 1 -1621 1 -1622 1 -1623 1 -1624 1 -1625 1 -1626 1 -1627 1 -1628 1 -1629 1 -1630 1 -1631 0 -1632 1 -1633 1 -1634 1 -1635 1 -1636 1 -1637 1 -1638 1 -1639 1 -1640 1 -1641 1 -1642 1 -1643 1 -1644 1 -1645 1 -1646 1 -1647 1 -1648 1 -1649 1 -1650 1 -1651 1 -1652 0 -1653 1 -1654 1 -1655 1 -1656 1 -1657 1 -1658 1 -1659 1 -1660 1 -1661 1 -1662 1 -1663 1 -1664 1 -1665 1 -1666 1 -1667 1 -1668 1 -1669 1 -1670 1 -1671 1 -1672 1 -1673 1 -1674 1 -1675 1 -1676 1 -1677 1 -1678 1 -1679 1 -1680 1 -1681 1 -1682 1 -1683 1 -1684 1 -1685 1 -1686 1 -1687 1 -1688 1 -1689 1 -1690 1 -1691 1 -1692 1 -1693 1 -1694 1 -1695 1 -1696 1 -1697 1 -1698 1 -1699 1 -1700 1 -1701 1 -1702 1 -1703 1 -1704 1 -1705 1 -1706 1 -1707 1 -1708 1 -1709 1 -1710 1 -1711 1 -1712 1 -1713 1 -1714 1 -1715 1 -1716 1 -1717 1 -1718 1 -1719 1 -1720 1 -1721 0 -1722 1 -1723 1 -1724 1 diff --git a/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k1_polysemy.tsv b/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k1_polysemy.tsv deleted file mode 100644 index f4ce503..0000000 --- a/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l4_k1_polysemy.tsv +++ /dev/null @@ -1,1726 +0,0 @@ -index prediction -0 1 -1 1 -2 1 -3 1 -4 1 -5 1 -6 1 -7 1 -8 1 -9 1 -10 1 -11 1 -12 1 -13 1 -14 1 -15 0 -16 1 -17 1 -18 1 -19 1 -20 1 -21 0 -22 1 -23 1 -24 1 -25 1 -26 1 -27 1 -28 1 -29 1 -30 1 -31 1 -32 1 -33 1 -34 1 -35 1 -36 1 -37 1 -38 1 -39 1 -40 1 -41 1 -42 1 -43 1 -44 1 -45 1 -46 1 -47 1 -48 1 -49 1 -50 0 -51 1 -52 1 -53 1 -54 1 -55 1 -56 1 -57 1 -58 1 -59 1 -60 1 -61 1 -62 1 -63 1 -64 1 -65 1 -66 1 -67 1 -68 1 -69 1 -70 1 -71 1 -72 1 -73 1 -74 1 -75 1 -76 1 -77 1 -78 1 -79 1 -80 1 -81 1 -82 1 -83 0 -84 1 -85 1 -86 1 -87 1 -88 1 -89 1 -90 1 -91 1 -92 1 -93 1 -94 1 -95 1 -96 1 -97 1 -98 1 -99 1 -100 1 -101 1 -102 1 -103 1 -104 1 -105 1 -106 1 -107 1 -108 1 -109 1 -110 1 -111 0 -112 0 -113 1 -114 1 -115 1 -116 1 -117 1 -118 1 -119 1 -120 1 -121 1 -122 1 -123 1 -124 1 -125 1 -126 1 -127 1 -128 1 -129 1 -130 1 -131 1 -132 1 -133 1 -134 1 -135 1 -136 1 -137 1 -138 1 -139 1 -140 1 -141 1 -142 0 -143 1 -144 1 -145 1 -146 1 -147 1 -148 1 -149 1 -150 1 -151 1 -152 1 -153 1 -154 1 -155 1 -156 1 -157 1 -158 1 -159 1 -160 1 -161 1 -162 1 -163 1 -164 1 -165 1 -166 1 -167 1 -168 1 -169 1 -170 1 -171 1 -172 1 -173 1 -174 1 -175 1 -176 1 -177 0 -178 1 -179 1 -180 1 -181 1 -182 1 -183 1 -184 1 -185 1 -186 1 -187 1 -188 1 -189 1 -190 1 -191 1 -192 1 -193 1 -194 1 -195 1 -196 1 -197 1 -198 1 -199 1 -200 1 -201 1 -202 1 -203 1 -204 1 -205 1 -206 1 -207 1 -208 1 -209 1 -210 1 -211 1 -212 1 -213 1 -214 1 -215 1 -216 1 -217 1 -218 1 -219 1 -220 1 -221 1 -222 1 -223 1 -224 1 -225 1 -226 1 -227 1 -228 1 -229 1 -230 1 -231 1 -232 1 -233 1 -234 1 -235 1 -236 1 -237 1 -238 1 -239 1 -240 1 -241 1 -242 1 -243 1 -244 1 -245 1 -246 1 -247 1 -248 1 -249 1 -250 1 -251 1 -252 0 -253 1 -254 1 -255 1 -256 1 -257 1 -258 1 -259 1 -260 1 -261 1 -262 1 -263 1 -264 1 -265 1 -266 1 -267 1 -268 1 -269 1 -270 1 -271 1 -272 1 -273 0 -274 1 -275 1 -276 1 -277 1 -278 1 -279 1 -280 1 -281 1 -282 1 -283 1 -284 1 -285 1 -286 1 -287 0 -288 1 -289 1 -290 1 -291 1 -292 1 -293 1 -294 1 -295 1 -296 1 -297 1 -298 1 -299 1 -300 1 -301 1 -302 1 -303 1 -304 1 -305 1 -306 0 -307 1 -308 1 -309 1 -310 1 -311 1 -312 1 -313 1 -314 1 -315 1 -316 1 -317 1 -318 1 -319 0 -320 1 -321 1 -322 1 -323 1 -324 1 -325 1 -326 1 -327 1 -328 1 -329 1 -330 1 -331 1 -332 1 -333 1 -334 1 -335 1 -336 1 -337 1 -338 1 -339 1 -340 1 -341 1 -342 1 -343 1 -344 1 -345 1 -346 1 -347 1 -348 1 -349 1 -350 1 -351 1 -352 1 -353 1 -354 1 -355 1 -356 1 -357 1 -358 1 -359 1 -360 1 -361 1 -362 1 -363 1 -364 1 -365 1 -366 1 -367 0 -368 1 -369 1 -370 1 -371 1 -372 1 -373 1 -374 1 -375 1 -376 1 -377 1 -378 1 -379 1 -380 1 -381 1 -382 1 -383 1 -384 1 -385 1 -386 1 -387 1 -388 1 -389 1 -390 1 -391 0 -392 1 -393 1 -394 1 -395 1 -396 1 -397 1 -398 1 -399 1 -400 1 -401 1 -402 1 -403 1 -404 0 -405 1 -406 1 -407 1 -408 1 -409 1 -410 1 -411 1 -412 1 -413 1 -414 1 -415 1 -416 1 -417 1 -418 1 -419 1 -420 1 -421 1 -422 1 -423 1 -424 1 -425 1 -426 1 -427 1 -428 1 -429 1 -430 1 -431 1 -432 1 -433 1 -434 1 -435 1 -436 1 -437 1 -438 1 -439 1 -440 1 -441 1 -442 1 -443 1 -444 1 -445 1 -446 1 -447 1 -448 1 -449 1 -450 0 -451 1 -452 1 -453 1 -454 1 -455 0 -456 1 -457 1 -458 1 -459 1 -460 1 -461 1 -462 1 -463 1 -464 1 -465 1 -466 1 -467 1 -468 1 -469 1 -470 1 -471 1 -472 1 -473 1 -474 1 -475 1 -476 1 -477 1 -478 1 -479 1 -480 1 -481 1 -482 1 -483 1 -484 0 -485 1 -486 1 -487 1 -488 1 -489 1 -490 1 -491 1 -492 1 -493 1 -494 1 -495 1 -496 1 -497 1 -498 1 -499 1 -500 1 -501 1 -502 1 -503 1 -504 1 -505 1 -506 0 -507 1 -508 1 -509 1 -510 1 -511 1 -512 1 -513 1 -514 1 -515 1 -516 1 -517 1 -518 1 -519 1 -520 1 -521 1 -522 1 -523 1 -524 1 -525 1 -526 1 -527 1 -528 1 -529 1 -530 1 -531 1 -532 1 -533 1 -534 1 -535 1 -536 1 -537 1 -538 1 -539 1 -540 1 -541 1 -542 1 -543 1 -544 1 -545 1 -546 1 -547 1 -548 1 -549 0 -550 1 -551 1 -552 1 -553 1 -554 0 -555 1 -556 1 -557 1 -558 1 -559 1 -560 1 -561 1 -562 1 -563 1 -564 1 -565 1 -566 1 -567 1 -568 1 -569 1 -570 1 -571 1 -572 1 -573 1 -574 1 -575 1 -576 1 -577 1 -578 1 -579 1 -580 1 -581 1 -582 1 -583 1 -584 1 -585 1 -586 1 -587 1 -588 1 -589 1 -590 0 -591 1 -592 1 -593 1 -594 1 -595 1 -596 1 -597 1 -598 1 -599 1 -600 1 -601 1 -602 1 -603 1 -604 1 -605 1 -606 1 -607 0 -608 1 -609 1 -610 1 -611 1 -612 1 -613 1 -614 1 -615 1 -616 1 -617 1 -618 1 -619 1 -620 1 -621 1 -622 1 -623 1 -624 1 -625 1 -626 1 -627 1 -628 1 -629 1 -630 1 -631 1 -632 1 -633 1 -634 1 -635 1 -636 1 -637 1 -638 1 -639 1 -640 1 -641 1 -642 1 -643 1 -644 1 -645 0 -646 1 -647 1 -648 1 -649 0 -650 1 -651 1 -652 1 -653 1 -654 1 -655 1 -656 1 -657 1 -658 1 -659 1 -660 1 -661 1 -662 1 -663 1 -664 1 -665 1 -666 1 -667 1 -668 1 -669 1 -670 1 -671 1 -672 1 -673 1 -674 1 -675 1 -676 1 -677 1 -678 1 -679 1 -680 1 -681 1 -682 1 -683 1 -684 1 -685 1 -686 1 -687 1 -688 1 -689 1 -690 1 -691 1 -692 1 -693 1 -694 1 -695 1 -696 1 -697 1 -698 1 -699 1 -700 1 -701 1 -702 1 -703 1 -704 1 -705 1 -706 1 -707 1 -708 1 -709 1 -710 1 -711 1 -712 1 -713 1 -714 1 -715 1 -716 1 -717 1 -718 1 -719 1 -720 1 -721 1 -722 1 -723 1 -724 1 -725 1 -726 1 -727 1 -728 1 -729 1 -730 1 -731 1 -732 1 -733 1 -734 1 -735 1 -736 1 -737 1 -738 1 -739 1 -740 1 -741 1 -742 1 -743 1 -744 1 -745 1 -746 1 -747 1 -748 1 -749 1 -750 0 -751 1 -752 1 -753 1 -754 1 -755 1 -756 1 -757 1 -758 1 -759 1 -760 1 -761 1 -762 1 -763 1 -764 1 -765 1 -766 1 -767 1 -768 1 -769 1 -770 1 -771 1 -772 1 -773 1 -774 1 -775 1 -776 1 -777 1 -778 1 -779 1 -780 1 -781 1 -782 1 -783 1 -784 1 -785 1 -786 1 -787 1 -788 1 -789 1 -790 1 -791 1 -792 1 -793 1 -794 1 -795 1 -796 1 -797 1 -798 1 -799 1 -800 1 -801 1 -802 1 -803 1 -804 0 -805 1 -806 1 -807 1 -808 1 -809 1 -810 1 -811 1 -812 1 -813 1 -814 1 -815 1 -816 1 -817 1 -818 1 -819 1 -820 1 -821 1 -822 1 -823 1 -824 1 -825 1 -826 1 -827 1 -828 0 -829 1 -830 1 -831 1 -832 1 -833 1 -834 1 -835 1 -836 1 -837 1 -838 1 -839 1 -840 1 -841 1 -842 1 -843 1 -844 1 -845 1 -846 1 -847 1 -848 1 -849 1 -850 1 -851 1 -852 1 -853 1 -854 1 -855 1 -856 1 -857 1 -858 1 -859 1 -860 1 -861 1 -862 1 -863 1 -864 1 -865 1 -866 1 -867 1 -868 1 -869 1 -870 1 -871 1 -872 1 -873 1 -874 1 -875 1 -876 1 -877 1 -878 1 -879 1 -880 1 -881 1 -882 1 -883 1 -884 1 -885 1 -886 1 -887 1 -888 1 -889 1 -890 1 -891 1 -892 1 -893 1 -894 1 -895 1 -896 1 -897 1 -898 1 -899 1 -900 1 -901 1 -902 1 -903 1 -904 1 -905 1 -906 1 -907 1 -908 1 -909 1 -910 1 -911 1 -912 1 -913 1 -914 1 -915 1 -916 1 -917 1 -918 1 -919 1 -920 1 -921 1 -922 1 -923 1 -924 1 -925 1 -926 1 -927 1 -928 1 -929 1 -930 1 -931 1 -932 1 -933 1 -934 1 -935 0 -936 1 -937 1 -938 0 -939 1 -940 1 -941 1 -942 1 -943 0 -944 1 -945 1 -946 1 -947 1 -948 1 -949 1 -950 0 -951 1 -952 1 -953 1 -954 1 -955 1 -956 1 -957 1 -958 1 -959 1 -960 1 -961 1 -962 1 -963 1 -964 1 -965 1 -966 1 -967 1 -968 1 -969 1 -970 1 -971 1 -972 1 -973 1 -974 1 -975 1 -976 1 -977 1 -978 1 -979 1 -980 1 -981 1 -982 1 -983 1 -984 1 -985 1 -986 1 -987 1 -988 1 -989 1 -990 1 -991 1 -992 1 -993 0 -994 1 -995 1 -996 1 -997 1 -998 1 -999 1 -1000 1 -1001 1 -1002 1 -1003 1 -1004 1 -1005 1 -1006 1 -1007 1 -1008 1 -1009 1 -1010 1 -1011 1 -1012 1 -1013 1 -1014 1 -1015 1 -1016 1 -1017 1 -1018 1 -1019 1 -1020 1 -1021 1 -1022 1 -1023 1 -1024 1 -1025 1 -1026 1 -1027 1 -1028 1 -1029 1 -1030 1 -1031 1 -1032 1 -1033 1 -1034 1 -1035 1 -1036 1 -1037 1 -1038 1 -1039 1 -1040 1 -1041 1 -1042 1 -1043 1 -1044 1 -1045 1 -1046 1 -1047 1 -1048 1 -1049 1 -1050 1 -1051 1 -1052 1 -1053 1 -1054 1 -1055 1 -1056 1 -1057 1 -1058 1 -1059 1 -1060 1 -1061 1 -1062 1 -1063 1 -1064 1 -1065 1 -1066 1 -1067 1 -1068 1 -1069 1 -1070 1 -1071 1 -1072 1 -1073 1 -1074 1 -1075 1 -1076 1 -1077 1 -1078 1 -1079 1 -1080 1 -1081 1 -1082 1 -1083 1 -1084 0 -1085 1 -1086 1 -1087 1 -1088 1 -1089 1 -1090 1 -1091 1 -1092 1 -1093 1 -1094 1 -1095 1 -1096 1 -1097 1 -1098 1 -1099 1 -1100 1 -1101 1 -1102 0 -1103 1 -1104 1 -1105 1 -1106 1 -1107 1 -1108 1 -1109 1 -1110 0 -1111 1 -1112 1 -1113 1 -1114 1 -1115 1 -1116 1 -1117 1 -1118 1 -1119 1 -1120 0 -1121 1 -1122 1 -1123 1 -1124 1 -1125 1 -1126 1 -1127 1 -1128 1 -1129 1 -1130 1 -1131 1 -1132 1 -1133 1 -1134 1 -1135 1 -1136 1 -1137 1 -1138 1 -1139 1 -1140 1 -1141 1 -1142 1 -1143 1 -1144 1 -1145 1 -1146 1 -1147 1 -1148 1 -1149 1 -1150 1 -1151 1 -1152 1 -1153 1 -1154 1 -1155 1 -1156 1 -1157 1 -1158 1 -1159 1 -1160 1 -1161 1 -1162 1 -1163 1 -1164 1 -1165 1 -1166 1 -1167 1 -1168 1 -1169 1 -1170 1 -1171 1 -1172 1 -1173 1 -1174 1 -1175 1 -1176 1 -1177 1 -1178 1 -1179 1 -1180 1 -1181 1 -1182 1 -1183 1 -1184 1 -1185 1 -1186 1 -1187 1 -1188 1 -1189 1 -1190 1 -1191 1 -1192 1 -1193 1 -1194 1 -1195 1 -1196 1 -1197 1 -1198 1 -1199 1 -1200 1 -1201 1 -1202 1 -1203 1 -1204 1 -1205 1 -1206 1 -1207 1 -1208 1 -1209 1 -1210 1 -1211 1 -1212 1 -1213 1 -1214 1 -1215 1 -1216 1 -1217 1 -1218 1 -1219 1 -1220 1 -1221 1 -1222 1 -1223 1 -1224 1 -1225 1 -1226 1 -1227 1 -1228 1 -1229 1 -1230 1 -1231 1 -1232 1 -1233 1 -1234 1 -1235 1 -1236 1 -1237 1 -1238 1 -1239 1 -1240 1 -1241 1 -1242 1 -1243 1 -1244 1 -1245 1 -1246 1 -1247 1 -1248 1 -1249 1 -1250 1 -1251 1 -1252 1 -1253 1 -1254 1 -1255 1 -1256 1 -1257 1 -1258 1 -1259 1 -1260 1 -1261 1 -1262 1 -1263 1 -1264 1 -1265 1 -1266 1 -1267 1 -1268 1 -1269 1 -1270 1 -1271 1 -1272 1 -1273 1 -1274 1 -1275 1 -1276 1 -1277 1 -1278 1 -1279 0 -1280 1 -1281 1 -1282 1 -1283 1 -1284 1 -1285 1 -1286 1 -1287 1 -1288 1 -1289 1 -1290 1 -1291 1 -1292 1 -1293 1 -1294 1 -1295 1 -1296 1 -1297 1 -1298 1 -1299 1 -1300 1 -1301 1 -1302 1 -1303 1 -1304 1 -1305 1 -1306 1 -1307 1 -1308 1 -1309 1 -1310 1 -1311 1 -1312 1 -1313 1 -1314 1 -1315 1 -1316 1 -1317 1 -1318 1 -1319 1 -1320 1 -1321 1 -1322 1 -1323 1 -1324 1 -1325 1 -1326 1 -1327 1 -1328 1 -1329 1 -1330 1 -1331 1 -1332 1 -1333 1 -1334 1 -1335 1 -1336 1 -1337 1 -1338 1 -1339 1 -1340 1 -1341 1 -1342 1 -1343 1 -1344 1 -1345 1 -1346 1 -1347 1 -1348 1 -1349 1 -1350 1 -1351 1 -1352 1 -1353 1 -1354 1 -1355 1 -1356 1 -1357 1 -1358 1 -1359 1 -1360 1 -1361 1 -1362 1 -1363 1 -1364 1 -1365 1 -1366 1 -1367 1 -1368 1 -1369 1 -1370 1 -1371 1 -1372 1 -1373 1 -1374 1 -1375 1 -1376 1 -1377 1 -1378 1 -1379 1 -1380 1 -1381 1 -1382 1 -1383 1 -1384 1 -1385 1 -1386 1 -1387 1 -1388 1 -1389 1 -1390 1 -1391 1 -1392 1 -1393 1 -1394 1 -1395 1 -1396 1 -1397 1 -1398 1 -1399 1 -1400 1 -1401 1 -1402 1 -1403 1 -1404 1 -1405 1 -1406 1 -1407 1 -1408 1 -1409 1 -1410 1 -1411 1 -1412 1 -1413 1 -1414 0 -1415 1 -1416 1 -1417 1 -1418 1 -1419 1 -1420 1 -1421 1 -1422 1 -1423 1 -1424 1 -1425 1 -1426 1 -1427 1 -1428 1 -1429 1 -1430 1 -1431 1 -1432 1 -1433 1 -1434 1 -1435 1 -1436 1 -1437 1 -1438 1 -1439 1 -1440 1 -1441 1 -1442 1 -1443 1 -1444 1 -1445 1 -1446 1 -1447 1 -1448 1 -1449 1 -1450 1 -1451 1 -1452 1 -1453 1 -1454 1 -1455 1 -1456 1 -1457 1 -1458 1 -1459 1 -1460 1 -1461 1 -1462 1 -1463 1 -1464 1 -1465 1 -1466 0 -1467 1 -1468 1 -1469 1 -1470 1 -1471 1 -1472 1 -1473 1 -1474 1 -1475 1 -1476 0 -1477 1 -1478 1 -1479 1 -1480 1 -1481 1 -1482 1 -1483 1 -1484 1 -1485 1 -1486 0 -1487 1 -1488 1 -1489 1 -1490 1 -1491 1 -1492 1 -1493 1 -1494 1 -1495 1 -1496 1 -1497 1 -1498 1 -1499 0 -1500 1 -1501 1 -1502 1 -1503 0 -1504 1 -1505 1 -1506 1 -1507 1 -1508 1 -1509 1 -1510 1 -1511 1 -1512 1 -1513 1 -1514 1 -1515 1 -1516 1 -1517 1 -1518 1 -1519 1 -1520 1 -1521 1 -1522 1 -1523 1 -1524 1 -1525 1 -1526 1 -1527 1 -1528 0 -1529 1 -1530 1 -1531 1 -1532 1 -1533 1 -1534 1 -1535 1 -1536 1 -1537 1 -1538 1 -1539 1 -1540 1 -1541 1 -1542 1 -1543 1 -1544 1 -1545 1 -1546 1 -1547 1 -1548 1 -1549 1 -1550 1 -1551 1 -1552 1 -1553 1 -1554 1 -1555 1 -1556 1 -1557 1 -1558 1 -1559 1 -1560 1 -1561 1 -1562 1 -1563 1 -1564 1 -1565 1 -1566 1 -1567 1 -1568 1 -1569 1 -1570 1 -1571 1 -1572 1 -1573 1 -1574 1 -1575 1 -1576 1 -1577 1 -1578 0 -1579 1 -1580 1 -1581 1 -1582 1 -1583 1 -1584 1 -1585 1 -1586 1 -1587 1 -1588 1 -1589 1 -1590 0 -1591 1 -1592 1 -1593 1 -1594 1 -1595 1 -1596 1 -1597 1 -1598 1 -1599 1 -1600 1 -1601 1 -1602 1 -1603 1 -1604 1 -1605 1 -1606 1 -1607 1 -1608 1 -1609 1 -1610 1 -1611 1 -1612 1 -1613 1 -1614 1 -1615 1 -1616 1 -1617 1 -1618 1 -1619 1 -1620 1 -1621 1 -1622 1 -1623 1 -1624 1 -1625 1 -1626 1 -1627 1 -1628 1 -1629 1 -1630 1 -1631 1 -1632 1 -1633 1 -1634 1 -1635 1 -1636 1 -1637 1 -1638 1 -1639 1 -1640 1 -1641 1 -1642 1 -1643 1 -1644 1 -1645 1 -1646 1 -1647 1 -1648 1 -1649 1 -1650 1 -1651 1 -1652 1 -1653 1 -1654 1 -1655 1 -1656 1 -1657 1 -1658 1 -1659 1 -1660 1 -1661 1 -1662 1 -1663 1 -1664 1 -1665 1 -1666 1 -1667 1 -1668 1 -1669 1 -1670 1 -1671 0 -1672 1 -1673 1 -1674 1 -1675 1 -1676 1 -1677 1 -1678 1 -1679 1 -1680 1 -1681 1 -1682 1 -1683 1 -1684 1 -1685 1 -1686 1 -1687 1 -1688 1 -1689 1 -1690 1 -1691 1 -1692 1 -1693 1 -1694 1 -1695 1 -1696 1 -1697 1 -1698 1 -1699 1 -1700 1 -1701 1 -1702 1 -1703 1 -1704 1 -1705 1 -1706 1 -1707 1 -1708 1 -1709 1 -1710 1 -1711 1 -1712 1 -1713 1 -1714 1 -1715 1 -1716 1 -1717 1 -1718 1 -1719 1 -1720 0 -1721 1 -1722 1 -1723 1 -1724 1 diff --git a/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l8_k20.tsv b/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l8_k20.tsv deleted file mode 100644 index 2e4ed1d..0000000 --- a/scripts/taskResults/MRPC/threshold/MRPC-wacky_mlplanardiv_d2_l8_k20.tsv +++ /dev/null @@ -1,1726 +0,0 @@ -index prediction -0 1 -1 1 -2 1 -3 1 -4 1 -5 1 -6 1 -7 1 -8 1 -9 1 -10 1 -11 1 -12 1 -13 1 -14 1 -15 1 -16 1 -17 1 -18 1 -19 1 -20 1 -21 1 -22 1 -23 1 -24 1 -25 1 -26 1 -27 1 -28 1 -29 0 -30 1 -31 1 -32 1 -33 1 -34 1 -35 1 -36 1 -37 1 -38 1 -39 1 -40 1 -41 1 -42 0 -43 1 -44 0 -45 1 -46 1 -47 1 -48 1 -49 1 -50 1 -51 1 -52 1 -53 1 -54 1 -55 1 -56 1 -57 1 -58 1 -59 1 -60 1 -61 1 -62 1 -63 1 -64 1 -65 1 -66 1 -67 1 -68 1 -69 1 -70 1 -71 1 -72 1 -73 1 -74 1 -75 1 -76 1 -77 1 -78 1 -79 1 -80 1 -81 1 -82 1 -83 1 -84 0 -85 1 -86 1 -87 1 -88 0 -89 1 -90 1 -91 1 -92 1 -93 1 -94 1 -95 1 -96 1 -97 1 -98 1 -99 1 -100 1 -101 0 -102 1 -103 1 -104 1 -105 0 -106 1 -107 1 -108 1 -109 0 -110 1 -111 0 -112 1 -113 1 -114 1 -115 1 -116 1 -117 1 -118 1 -119 1 -120 1 -121 1 -122 1 -123 1 -124 1 -125 1 -126 1 -127 1 -128 0 -129 1 -130 1 -131 1 -132 1 -133 1 -134 1 -135 1 -136 0 -137 1 -138 1 -139 1 -140 1 -141 1 -142 1 -143 1 -144 1 -145 1 -146 1 -147 1 -148 1 -149 1 -150 1 -151 1 -152 0 -153 1 -154 1 -155 1 -156 1 -157 1 -158 1 -159 1 -160 1 -161 1 -162 1 -163 1 -164 1 -165 1 -166 1 -167 1 -168 1 -169 1 -170 1 -171 1 -172 1 -173 1 -174 1 -175 1 -176 1 -177 1 -178 0 -179 1 -180 1 -181 1 -182 1 -183 1 -184 1 -185 1 -186 1 -187 1 -188 1 -189 1 -190 1 -191 1 -192 1 -193 1 -194 1 -195 1 -196 1 -197 1 -198 1 -199 1 -200 1 -201 1 -202 1 -203 1 -204 1 -205 1 -206 1 -207 1 -208 0 -209 1 -210 1 -211 1 -212 1 -213 1 -214 1 -215 1 -216 1 -217 1 -218 1 -219 1 -220 1 -221 1 -222 1 -223 1 -224 1 -225 1 -226 1 -227 0 -228 1 -229 1 -230 1 -231 1 -232 1 -233 1 -234 1 -235 1 -236 1 -237 1 -238 1 -239 1 -240 1 -241 1 -242 1 -243 1 -244 1 -245 1 -246 1 -247 1 -248 1 -249 1 -250 1 -251 1 -252 1 -253 1 -254 1 -255 1 -256 1 -257 1 -258 1 -259 1 -260 1 -261 1 -262 1 -263 1 -264 1 -265 1 -266 1 -267 0 -268 1 -269 0 -270 1 -271 1 -272 1 -273 1 -274 1 -275 1 -276 1 -277 1 -278 1 -279 1 -280 1 -281 1 -282 1 -283 1 -284 1 -285 1 -286 1 -287 0 -288 1 -289 1 -290 1 -291 1 -292 1 -293 1 -294 1 -295 1 -296 0 -297 1 -298 1 -299 1 -300 1 -301 1 -302 1 -303 1 -304 1 -305 1 -306 0 -307 1 -308 1 -309 1 -310 1 -311 1 -312 1 -313 1 -314 1 -315 1 -316 1 -317 1 -318 1 -319 0 -320 1 -321 1 -322 1 -323 1 -324 1 -325 1 -326 1 -327 1 -328 1 -329 1 -330 1 -331 1 -332 1 -333 1 -334 1 -335 1 -336 1 -337 1 -338 1 -339 1 -340 1 -341 1 -342 1 -343 1 -344 1 -345 1 -346 1 -347 1 -348 1 -349 1 -350 1 -351 1 -352 1 -353 1 -354 1 -355 1 -356 1 -357 1 -358 1 -359 1 -360 1 -361 1 -362 1 -363 1 -364 1 -365 1 -366 1 -367 0 -368 1 -369 1 -370 1 -371 1 -372 1 -373 1 -374 1 -375 1 -376 1 -377 1 -378 1 -379 1 -380 0 -381 1 -382 1 -383 1 -384 1 -385 1 -386 1 -387 1 -388 1 -389 1 -390 1 -391 1 -392 1 -393 1 -394 1 -395 1 -396 1 -397 1 -398 1 -399 1 -400 1 -401 1 -402 1 -403 1 -404 1 -405 0 -406 1 -407 1 -408 1 -409 1 -410 1 -411 1 -412 1 -413 1 -414 1 -415 1 -416 1 -417 1 -418 1 -419 1 -420 1 -421 1 -422 1 -423 1 -424 1 -425 1 -426 1 -427 1 -428 1 -429 1 -430 1 -431 1 -432 1 -433 1 -434 1 -435 1 -436 1 -437 1 -438 1 -439 1 -440 0 -441 1 -442 1 -443 1 -444 0 -445 1 -446 1 -447 1 -448 1 -449 1 -450 0 -451 1 -452 1 -453 1 -454 1 -455 0 -456 1 -457 1 -458 1 -459 1 -460 1 -461 1 -462 1 -463 1 -464 1 -465 1 -466 1 -467 1 -468 1 -469 1 -470 1 -471 1 -472 1 -473 1 -474 1 -475 1 -476 1 -477 1 -478 1 -479 1 -480 1 -481 1 -482 1 -483 1 -484 0 -485 1 -486 1 -487 1 -488 1 -489 1 -490 1 -491 1 -492 1 -493 1 -494 1 -495 1 -496 1 -497 1 -498 1 -499 1 -500 1 -501 1 -502 1 -503 1 -504 1 -505 1 -506 1 -507 1 -508 1 -509 1 -510 1 -511 1 -512 1 -513 1 -514 1 -515 1 -516 1 -517 1 -518 1 -519 1 -520 1 -521 1 -522 1 -523 1 -524 1 -525 1 -526 1 -527 0 -528 1 -529 1 -530 1 -531 1 -532 0 -533 1 -534 1 -535 1 -536 1 -537 1 -538 1 -539 1 -540 1 -541 1 -542 1 -543 1 -544 1 -545 1 -546 1 -547 1 -548 1 -549 1 -550 1 -551 1 -552 1 -553 1 -554 1 -555 1 -556 1 -557 1 -558 1 -559 1 -560 1 -561 1 -562 1 -563 1 -564 1 -565 1 -566 1 -567 1 -568 1 -569 1 -570 1 -571 1 -572 1 -573 1 -574 1 -575 1 -576 1 -577 1 -578 1 -579 1 -580 1 -581 1 -582 1 -583 1 -584 1 -585 1 -586 1 -587 1 -588 1 -589 1 -590 1 -591 1 -592 1 -593 1 -594 1 -595 1 -596 1 -597 1 -598 1 -599 1 -600 1 -601 1 -602 1 -603 1 -604 1 -605 1 -606 1 -607 0 -608 1 -609 1 -610 1 -611 1 -612 1 -613 1 -614 1 -615 1 -616 1 -617 1 -618 1 -619 1 -620 1 -621 1 -622 1 -623 1 -624 1 -625 1 -626 1 -627 1 -628 1 -629 1 -630 0 -631 1 -632 1 -633 0 -634 1 -635 1 -636 1 -637 1 -638 1 -639 1 -640 1 -641 1 -642 1 -643 1 -644 1 -645 0 -646 1 -647 1 -648 0 -649 0 -650 1 -651 1 -652 1 -653 1 -654 1 -655 1 -656 1 -657 1 -658 1 -659 1 -660 1 -661 1 -662 1 -663 1 -664 1 -665 1 -666 1 -667 1 -668 1 -669 1 -670 1 -671 1 -672 1 -673 1 -674 1 -675 1 -676 1 -677 1 -678 1 -679 1 -680 1 -681 1 -682 1 -683 1 -684 1 -685 1 -686 1 -687 1 -688 1 -689 1 -690 1 -691 1 -692 1 -693 1 -694 1 -695 1 -696 1 -697 1 -698 1 -699 1 -700 0 -701 1 -702 1 -703 1 -704 1 -705 1 -706 0 -707 1 -708 1 -709 1 -710 1 -711 0 -712 1 -713 1 -714 1 -715 1 -716 1 -717 1 -718 0 -719 1 -720 1 -721 1 -722 1 -723 1 -724 1 -725 1 -726 1 -727 1 -728 1 -729 1 -730 1 -731 1 -732 1 -733 1 -734 1 -735 1 -736 1 -737 1 -738 1 -739 1 -740 1 -741 1 -742 1 -743 1 -744 1 -745 1 -746 1 -747 1 -748 1 -749 1 -750 1 -751 1 -752 1 -753 1 -754 1 -755 1 -756 1 -757 1 -758 1 -759 1 -760 1 -761 1 -762 1 -763 1 -764 1 -765 1 -766 1 -767 1 -768 1 -769 0 -770 1 -771 1 -772 1 -773 1 -774 1 -775 1 -776 1 -777 1 -778 1 -779 1 -780 1 -781 1 -782 1 -783 1 -784 0 -785 1 -786 1 -787 1 -788 1 -789 1 -790 1 -791 1 -792 1 -793 1 -794 1 -795 1 -796 0 -797 1 -798 1 -799 1 -800 1 -801 1 -802 1 -803 1 -804 1 -805 1 -806 1 -807 0 -808 1 -809 1 -810 1 -811 1 -812 1 -813 1 -814 1 -815 1 -816 1 -817 1 -818 1 -819 1 -820 1 -821 1 -822 0 -823 1 -824 1 -825 1 -826 1 -827 1 -828 0 -829 0 -830 1 -831 1 -832 1 -833 0 -834 0 -835 1 -836 1 -837 1 -838 1 -839 1 -840 1 -841 1 -842 1 -843 1 -844 1 -845 1 -846 1 -847 1 -848 1 -849 1 -850 1 -851 1 -852 1 -853 1 -854 1 -855 1 -856 1 -857 1 -858 0 -859 1 -860 1 -861 1 -862 1 -863 1 -864 1 -865 1 -866 0 -867 1 -868 1 -869 1 -870 0 -871 1 -872 1 -873 1 -874 1 -875 1 -876 1 -877 1 -878 1 -879 1 -880 1 -881 1 -882 1 -883 1 -884 1 -885 1 -886 1 -887 1 -888 1 -889 1 -890 1 -891 1 -892 1 -893 1 -894 1 -895 1 -896 1 -897 1 -898 1 -899 1 -900 1 -901 1 -902 1 -903 1 -904 1 -905 1 -906 1 -907 1 -908 1 -909 1 -910 1 -911 1 -912 1 -913 1 -914 1 -915 1 -916 1 -917 1 -918 0 -919 1 -920 1 -921 1 -922 1 -923 1 -924 1 -925 1 -926 1 -927 1 -928 1 -929 1 -930 1 -931 1 -932 1 -933 1 -934 1 -935 1 -936 1 -937 1 -938 0 -939 1 -940 1 -941 1 -942 1 -943 0 -944 1 -945 1 -946 0 -947 1 -948 1 -949 1 -950 0 -951 1 -952 1 -953 1 -954 1 -955 1 -956 1 -957 1 -958 1 -959 1 -960 1 -961 1 -962 1 -963 1 -964 1 -965 1 -966 1 -967 1 -968 1 -969 1 -970 1 -971 1 -972 1 -973 1 -974 1 -975 0 -976 1 -977 1 -978 1 -979 1 -980 1 -981 1 -982 1 -983 1 -984 1 -985 1 -986 1 -987 1 -988 1 -989 1 -990 1 -991 1 -992 1 -993 1 -994 1 -995 1 -996 1 -997 1 -998 1 -999 1 -1000 0 -1001 1 -1002 0 -1003 1 -1004 1 -1005 1 -1006 1 -1007 1 -1008 1 -1009 1 -1010 1 -1011 1 -1012 1 -1013 1 -1014 1 -1015 1 -1016 1 -1017 1 -1018 1 -1019 1 -1020 1 -1021 1 -1022 1 -1023 1 -1024 1 -1025 1 -1026 1 -1027 1 -1028 1 -1029 1 -1030 1 -1031 1 -1032 1 -1033 0 -1034 1 -1035 1 -1036 1 -1037 1 -1038 1 -1039 1 -1040 1 -1041 1 -1042 1 -1043 0 -1044 1 -1045 1 -1046 1 -1047 1 -1048 1 -1049 1 -1050 1 -1051 1 -1052 1 -1053 1 -1054 1 -1055 1 -1056 1 -1057 1 -1058 1 -1059 1 -1060 1 -1061 1 -1062 0 -1063 1 -1064 1 -1065 1 -1066 1 -1067 1 -1068 1 -1069 1 -1070 1 -1071 1 -1072 1 -1073 1 -1074 1 -1075 0 -1076 1 -1077 1 -1078 1 -1079 1 -1080 1 -1081 1 -1082 1 -1083 1 -1084 0 -1085 1 -1086 1 -1087 1 -1088 0 -1089 1 -1090 1 -1091 1 -1092 1 -1093 1 -1094 1 -1095 1 -1096 1 -1097 1 -1098 1 -1099 1 -1100 1 -1101 1 -1102 1 -1103 1 -1104 1 -1105 1 -1106 1 -1107 1 -1108 1 -1109 1 -1110 1 -1111 1 -1112 1 -1113 1 -1114 1 -1115 1 -1116 1 -1117 1 -1118 1 -1119 1 -1120 0 -1121 1 -1122 1 -1123 1 -1124 1 -1125 1 -1126 1 -1127 1 -1128 1 -1129 1 -1130 1 -1131 1 -1132 1 -1133 0 -1134 1 -1135 1 -1136 1 -1137 1 -1138 1 -1139 1 -1140 1 -1141 1 -1142 1 -1143 1 -1144 1 -1145 1 -1146 1 -1147 1 -1148 1 -1149 1 -1150 1 -1151 1 -1152 1 -1153 1 -1154 1 -1155 1 -1156 1 -1157 1 -1158 1 -1159 1 -1160 1 -1161 1 -1162 1 -1163 1 -1164 1 -1165 0 -1166 1 -1167 1 -1168 1 -1169 1 -1170 1 -1171 1 -1172 1 -1173 1 -1174 1 -1175 1 -1176 1 -1177 1 -1178 1 -1179 1 -1180 1 -1181 1 -1182 1 -1183 1 -1184 1 -1185 1 -1186 1 -1187 1 -1188 1 -1189 1 -1190 1 -1191 1 -1192 1 -1193 1 -1194 1 -1195 1 -1196 1 -1197 1 -1198 1 -1199 1 -1200 1 -1201 1 -1202 1 -1203 1 -1204 0 -1205 1 -1206 1 -1207 1 -1208 1 -1209 1 -1210 1 -1211 1 -1212 1 -1213 1 -1214 1 -1215 1 -1216 1 -1217 1 -1218 1 -1219 1 -1220 1 -1221 1 -1222 1 -1223 1 -1224 1 -1225 1 -1226 1 -1227 1 -1228 1 -1229 1 -1230 1 -1231 1 -1232 1 -1233 1 -1234 1 -1235 1 -1236 1 -1237 1 -1238 1 -1239 1 -1240 1 -1241 1 -1242 1 -1243 1 -1244 1 -1245 1 -1246 1 -1247 1 -1248 1 -1249 1 -1250 1 -1251 1 -1252 1 -1253 1 -1254 1 -1255 1 -1256 1 -1257 1 -1258 1 -1259 1 -1260 1 -1261 1 -1262 1 -1263 1 -1264 1 -1265 1 -1266 1 -1267 1 -1268 1 -1269 1 -1270 1 -1271 1 -1272 0 -1273 1 -1274 1 -1275 1 -1276 1 -1277 1 -1278 0 -1279 1 -1280 1 -1281 1 -1282 1 -1283 1 -1284 1 -1285 1 -1286 1 -1287 1 -1288 1 -1289 1 -1290 1 -1291 1 -1292 1 -1293 1 -1294 1 -1295 1 -1296 1 -1297 1 -1298 1 -1299 1 -1300 1 -1301 1 -1302 1 -1303 1 -1304 1 -1305 1 -1306 1 -1307 1 -1308 1 -1309 1 -1310 1 -1311 1 -1312 1 -1313 1 -1314 1 -1315 1 -1316 1 -1317 1 -1318 1 -1319 1 -1320 1 -1321 1 -1322 1 -1323 1 -1324 1 -1325 1 -1326 1 -1327 1 -1328 1 -1329 1 -1330 1 -1331 1 -1332 0 -1333 1 -1334 1 -1335 1 -1336 1 -1337 1 -1338 1 -1339 1 -1340 1 -1341 1 -1342 1 -1343 1 -1344 1 -1345 1 -1346 1 -1347 1 -1348 1 -1349 1 -1350 1 -1351 1 -1352 1 -1353 1 -1354 1 -1355 1 -1356 1 -1357 1 -1358 1 -1359 1 -1360 1 -1361 1 -1362 1 -1363 1 -1364 1 -1365 1 -1366 1 -1367 1 -1368 1 -1369 0 -1370 1 -1371 1 -1372 1 -1373 1 -1374 1 -1375 1 -1376 1 -1377 1 -1378 1 -1379 1 -1380 1 -1381 1 -1382 1 -1383 1 -1384 1 -1385 1 -1386 1 -1387 1 -1388 1 -1389 1 -1390 1 -1391 1 -1392 1 -1393 1 -1394 1 -1395 1 -1396 1 -1397 0 -1398 1 -1399 1 -1400 1 -1401 1 -1402 1 -1403 1 -1404 1 -1405 0 -1406 1 -1407 1 -1408 1 -1409 1 -1410 1 -1411 1 -1412 1 -1413 1 -1414 0 -1415 1 -1416 1 -1417 1 -1418 1 -1419 1 -1420 1 -1421 1 -1422 1 -1423 1 -1424 1 -1425 1 -1426 1 -1427 1 -1428 1 -1429 1 -1430 1 -1431 1 -1432 1 -1433 0 -1434 1 -1435 1 -1436 1 -1437 1 -1438 1 -1439 1 -1440 1 -1441 1 -1442 1 -1443 1 -1444 1 -1445 1 -1446 1 -1447 1 -1448 1 -1449 1 -1450 1 -1451 1 -1452 1 -1453 1 -1454 1 -1455 0 -1456 1 -1457 1 -1458 1 -1459 1 -1460 1 -1461 1 -1462 1 -1463 1 -1464 1 -1465 1 -1466 1 -1467 1 -1468 1 -1469 1 -1470 1 -1471 1 -1472 1 -1473 1 -1474 1 -1475 1 -1476 1 -1477 1 -1478 1 -1479 1 -1480 1 -1481 1 -1482 1 -1483 1 -1484 1 -1485 1 -1486 1 -1487 0 -1488 1 -1489 1 -1490 1 -1491 1 -1492 1 -1493 1 -1494 1 -1495 1 -1496 1 -1497 1 -1498 1 -1499 1 -1500 1 -1501 1 -1502 1 -1503 1 -1504 1 -1505 1 -1506 1 -1507 1 -1508 1 -1509 1 -1510 1 -1511 1 -1512 1 -1513 1 -1514 0 -1515 1 -1516 1 -1517 1 -1518 1 -1519 1 -1520 1 -1521 0 -1522 1 -1523 1 -1524 1 -1525 1 -1526 1 -1527 1 -1528 1 -1529 1 -1530 1 -1531 1 -1532 1 -1533 1 -1534 1 -1535 1 -1536 1 -1537 0 -1538 1 -1539 1 -1540 1 -1541 0 -1542 1 -1543 1 -1544 1 -1545 1 -1546 0 -1547 1 -1548 1 -1549 1 -1550 1 -1551 1 -1552 1 -1553 1 -1554 1 -1555 1 -1556 1 -1557 1 -1558 1 -1559 1 -1560 1 -1561 1 -1562 1 -1563 1 -1564 1 -1565 1 -1566 1 -1567 1 -1568 1 -1569 1 -1570 1 -1571 1 -1572 1 -1573 1 -1574 1 -1575 1 -1576 1 -1577 1 -1578 0 -1579 1 -1580 1 -1581 1 -1582 1 -1583 1 -1584 1 -1585 1 -1586 1 -1587 1 -1588 1 -1589 1 -1590 1 -1591 1 -1592 1 -1593 1 -1594 1 -1595 1 -1596 1 -1597 1 -1598 1 -1599 1 -1600 1 -1601 1 -1602 1 -1603 0 -1604 1 -1605 1 -1606 1 -1607 0 -1608 1 -1609 1 -1610 0 -1611 1 -1612 1 -1613 1 -1614 1 -1615 1 -1616 1 -1617 1 -1618 1 -1619 1 -1620 0 -1621 1 -1622 1 -1623 1 -1624 1 -1625 1 -1626 1 -1627 1 -1628 1 -1629 1 -1630 1 -1631 0 -1632 1 -1633 1 -1634 1 -1635 1 -1636 1 -1637 1 -1638 1 -1639 1 -1640 1 -1641 1 -1642 1 -1643 1 -1644 1 -1645 1 -1646 1 -1647 1 -1648 1 -1649 1 -1650 1 -1651 1 -1652 1 -1653 1 -1654 1 -1655 1 -1656 1 -1657 1 -1658 1 -1659 1 -1660 0 -1661 1 -1662 1 -1663 1 -1664 1 -1665 1 -1666 1 -1667 1 -1668 1 -1669 1 -1670 1 -1671 1 -1672 1 -1673 1 -1674 1 -1675 1 -1676 1 -1677 1 -1678 1 -1679 1 -1680 1 -1681 1 -1682 1 -1683 1 -1684 1 -1685 1 -1686 1 -1687 1 -1688 1 -1689 1 -1690 1 -1691 1 -1692 1 -1693 1 -1694 1 -1695 1 -1696 1 -1697 1 -1698 1 -1699 1 -1700 1 -1701 1 -1702 1 -1703 1 -1704 1 -1705 1 -1706 1 -1707 1 -1708 1 -1709 1 -1710 1 -1711 1 -1712 1 -1713 1 -1714 1 -1715 1 -1716 1 -1717 1 -1718 1 -1719 1 -1720 1 -1721 0 -1722 1 -1723 1 -1724 1