From 32af2aca61f6b44dbaa9debefce74b3af5f45dd7 Mon Sep 17 00:00:00 2001 From: Mirco Ravanelli Date: Sun, 13 Aug 2023 22:18:18 +0000 Subject: [PATCH 1/5] update requirement (to avoid deprecation error) --- recipes/MOABB/extra_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/MOABB/extra_requirements.txt b/recipes/MOABB/extra_requirements.txt index 966b7176a4..7c1e464422 100644 --- a/recipes/MOABB/extra_requirements.txt +++ b/recipes/MOABB/extra_requirements.txt @@ -1,5 +1,5 @@ mne moabb orion[profet] -sklearn +scikit-learn torchinfo From b039df1b68ad1fe5ed33ec9fda0ad9d4755007d2 Mon Sep 17 00:00:00 2001 From: Mirco Ravanelli Date: Sun, 13 Aug 2023 22:33:19 +0000 Subject: [PATCH 2/5] small fixes --- recipes/MOABB/hparams/orion/hparams_tpe.yaml | 2 +- recipes/MOABB/run_experiments.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes/MOABB/hparams/orion/hparams_tpe.yaml b/recipes/MOABB/hparams/orion/hparams_tpe.yaml index fb6a7c9b04..a10273030d 100755 --- a/recipes/MOABB/hparams/orion/hparams_tpe.yaml +++ b/recipes/MOABB/hparams/orion/hparams_tpe.yaml @@ -1,5 +1,5 @@ experiment: - algorithms: + algorithm: tpe: seed: 1986 n_initial_points: 20 diff --git a/recipes/MOABB/run_experiments.sh b/recipes/MOABB/run_experiments.sh index 1de6416dfe..141149afce 100755 --- a/recipes/MOABB/run_experiments.sh +++ b/recipes/MOABB/run_experiments.sh @@ -173,6 +173,7 @@ fi if [ "$rnd_dir" = True ]; then rnd_dirname=$(tr -dc 'a-zA-Z' < /dev/urandom | head -c 6) output_folder="$output_folder/$rnd_dirname" + mkdir -p $output_folder fi # Print command line arguments and save to file From 07e7c7375299805835e8e42c6356f3757039f4ac Mon Sep 17 00:00:00 2001 From: Mirco Ravanelli Date: Sun, 13 Aug 2023 23:40:49 +0000 Subject: [PATCH 3/5] small fixes --- recipes/MOABB/extra_requirements.txt | 1 + recipes/MOABB/run_experiments.sh | 4 +++- recipes/MOABB/run_hparam_optimization.sh | 5 ++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/recipes/MOABB/extra_requirements.txt b/recipes/MOABB/extra_requirements.txt index 7c1e464422..b2d0835fa3 100644 --- a/recipes/MOABB/extra_requirements.txt +++ b/recipes/MOABB/extra_requirements.txt @@ -1,3 +1,4 @@ +braindecode mne moabb orion[profet] diff --git a/recipes/MOABB/run_experiments.sh b/recipes/MOABB/run_experiments.sh index 141149afce..37af36f9af 100755 --- a/recipes/MOABB/run_experiments.sh +++ b/recipes/MOABB/run_experiments.sh @@ -173,9 +173,11 @@ fi if [ "$rnd_dir" = True ]; then rnd_dirname=$(tr -dc 'a-zA-Z' < /dev/urandom | head -c 6) output_folder="$output_folder/$rnd_dirname" - mkdir -p $output_folder fi +# Make sure the output_folder is created +mkdir -p $output_folder + # Print command line arguments and save to file { echo "hparams: $hparams" diff --git a/recipes/MOABB/run_hparam_optimization.sh b/recipes/MOABB/run_hparam_optimization.sh index 432a5d0348..9fdb0afae1 100755 --- a/recipes/MOABB/run_hparam_optimization.sh +++ b/recipes/MOABB/run_hparam_optimization.sh @@ -46,7 +46,6 @@ # - Davide Borra (2023) ########################################################### - # Initialize variables exp_name="" output_folder="" @@ -65,8 +64,8 @@ mne_dir="" orion_db_address="" orion_db_type="PickledDB" exp_max_trials=50 -store_all=False -compress_exp=False +store_all=True +compress_exp=True # Function to print argument descriptions and exit print_argument_descriptions() { From dac684210aacb38b3746316833ad9fbc741748db Mon Sep 17 00:00:00 2001 From: Mirco Ravanelli Date: Sun, 13 Aug 2023 19:54:34 -0400 Subject: [PATCH 4/5] Update README.md --- recipes/MOABB/README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/recipes/MOABB/README.md b/recipes/MOABB/README.md index b40ae6eb87..d5b632d997 100644 --- a/recipes/MOABB/README.md +++ b/recipes/MOABB/README.md @@ -130,7 +130,6 @@ For training the model using the `leave_one_subject_out` training approach, just ### Hyperparameter Tuning - Efficient hyperparameter tuning is paramount when introducing novel models or experimenting with diverse datasets. Our benchmark establishes a standardized protocol for hyperparameter tuning, utilizing [Orion](https://orion.readthedocs.io/en/stable/) to ensure fair model comparisons. #### **Overview** @@ -192,6 +191,19 @@ As evident from the example, you need to configure the hyperparameter file, spec When it comes to training the model utilizing the leave_one_subject_out approach, simply employ the `--train_mode leave-one-subject-out` flag. By default trainings are performed on gpu. However, in case you do not have any gpu available on your machine, you can train models on cpu by specifying the `--device cpu` flag. + +**Note:** +- To monitor the status of the hyperparameter optimization, simply enter the following command: `orion status --all`. Ensure that you have added the necessary variables required by orion to your bash environment. You can achieve this by executing the following code within your terminal: + +```bash +export ORION_DB_ADDRESS=results/MotorImagery/BNCI2014001/EEGNet/hopt/EEGNet_BNCI2014001_hopt.pkl +export ORION_DB_TYPE=pickleddb +``` + +Please note that the value of the `ORION_DB_ADDRESS` variable will vary depending on the experiment. Adjust it accordingly. + +- If needed, you can interrupt the code at any point, and it will resume from the last successfully completed experiment. + #### **Output Structure** Results are organized within the specified output folder (`--output_folder`): @@ -213,8 +225,6 @@ For further details on arguments and customization options, consult `./run_hpara - If you intend to perform multiple repetitions of the same hparam optimization, it is necessary to modify the `--exp_name`. -- Feel free to interrupt and resume the hyperparameter optimization script at any point without encountering any complications. Orion is resumable and will restart from the last experiment completed. - - This script is designed for a Linux-based system. In this context, we provide a bash script instead of a Python script due to its natural ability of orchestrating diverse training loops across various subjects and sessions. From 75f4c6659d2fcd498679002846c74cbb146a4324 Mon Sep 17 00:00:00 2001 From: Mirco Ravanelli Date: Sun, 13 Aug 2023 20:02:18 -0400 Subject: [PATCH 5/5] Update README.md --- recipes/MOABB/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/MOABB/README.md b/recipes/MOABB/README.md index d5b632d997..e5ff5d9f2b 100644 --- a/recipes/MOABB/README.md +++ b/recipes/MOABB/README.md @@ -192,7 +192,7 @@ When it comes to training the model utilizing the leave_one_subject_out approach By default trainings are performed on gpu. However, in case you do not have any gpu available on your machine, you can train models on cpu by specifying the `--device cpu` flag. -**Note:** +**Note:** - To monitor the status of the hyperparameter optimization, simply enter the following command: `orion status --all`. Ensure that you have added the necessary variables required by orion to your bash environment. You can achieve this by executing the following code within your terminal: ```bash