update my branch with garbo

allenai · jacob-morrison · Aug 16, 2024 · Aug 16, 2024 · Aug 17, 2024 · Aug 17, 2024
commit a0fc16f3f97dbf2ba3cfc93130e0f8a8477b5d1a
diff --git a/configs/beaker_configs/default_dpo.yaml b/configs/beaker_configs/default_dpo.yaml
@@ -8,7 +8,7 @@ tasks:
     command: [
       '/bin/sh', '-c'
     ]
-    arguments: ['PYTHONPATH="/stage:$PYTHONPATH" accelerate launch
+    arguments: ['pip install --upgrade transformers && PYTHONPATH="/stage:$PYTHONPATH" accelerate launch
       --mixed_precision bf16
       --num_machines 1
       --num_processes 4

diff --git a/configs/beaker_configs/default_finetune_multinode.yaml b/configs/beaker_configs/default_finetune_multinode.yaml
@@ -15,7 +15,7 @@ tasks:
       '/bin/sh', '-c'
     ]
     arguments: ['
-        unset CUDA_LAUNCH_BLOCKING && PYTHONPATH="/stage:$PYTHONPATH" pip install git+https://github.com/vwxyzjn/transformers.git@olmo1124_classification && accelerate launch
+        unset CUDA_LAUNCH_BLOCKING && PYTHONPATH="/stage:$PYTHONPATH" accelerate launch
         --mixed_precision bf16
         --num_machines 4
         --num_processes 32
@@ -66,9 +66,9 @@ tasks:
       - mountPath: /oe-adapt-default
         source:
           weka: oe-adapt-default
-      - mountPath: /model
-        source:
-          beaker: jacobm/llama-3.1-8b
+      # - mountPath: /model
+        # source:
+          # beaker: jacobm/llama-3.1-8b
     result:
       path: /output
     resources:

diff --git a/configs/beaker_configs/default_finetune_multinode_olmo.yaml b/configs/beaker_configs/default_finetune_multinode_olmo.yaml
@@ -0,0 +1,78 @@
+version: v2
+description: open-instruct-finetune-multinode
+budget: ai2/oe-adapt
+tasks:
+  - name: open-instruct-finetune-multinode
+    replicas: 4
+    leaderSelection: true
+    hostNetworking: true
+    propagateFailure: true
+    propagatePreemption: true
+    synchronizedStartTimeout: 60m
+    image:
+      beaker: nathanl/open_instruct_auto
+    command: [
+      '/bin/sh', '-c'
+    ]
+    arguments: ['
+        unset CUDA_LAUNCH_BLOCKING && pip install git+https://github.com/vwxyzjn/transformers.git@olmo1124_classification && PYTHONPATH="/stage:$PYTHONPATH" accelerate launch
+        --mixed_precision bf16
+        --num_machines 4
+        --num_processes 32
+        --machine_rank $BEAKER_REPLICA_RANK
+        --main_process_ip $BEAKER_LEADER_REPLICA_HOSTNAME
+        --main_process_port 29400
+        --use_deepspeed
+        --deepspeed_config_file configs/ds_configs/stage3_no_offloading_accelerate.conf
+        --deepspeed_multinode_launcher standard
+        open_instruct/finetune.py
+        --model_name_or_path meta-llama/Meta-Llama-3-8B
+        --tokenizer_name meta-llama/Meta-Llama-3-8B
+        --use_slow_tokenizer
+        --use_flash_attn
+        --max_seq_length 4096 
+        --preprocessing_num_workers 16
+        --per_device_train_batch_size 1
+        --gradient_accumulation_steps 4
+        --learning_rate 5e-6
+        --lr_scheduler_type linear
+        --warmup_ratio 0.03
+        --weight_decay 0.
+        --num_train_epochs 2
+        --output_dir /output/
+        --with_tracking
+        --report_to tensorboard
+        --logging_steps 1
+        --reduce_loss sum
+    ']
+    envVars:
+      - name: CUDA_DEVICE_ORDER
+        value: PCI_BUS_ID
+      - name: TRANSFORMERS_CACHE
+        value: ./cache/
+      - name: WANDB_API_KEY
+        secret: jacobm_WANDB_API_KEY
+      - name: WANDB_PROJECT
+        value: open-instruct
+      - name: WANDB_WATCH
+        value: false
+      - name: WANDB_LOG_MODEL
+        value: false
+      - name: WANDB_DISABLED
+        value: true
+      - name: HF_TOKEN
+        secret: jacobm_HF_TOKEN
+    datasets:
+      - mountPath: /oe-adapt-default
+        source:
+          weka: oe-adapt-default
+      # - mountPath: /model
+      #   source:
+      #     beaker: jacobm/llama-3.1-8b
+    result:
+      path: /output
+    resources:
+      gpuCount: 8
+    context:
+      priority: normal
+      preemptible: true
diff --git a/configs/beaker_configs/default_finetune_multinode_olmoe.yaml b/configs/beaker_configs/default_finetune_multinode_olmoe.yaml
@@ -0,0 +1,78 @@
+version: v2
+description: open-instruct-finetune-multinode
+budget: ai2/oe-adapt
+tasks:
+  - name: open-instruct-finetune-multinode
+    replicas: 4
+    leaderSelection: true
+    hostNetworking: true
+    propagateFailure: true
+    propagatePreemption: true
+    synchronizedStartTimeout: 60m
+    image:
+      beaker: nathanl/open_instruct_auto
+    command: [
+      '/bin/sh', '-c'
+    ]
+    arguments: ['
+        unset CUDA_LAUNCH_BLOCKING && pip install --upgrade transformers && PYTHONPATH="/stage:$PYTHONPATH" accelerate launch
+        --mixed_precision bf16
+        --num_machines 4
+        --num_processes 32
+        --machine_rank $BEAKER_REPLICA_RANK
+        --main_process_ip $BEAKER_LEADER_REPLICA_HOSTNAME
+        --main_process_port 29400
+        --use_deepspeed
+        --deepspeed_config_file configs/ds_configs/stage3_no_offloading_accelerate.conf
+        --deepspeed_multinode_launcher standard
+        open_instruct/finetune.py
+        --model_name_or_path meta-llama/Meta-Llama-3-8B
+        --tokenizer_name meta-llama/Meta-Llama-3-8B
+        --use_slow_tokenizer
+        --use_flash_attn
+        --max_seq_length 4096 
+        --preprocessing_num_workers 16
+        --per_device_train_batch_size 1
+        --gradient_accumulation_steps 4
+        --learning_rate 5e-6
+        --lr_scheduler_type linear
+        --warmup_ratio 0.03
+        --weight_decay 0.
+        --num_train_epochs 2
+        --output_dir /output/
+        --with_tracking
+        --report_to tensorboard
+        --logging_steps 1
+        --reduce_loss sum
+    ']
+    envVars:
+      - name: CUDA_DEVICE_ORDER
+        value: PCI_BUS_ID
+      - name: TRANSFORMERS_CACHE
+        value: ./cache/
+      - name: WANDB_API_KEY
+        secret: jacobm_WANDB_API_KEY
+      - name: WANDB_PROJECT
+        value: open-instruct
+      - name: WANDB_WATCH
+        value: false
+      - name: WANDB_LOG_MODEL
+        value: false
+      - name: WANDB_DISABLED
+        value: true
+      - name: HF_TOKEN
+        secret: jacobm_HF_TOKEN
+    datasets:
+      - mountPath: /oe-adapt-default
+        source:
+          weka: oe-adapt-default
+      # - mountPath: /model
+      #   source:
+      #     beaker: jacobm/llama-3.1-8b
+    result:
+      path: /output
+    resources:
+      gpuCount: 8
+    context:
+      priority: normal
+      preemptible: true
diff --git a/configs/beaker_configs/default_finetune_olmo.yaml b/configs/beaker_configs/default_finetune_olmo.yaml
@@ -0,0 +1,65 @@
+version: v2
+description: open-instruct-finetune
+budget: ai2/oe-adapt
+tasks:
+  - name: open-instruct-finetune
+    image:
+      beaker: nathanl/open_instruct_auto
+    command: [
+      '/bin/sh', '-c'
+    ]
+    arguments: ['pip install git+https://github.com/vwxyzjn/transformers.git@olmo1124_classification && PYTHONPATH="/stage:$PYTHONPATH" accelerate launch
+      --mixed_precision bf16
+      --num_machines 1
+      --num_processes 4
+      --use_deepspeed
+      --deepspeed_config_file configs/ds_configs/stage3_no_offloading_accelerate.conf
+      open_instruct/finetune.py
+      --model_name_or_path /hf_llama_models
+      --use_flash_attn
+      --max_seq_length 2048
+      --preprocessing_num_workers 16
+      --per_device_train_batch_size 2
+      --gradient_accumulation_steps 16
+      --learning_rate 2e-5
+      --lr_scheduler_type linear
+      --warmup_ratio 0.03
+      --weight_decay 0.
+      --num_train_epochs 2
+      --output_dir /output/
+      --with_tracking
+      --report_to tensorboard
+      --logging_steps 1
+    ']
+    envVars:
+      - name: CUDA_DEVICE_ORDER
+        value: PCI_BUS_ID
+      - name: TRANSFORMERS_CACHE
+        value: ./cache/
+      - name: WANDB_API_KEY
+        secret: jacobm_WANDB_API_KEY
+      - name: WANDB_PROJECT
+        value: open-instruct
+      - name: WANDB_WATCH
+        value: false
+      - name: WANDB_LOG_MODEL
+        value: false
+      - name: WANDB_DISABLED
+        value: true
+      - name: HF_TOKEN
+        secret: jacobm_HF_TOKEN
+    datasets:
+      - mountPath: /oe-adapt-default
+        source:
+          weka: oe-adapt-default
+      - mountPath: /oe-training-default
+        source:
+          weka: oe-training-default
+    result:
+      path: /output
+    resources:
+      gpuCount: 4
+    context:
+      cluster: ai2/allennlp-cirrascale
+      priority: high
+      preemptible: false
diff --git a/configs/merge_configs/my-merge-config.yaml b/configs/merge_configs/my-merge-config.yaml
@@ -1,15 +1,26 @@
 merge_method: linear
 normalize: true
 models:
-  - name: llama-3.1-8b-resized
-    location: huggingface
-    path: ai2-adapt-dev/llama-3.1-8b-resized
-    weight: 0.5
-  - name: L3.1-8B-v3.9-nc-fixed-soup-best_2
+  # - name: llama-3.1-8b-resized
+  #   location: huggingface
+  #   path: ai2-adapt-dev/llama-3.1-8b-resized
+  #   weight: 0.5
+  # - name: L3.1-8B-v3.9-nc-fixed-soup-best_2
+  #   location: weka
+  #   path: /oe-adapt-default/jacobm/tulu-3-dev/checkpoints/base_models/L3.1-8B-v3.9-nc-fixed-best_2/
+  #   wekaBucket: "oe-adapt-default"
+  #   weight: 0.5
+
+  - name: gsm_math_if_valpy_best_overall_avg_8b_beta0.05-step_200
+    location: weka
+    path: /oe-adapt-default/hamishi/model_checkpoints/gsm_math_if_valpy_best_overall_avg_8b_beta0.05_checkpoints/step_200/
+    wekaBucket: "oe-adapt-default"
+    weight: 1.0
+  - name: gsm_math_if_valpy_best_and_if_avg_8b_beta0.05-step_200
     location: weka
-    path: /oe-adapt-default/jacobm/tulu-3-dev/checkpoints/base_models/L3.1-8B-v3.9-nc-fixed-best_2/
+    path: /oe-adapt-default/hamishi/model_checkpoints/gsm_math_if_valpy_best_and_if_avg_8b_beta0.05_checkpoints/step_200/
     wekaBucket: "oe-adapt-default"
-    weight: 0.5
+    weight: 1.0
   # - name: L3.1-8B-v3.9-nc-fixed-2
   #   location: weka
   #   path: /oe-adapt-default/jacobm/tulu-3-dev/checkpoints/base_models/L3.1-8B-v3.9-nc-fixed-2/

diff --git a/configs/train_configs/dpo/olmoe_dpo_test.yaml b/configs/train_configs/dpo/olmoe_dpo_test.yaml
@@ -0,0 +1,37 @@
+model_name_or_path: /model
+tokenizer_name: /model
+model_revision: main
+use_flash_attn: true
+gradient_checkpointing: true
+dataset_mixer:
+  # ai2-adapt-dev/sft_v3.9_used_off_policy: 1.0
+  # ai2-adapt-dev/sft_v3.9_used_on_policy_large_70b_ckpt: 1.0
+  # ai2-adapt-dev/DaringAnteater-prefs-RM-filter-uf-pipeline-regen-v3.9_large_70b_ckpt: 1.0
+  # ai2-adapt-dev/WildChat-prefs-280824-uf-pipeline-regen-v3.9_large_70b_ckpt: 1.0
+  # ai2-adapt-dev/Llama-3.1-if_taxonomy_tulu-uf-pipeline-regen-v3.9_large_70b_ckpt: 1.0
+  ai2-adapt-dev/wildchat_v3.9_unused_off_policy: 1.0
+
+  ai2-adapt-dev/sft_v3.9_used_p0_olmoe-1b-7b: 1.0
+  ai2-adapt-dev/sft_v3.9_used_p1_olmoe-1b-7b: 1.0
+  ai2-adapt-dev/daring_anteater_olmoe-1b-7b: 1.0
+  ai2-adapt-dev/wildchat-prefs-280824_olmoe-1b-7b: 1.0
+  ai2-adapt-dev/llama3.1-if_taxonomy_tulu_olmoe-1b-7b: 1.0
+use_slow_tokenizer: true
+max_seq_length: 2048
+preprocessing_num_workers: 16
+per_device_train_batch_size: 2
+gradient_accumulation_steps: 8 # designed for 8 GPUs, so batch size 128
+learning_rate: 5.0e-7
+lr_scheduler_type: linear
+warmup_ratio: 0.1
+weight_decay: 0.0
+num_train_epochs: 1
+output_dir: /output
+with_tracking: true
+report_to:
+  - wandb
+logging_steps: 1
+use_lora: false
+dpo_loss_type: dpo_norm
+dpo_beta: 5
+checkpointing_steps: 1000