update ppo

Joyce94 · Aug 26, 2023 · 0d81294 · 0d81294
1 parent 16bc92e
commit 0d81294
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/script/utils/ppo_models.py b/script/utils/ppo_models.py
@@ -79,7 +79,7 @@ def _init_actor(self):
         )
 
         if self.model_args.actor_peft_path is not None:
-            model = PeftModel.from_pretrained(model, self.model_args.actor_peft_path)
+            model = PeftModel.from_pretrained(model, self.model_args.actor_peft_path, is_trainable=True)
         else:
             lora_config = LoraConfig(
                 task_type=TaskType.CAUSAL_LM,
@@ -109,7 +109,7 @@ def _init_critic(self):
         model = model.merge_and_unload()
 
         if self.model_args.critic_peft_path is not None:
-            model = PeftModel.from_pretrained(model, self.model_args.critic_peft_path, adapter_name="default")
+            model = PeftModel.from_pretrained(model, self.model_args.critic_peft_path, adapter_name="default", is_trainable=True)
 
         else:
             lora_config = LoraConfig(