hybrid_engine: check tuple size when fusing lora params (microsoft#3311)

Co-authored-by: Olatunji Ruwase <olruwase@microsoft.com> Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com>
davidsolomon21cn · Apr 24, 2023 · c9196b6 · c9196b6
1 parent 39b429d
commit c9196b6
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/deepspeed/runtime/hybrid_engine.py b/deepspeed/runtime/hybrid_engine.py
@@ -120,7 +120,7 @@ def populate_all_inference_policies(self):
     def _fuse_lora(self, params, lora_params):
         maybe_has_lora_params = [p for p in params if len(p.shape) > 1]
         for lora_param, weight in zip(lora_params, maybe_has_lora_params):
-            if len(lora_params) > 0:
+            if len(lora_param) == 3:
                 lora_right_weight, \
                 lora_left_weight, \
                 lora_scaling = lora_param
@@ -133,7 +133,7 @@ def fuse_lora_weight(self):
     def _unfuse_lora(self, params, lora_params):
         maybe_has_lora_params = [p for p in params if len(p.shape) > 1]
         for lora_param, weight in zip(lora_params, maybe_has_lora_params):
-            if len(lora_params) > 0:
+            if len(lora_param) == 3:
                 lora_right_weight, \
                 lora_left_weight, \
                 lora_scaling = lora_param