[Model] Add new model: varco-vision-hf (#678)

* add model named varco-vision-hf * fix def generate_inner_image --------- Co-authored-by: kim-youngjune <youngjune@ncsoft.com> Co-authored-by: Haodong Duan <dhd@pku.edu.cn>
open-compass · Dec 24, 2024 · 070f645 · 070f645
1 parent e342c81
commit 070f645
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/vlmeval/config.py b/vlmeval/config.py
@@ -165,6 +165,7 @@
     'Aquila-VL-2B': partial(LLaVA_OneVision, model_path='BAAI/Aquila-VL-2B-llava-qwen'),
     'llava_video_qwen2_7b':partial(LLaVA_OneVision, model_path='lmms-lab/LLaVA-Video-7B-Qwen2'),
     'llava_video_qwen2_72b':partial(LLaVA_OneVision, model_path='lmms-lab/LLaVA-Video-72B-Qwen2'),
+    'varco-vision-hf':partial(LLaVA_OneVision_HF, model_path='NCSOFT/VARCO-VISION-14B-HF'),
 }
 
 internvl_series = {
@@ -399,4 +400,3 @@
 
 for grp in model_groups:
     supported_VLM.update(grp)
-
diff --git a/vlmeval/vlm/llava/llava.py b/vlmeval/vlm/llava/llava.py
@@ -796,6 +796,7 @@ def __init__(self, model_path="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", **kwa
         self.force_sample = self.video_kwargs.get("force_sample", False)
         self.nframe = kwargs.get("nframe", 8)
         self.fps = 1
+        self.model_path = model_path
 
     def generate_inner_image(self, message, dataset=None):
         content, images = "", []
@@ -823,6 +824,8 @@ def generate_inner_image(self, message, dataset=None):
         inputs = self.processor(images=images, text=prompt, return_tensors="pt").to(0, torch.float16)
 
         output = self.model.generate(**inputs, max_new_tokens=100)
+        if self.model_path == "NCSOFT/VARCO-VISION-14B-HF":
+            return self.processor.decode(output[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
         return self.processor.decode(output[0], skip_special_tokens=True)
 
     def generate_inner_video(self, message, dataset=None):