crossmodal + cider

stanford-crfm · teetone · May 3, 2024 · Feb 29, 2024 · Feb 29, 2024 · Feb 29, 2024
commit 122a417b63dd0c716610a12d3f9b9c2da067d85f
diff --git a/setup.cfg b/setup.cfg
@@ -47,6 +47,7 @@ install_requires=
     # Basic metrics
     nltk~=3.7
     pyext~=0.7
+    pycocoevalcap~=1.2
     rouge-score~=0.1.2
     scipy~=1.10
     uncertainty-calibration~=0.1.4

diff --git a/src/helm/benchmark/metrics/common_metric_specs.py b/src/helm/benchmark/metrics/common_metric_specs.py
@@ -164,4 +164,6 @@ def get_disinformation_metric_specs(args: Optional[Dict] = None) -> List[MetricS
 
 
 def get_open_ended_generation_metric_specs() -> List[MetricSpec]:
-    return get_basic_metric_specs(["exact_match", "quasi_exact_match", "f1_score", "rouge_l", "bleu_1", "bleu_4"])
+    return get_basic_metric_specs(
+        ["exact_match", "quasi_exact_match", "f1_score", "rouge_l", "bleu_1", "bleu_4", "cider"]
+    )
diff --git a/src/helm/benchmark/metrics/evaluate_reference_metrics.py b/src/helm/benchmark/metrics/evaluate_reference_metrics.py
@@ -20,6 +20,7 @@
 import string
 from . import code_metrics_helper
 import nltk
+from pycocoevalcap.cider.cider import Cider
 
 try:
     nltk.data.find("tokenizers/punkt")
@@ -188,6 +189,14 @@ def bleu_4(gold: str, pred: str) -> float:
     return sentence_bleu([word_tokenize(gold)], word_tokenize(pred), weights=(0, 0, 0, 1))
 
 
+def cider(gold: str, pred: str) -> float:
+    cider_evaluator = Cider()
+    candidate = {"caption": [pred]}
+    reference = {"caption": [gold]}
+    average_score, _ = cider_evaluator.compute_score(reference, candidate)
+    return average_score
+
+
 def extract_set_from_text(
     set_str: str,
     set_start_str: str = " is ",
@@ -325,6 +334,7 @@ def compute_metrics_helper(
         "math_equiv_chain_of_thought": is_equiv_chain_of_thought,
         "code_eval_acc": code_eval,
         "pass": code_eval,
+        "cider": cider,
         "f1_score": f1_score,
         "rouge_1": get_rouge_function("rouge1"),
         "rouge_2": get_rouge_function("rouge2"),

diff --git a/src/helm/benchmark/presentation/run_specs_debug.conf b/src/helm/benchmark/presentation/run_specs_debug.conf
@@ -1,8 +1,7 @@
 entries: [
-    # {description: "vqa:model=vlm,data_augmentation=chinese", priority: 1, groups: ["vqa_chinese"]}
-    # {description: "vqa:model=vlm,data_augmentation=hindi", priority: 1, groups: ["vqa_hindi"]}
-    # {description: "vqa:model=vlm,data_augmentation=spanish", priority: 1, groups: ["vqa_spanish"]}
 
-    {description: "vqa:model=vlm,data_augmentation=dialect_deterministic", priority: 1, groups: ["vqa_dialect"]}
-    {description: "vqa:model=vlm,data_augmentation=robustness", priority: 1, groups: ["vqa_robustness"]}
+    {description: "crossmodal_3600:model=vlm,language=english", priority: 1}
+    {description: "crossmodal_3600:model=vlm,language=spanish", priority: 1}
+    {description: "crossmodal_3600:model=vlm,language=chinese", priority: 1}
+    {description: "crossmodal_3600:model=vlm,language=hindi", priority: 1}
 ]
diff --git a/src/helm/benchmark/presentation/run_specs_vhelm.conf b/src/helm/benchmark/presentation/run_specs_vhelm.conf
@@ -6,6 +6,8 @@ entries: [
     ####################################################################################################################
     # Accuracy: Is the output semantically correct, given the text and image inputs?
     ####################################################################################################################
+
+    # Questions about natural images
     {description: "vqa:model=vlm", priority: 1, groups: ["vqa_base"]}
     {description: "viz_wiz:model=vlm", priority: 1}
 
@@ -72,11 +74,15 @@ entries: [
     #              Does the model generate creative content (e.g., poetry, art)?
     ####################################################################################################################
 
+    # TODO: story generation, poetry generation for given images
 
     ####################################################################################################################
     # Bias: Are the generations biased in demographic representation (e.g., gender, skin tone)?
     ####################################################################################################################
 
+    # TODO: implement https://github.com/gzcch/Bingo
+
+    # Crossmodal-3600 dataset also measures geographic bias
 
     ####################################################################################################################
     # Fairness: Does the model exhibit performance disparities across social groups (e.g., gender, dialect)?
@@ -99,13 +105,16 @@ entries: [
 
     {description: "vqa:model=vlm,data_augmentation=robustness", priority: 1, groups: ["vqa_robustness"]}
 
+    # Robustness https://arxiv.org/pdf/2311.16101.pdf
+
     ####################################################################################################################
     # Multilinguality: Does the model support non-English languages?
     ####################################################################################################################
 
-    {description: "vqa:model=vlm,data_augmentation=chinese", priority: 1, groups: ["vqa_chinese"]}
-    {description: "vqa:model=vlm,data_augmentation=hindi", priority: 1, groups: ["vqa_hindi"]}
-    {description: "vqa:model=vlm,data_augmentation=spanish", priority: 1, groups: ["vqa_spanish"]}
+    {description: "crossmodal_3600:model=vlm,language=english", priority: 1}
+    {description: "crossmodal_3600:model=vlm,language=spanish", priority: 1}
+    {description: "crossmodal_3600:model=vlm,language=chinese", priority: 1}
+    {description: "crossmodal_3600:model=vlm,language=hindi", priority: 1}
 
     ####################################################################################################################
     # Efficiency: How fast is the inference for the model?

diff --git a/src/helm/benchmark/run_specs/vlm_run_specs.py b/src/helm/benchmark/run_specs/vlm_run_specs.py
@@ -146,6 +146,25 @@ def get_chart2csv_spec() -> RunSpec:
     )
 
 
+@run_spec_function("crossmodal_3600")
+def get_crossmodal_3600_spec(language: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.crossmodal_3600_scenario.Crossmodal3600Scenario",
+        args={"language": language},
+    )
+    adapter_spec: AdapterSpec = get_short_answer_generation_adapter_spec()
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + get_open_ended_generation_metric_specs()
+
+    run_spec_name: str = "crossmodal_3600"
+    return RunSpec(
+        name=f"{run_spec_name}:language={language}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
 @run_spec_function("hateful_memes")
 def get_hateful_memes_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(

diff --git a/src/helm/benchmark/scenarios/vision_language/crossmodal_3600_scenario.py b/src/helm/benchmark/scenarios/vision_language/crossmodal_3600_scenario.py
@@ -0,0 +1,133 @@
+import json
+import os
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+class Crossmodal3600Scenario(Scenario):
+    """
+    Crossmodal-3600 dataset (XM3600 in short), a geographically-diverse set of 3600 images annotated
+    with human-generated reference captions in 36 languages.
+
+    @inproceedings{ThapliyalCrossmodal2022,
+      author        = {Ashish Thapliyal and Jordi Pont-Tuset and Xi Chen and Radu Soricut},
+      title         = {{Crossmodal-3600: A Massively Multilingual Multimodal Evaluation Dataset}},
+      booktitle     = {EMNLP},
+      year          = {2022}
+    }
+
+    Paper: https://arxiv.org/abs/2205.12522
+    Website: https://google.github.io/crossmodal-3600/
+    """
+
+    LANGUAGE_TO_ID: Dict[str, str] = {
+        "arabic": "ar",
+        "bengali": "bn",
+        "chinese": "zh",
+        "croatian": "hr",
+        "cusco_quechua": "quz",
+        "czech": "cs",
+        "danish": "da",
+        "dutch": "nl",
+        "english": "en",
+        "persian": "fa",
+        "finnish": "fi",
+        "french": "fr",
+        "german": "de",
+        "greek": "el",
+        "hebrew": "he",
+        "hindi": "hi",
+        "hungarian": "hu",
+        "indonesian": "id",
+        "italian": "it",
+        "japanese": "ja",
+        "korean": "ko",
+        "maori": "mi",
+        "norwegian": "no",
+        "polish": "pl",
+        "portuguese": "pt",
+        "romanian": "ro",
+        "russian": "ru",
+        "spanish": "es",
+        "swahili": "sw",
+        "swedish": "sv",
+        "telugu": "te",
+        "thai": "th",
+        "turkish": "tr",
+        "ukrainian": "uk",
+        "vietnamese": "vi",
+    }
+
+    IMAGES_URL: str = "https://open-images-dataset.s3.amazonaws.com/crossmodal-3600/images.tgz"
+    CAPTIONS_URL: str = "https://google.github.io/crossmodal-3600/web-data/captions.zip"
+
+    name = "crossmodal_3600"
+    description = (
+        "Crossmodal-3600 dataset (XM3600 in short), a geographically-diverse set of 3600 images annotated "
+        "with human-generated reference captions in 36 languages. ([paper](https://arxiv.org/abs/2205.12522))."
+    )
+    tags = ["vision-language", "multilinguality"]
+
+    def __init__(self, language: str):
+        super().__init__()
+        self._language_id: str = self.LANGUAGE_TO_ID[language]
+        self._instruction: str = f"Generate a short caption for the following image in {language}."
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_file_downloaded(
+            source_url=self.IMAGES_URL,
+            target_path=images_path,
+            unpack=True,
+            unpack_type="untar",
+        )
+
+        captions_path: str = os.path.join(output_path, "captions.jsonl")
+        ensure_file_downloaded(
+            source_url=self.CAPTIONS_URL,
+            target_path=captions_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+        instances: List[Instance] = []
+        with open(captions_path, "r") as captions_file:
+            for line in captions_file:
+                example: Dict = json.loads(line)
+
+                language_id: str = example["image/locale"]
+                if language_id != self._language_id:
+                    continue
+
+                key: str = example["image/key"]
+                image_path: str = os.path.join(images_path, f"{key}.jpg")
+                assert os.path.exists(image_path), f"Image {image_path} does not exist"
+
+                assert language_id in example, f"Language {language_id} not found in example"
+                all_captions: Dict = example[language_id]
+                captions: List[str] = all_captions["caption"]
+
+                content: List[MediaObject] = [
+                    MediaObject(text=self._instruction, content_type="text/plain"),
+                    MediaObject(location=image_path, content_type="image/jpeg"),
+                ]
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[Reference(Output(text=caption), tags=[CORRECT_TAG]) for caption in captions],
+                        split=TEST_SPLIT,
+                    )
+                )
+
+        return instances
diff --git a/src/helm/benchmark/static/schema_vlm.yaml b/src/helm/benchmark/static/schema_vlm.yaml
@@ -196,6 +196,10 @@ metrics:
     display_name: F1
     description: Average F1 score in terms of word overlap between the model output and correct reference.
     lower_is_better: false
+  - name: cider
+    display_name: CIDEr
+    description: Evaluates the quality of generated caption by measuring the weighted similarity of n-grams between the captions and a set of human-written reference captions, emphasizing informativeness and consensus.
+    lower_is_better: false
 
 ############################################################
 perturbations:
@@ -294,6 +298,23 @@ run_groups:
       - mmmu
       - image2structure
 
+  - name: crossmodal_3600
+    display_name: Crossmodal 3600
+    description: Crossmodal-3600 dataset (XM3600 in short), a geographically-diverse set of 3600 images annotated with human-generated reference captions in 36 languages. ([paper](https://arxiv.org/abs/2205.12522))
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: f1_score
+      main_split: valid
+    taxonomy:
+      task: multilingual captioning
+      what: Real-world images
+      who: Human experts
+      when: "2022"
+      language: 36 languages
+
   - name: heim_human_eval
     display_name: HEIM Human Eval Scenario
     description: Seeing if we can use VLMs to evaluate AI-generated images from HEIM