advqa_eval.py

import json
from tqdm.notebook import tqdm
import json
from pprint import PrettyPrinter
from vqa_eval_tools import VQA, VQAEval
from argparse import ArgumentParser
from pathlib import Path


pp = PrettyPrinter()

# The annotations are missing an "question_type" key, so we create a new annotation file which does
# have the key. We just copy the "answer_type" key to "question_type", they are the same thing, I think.
original_annotation_file = (
    "/net/acadia10a/data/zkhan/advqa/v1_mscoco_val2017_advqa_annotations.json"
)
question_file = (
    "/net/acadia10a/data/zkhan/advqa/v1_OpenEnded_mscoco_val2017_advqa_questions.json"
)

# This one doesn't have to exist, we create it from the original annotation file.
modified_annotations_file = (
    "/net/acadia10a/data/zkhan/advqa/nb017_val2017_annotations_w_qtype.json"
)


if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument(
        "result_file", help="Path to a JSON result file generated by an evaluation."
    )
    args = parser.parse_args()

    results_file = args.result_file
    # results_file = '/net/acadia4a/data/zkhan/mithril/advqa-0-shot-evals/35_blip_vqa_baseline/result/vqa_result.json'

    # The annotations are missing an "question_type" key, so we create a new annotation file which does
    # have the key. We just copy the "answer_type" key to "question_type", they are the same thing, I think.
    with open(original_annotation_file, "r") as f:
        annotations = json.load(f)

    for record in annotations["annotations"]:
        record["question_type"] = record["answer_type"]

    with open(modified_annotations_file, "w") as f:
        json.dump(annotations, f)

    advqa_obj = VQA(
        annotation_file=modified_annotations_file, question_file=question_file
    )

    # We have to convert the question_id field to be an integer >.<
    with open(results_file, "r") as f:
        predicted = json.load(f)

    for element in predicted:
        element["question_id"] = int(element["question_id"])

    with open(results_file, "w") as f:
        json.dump(predicted, f)

    result_obj = advqa_obj.loadRes(
        resFile=results_file,
        quesFile="/net/acadia10a/data/zkhan/advqa/v1_OpenEnded_mscoco_val2017_advqa_questions.json",
    )

    advqa_eval = VQAEval(advqa_obj, result_obj, n=2)
    advqa_eval.evaluate()
    print(f"Completed evaluation of {results_file}")
    pp.pprint(advqa_eval.accuracy)
    with open(Path(results_file).parent / "advqa_eval.json", "w") as f:
        json.dump(advqa_eval.accuracy, f)