okvqa_eval.py

import json
from tqdm.notebook import tqdm
import json
from pprint import PrettyPrinter
from vqa_eval_tools import VQA, VQAEval
from argparse import ArgumentParser
from pathlib import Path


pp = PrettyPrinter()

# The annotations are missing an "question_type" key, so we create a new annotation file which does
# have the key. We just copy the "answer_type" key to "question_type", they are the same thing, I think.
annotation_file = "/net/acadia10a/data/zkhan/ok-vqa/mscoco_val2014_annotations.json"
question_file = (
    "/net/acadia10a/data/zkhan/ok-vqa/OpenEnded_mscoco_val2014_questions.json"
)


if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument(
        "result_file", help="Path to a JSON result file generated by an evaluation."
    )
    args = parser.parse_args()

    results_file = args.result_file
    # results_file = '/net/acadia4a/data/zkhan/mithril/advqa-0-shot-evals/35_blip_vqa_baseline/result/vqa_result.json'

    advqa_obj = VQA(annotation_file=annotation_file, question_file=question_file)

    # We have to convert the question_id field to be an integer >.<
    with open(results_file, "r") as f:
        predicted = json.load(f)

    for element in predicted:
        element["question_id"] = int(element["question_id"])

    with open(results_file, "w") as f:
        json.dump(predicted, f)

    result_obj = advqa_obj.loadRes(resFile=results_file, quesFile=question_file)

    advqa_eval = VQAEval(advqa_obj, result_obj, n=2)
    advqa_eval.evaluate()
    print(f"Completed evaluation of {results_file}")
    pp.pprint(advqa_eval.accuracy)
    with open(Path(results_file).parent / "okvqa_eval.json", "w") as f:
        json.dump(advqa_eval.accuracy, f)