fix: verify jsonlines file in run_translation (#14660) (#14661)

* fix: verify jsonl in run_translation (#14660) * fix(run_translation.py): json/jsonl validation Both json and jsonl are to be accepted as valid jsonlines file extension * fix(run_translation.py): make black happy * Ran make style
huggingface · Dec 8, 2021 · 4ea19de · 4ea19de
1 parent cf36f4d
commit 4ea19de
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/examples/pytorch/translation/run_translation.py b/examples/pytorch/translation/run_translation.py
@@ -216,12 +216,16 @@ def __post_init__(self):
         elif self.source_lang is None or self.target_lang is None:
             raise ValueError("Need to specify the source language and the target language.")
 
+        # accepting both json and jsonl file extensions, as
+        # many jsonlines files actually have a .json extension
+        valid_extensions = ["json", "jsonl"]
+
         if self.train_file is not None:
             extension = self.train_file.split(".")[-1]
-            assert extension == "json", "`train_file` should be a json file."
+            assert extension in valid_extensions, "`train_file` should be a jsonlines file."
         if self.validation_file is not None:
             extension = self.validation_file.split(".")[-1]
-            assert extension == "json", "`validation_file` should be a json file."
+            assert extension in valid_extensions, "`validation_file` should be a jsonlines file."
         if self.val_max_target_length is None:
             self.val_max_target_length = self.max_target_length