Add char_quantize for AAPD (castorini#173)

chaojiang06 · Jan 29, 2019 · 84225d2 · 84225d2
1 parent aec0826
commit 84225d2
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/datasets/aapd.py b/datasets/aapd.py
@@ -9,6 +9,13 @@
 
 from datasets.reuters import clean_string, clean_string_fl, split_sents
 
+def char_quantize(string, max_length=1000):
+    identity = np.identity(len(AAPDCharQuantized.ALPHABET))
+    quantized_string = np.array([identity[AAPDCharQuantized.ALPHABET[char]] for char in list(string.lower()) if char in AAPDCharQuantized.ALPHABET], dtype=np.float32)
+    if len(quantized_string) > max_length:
+        return quantized_string[:max_length]
+    else:
+        return np.concatenate((quantized_string, np.zeros((max_length - len(quantized_string), len(AAPDCharQuantized.ALPHABET)), dtype=np.float32)))
 
 def process_labels(string):
     """