updata support_CPM_generate (mindspore-lab#567)

qurikuduo · Jun 8, 2023 · 73f35ee · 73f35ee
1 parent 10303a3
commit 73f35ee
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/mindnlp/transforms/tokenizers/cpm_tokenizer.py b/mindnlp/transforms/tokenizers/cpm_tokenizer.py
@@ -108,3 +108,9 @@ def _convert_to_unicode(self, text_input):
                 text_input = np.char.decode(text_input, "utf-8")
             return str(text_input)
         raise ValueError(f"Unsupported string type: {type(text_input)}, {text_input.dtype}")
+
+    def _convert_token_to_id(self, token):
+        return self._tokenizer.token_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        return self._tokenizer.id_to_token(index)