Add DeBERTa head models (huggingface#9691)

* Add DebertaForMaskedLM, DebertaForTokenClassification, DebertaForQuestionAnswering * Add docs and fix quality * Fix Deberta not having pooler
sorami · Jan 20, 2021 · d1370d2 · d1370d2
1 parent a7b62fe
commit d1370d2
Show file tree

Hide file tree

Showing 7 changed files with 686 additions and 253 deletions.
diff --git a/docs/source/model_doc/deberta.rst b/docs/source/model_doc/deberta.rst
@@ -70,8 +70,29 @@ DebertaPreTrainedModel
     :members:
 
 
+DebertaForMaskedLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DebertaForMaskedLM
+    :members:
+
+
 DebertaForSequenceClassification
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.DebertaForSequenceClassification
     :members:
+
+
+DebertaForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DebertaForTokenClassification
+    :members:
+
+
+DebertaForQuestionAnswering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DebertaForQuestionAnswering
+    :members:
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
@@ -477,7 +477,10 @@
             "DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST",
             "DebertaForSequenceClassification",
             "DebertaModel",
+            "DebertaForMaskedLM",
             "DebertaPreTrainedModel",
+            "DebertaForTokenClassification",
+            "DebertaForQuestionAnswering",
         ]
     )
     _import_structure["models.distilbert"].extend(
@@ -1527,7 +1530,10 @@
         )
         from .models.deberta import (
             DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
+            DebertaForMaskedLM,
+            DebertaForQuestionAnswering,
             DebertaForSequenceClassification,
+            DebertaForTokenClassification,
             DebertaModel,
             DebertaPreTrainedModel,
         )

diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py
@@ -62,7 +62,13 @@
     CamembertModel,
 )
 from ..ctrl.modeling_ctrl import CTRLForSequenceClassification, CTRLLMHeadModel, CTRLModel
-from ..deberta.modeling_deberta import DebertaForSequenceClassification, DebertaModel
+from ..deberta.modeling_deberta import (
+    DebertaForMaskedLM,
+    DebertaForQuestionAnswering,
+    DebertaForSequenceClassification,
+    DebertaForTokenClassification,
+    DebertaModel,
+)
 from ..distilbert.modeling_distilbert import (
     DistilBertForMaskedLM,
     DistilBertForMultipleChoice,
@@ -378,6 +384,7 @@
         (FunnelConfig, FunnelForMaskedLM),
         (MPNetConfig, MPNetForMaskedLM),
         (TapasConfig, TapasForMaskedLM),
+        (DebertaConfig, DebertaForMaskedLM),
     ]
 )
 
@@ -426,6 +433,7 @@
         (FunnelConfig, FunnelForMaskedLM),
         (MPNetConfig, MPNetForMaskedLM),
         (TapasConfig, TapasForMaskedLM),
+        (DebertaConfig, DebertaForMaskedLM),
     ]
 )
 
@@ -503,6 +511,7 @@
         (FunnelConfig, FunnelForQuestionAnswering),
         (LxmertConfig, LxmertForQuestionAnswering),
         (MPNetConfig, MPNetForQuestionAnswering),
+        (DebertaConfig, DebertaForQuestionAnswering),
     ]
 )
 
@@ -533,6 +542,7 @@
         (FlaubertConfig, FlaubertForTokenClassification),
         (FunnelConfig, FunnelForTokenClassification),
         (MPNetConfig, MPNetForTokenClassification),
+        (DebertaConfig, DebertaForTokenClassification),
     ]
 )
 

diff --git a/src/transformers/models/deberta/__init__.py b/src/transformers/models/deberta/__init__.py
@@ -31,7 +31,10 @@
         "DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST",
         "DebertaForSequenceClassification",
         "DebertaModel",
+        "DebertaForMaskedLM",
         "DebertaPreTrainedModel",
+        "DebertaForTokenClassification",
+        "DebertaForQuestionAnswering",
     ]
 
 
@@ -42,7 +45,10 @@
     if is_torch_available():
         from .modeling_deberta import (
             DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
+            DebertaForMaskedLM,
+            DebertaForQuestionAnswering,
             DebertaForSequenceClassification,
+            DebertaForTokenClassification,
             DebertaModel,
             DebertaPreTrainedModel,
         )