chore: update models_tokens.py with new model configurations

ScrapeGraphAI · May 13, 2024 · d9752b1 · d9752b1
1 parent fc56d6b
commit d9752b1
Showing 1 changed file with 15 additions and 2 deletions.
diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py
@@ -1,5 +1,5 @@
 """
-Models token 
+Models token
 """
 
 models_tokens = {
@@ -39,7 +39,20 @@
         "dolphin-mixtral": 32000,
         "mistral-openorca": 32000,
         "stablelm-zephyr": 8192,
-        "nomic-embed-text": 8192
+        "command-r-plus": 12800,
+        "command-r": 12800,
+        "mistral:7b-instruct": 32768,
+        "llama3:70b-instruct": 8192,
+        "mixtral:8x22b-instruct": 65536,
+        "wizardlm2:8x22b": 65536,
+        "dbrx": 32768,
+        "dbrx:instruct": 32768,
+        "nous-hermes2:34b": 4096,
+        # embedding models
+        "nomic-embed-text": 8192,
+        "snowflake-arctic-embed:335m": 8192,
+        "snowflake-arctic-embed:l": 8192,
+        "mxbai-embed-large": 512,
     },
     "groq": {
         "llama3-8b-8192": 8192,