Auto-Detect if Better Default LLM available for OpenAI (onyx-dot-app#…

…1106)
Nibezak · Feb 22, 2024 · 7748f4d · 7748f4d
1 parent 918bc38
commit 7748f4d
Show file tree

Hide file tree

Showing 14 changed files with 134 additions and 73 deletions.
diff --git a/backend/danswer/chat/chat_utils.py b/backend/danswer/chat/chat_utils.py
@@ -17,14 +17,14 @@
 from danswer.configs.chat_configs import STOP_STREAM_PAT
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import IGNORE_FOR_QA
-from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
 from danswer.configs.model_configs import GEN_AI_SINGLE_USER_MESSAGE_EXPECTED_MAX_TOKENS
 from danswer.db.chat import get_chat_messages_by_session
 from danswer.db.models import ChatMessage
 from danswer.db.models import Persona
 from danswer.db.models import Prompt
 from danswer.indexing.models import InferenceChunk
 from danswer.llm.utils import check_number_of_tokens
+from danswer.llm.utils import get_default_llm_version
 from danswer.llm.utils import get_max_input_tokens
 from danswer.prompts.chat_prompts import CHAT_USER_CONTEXT_FREE_PROMPT
 from danswer.prompts.chat_prompts import CHAT_USER_PROMPT
@@ -577,7 +577,7 @@ def compute_max_document_tokens(
     if we're trying to determine if the user should be able to select another document) then we just set an
     arbitrary "upper bound".
     """
-    llm_name = GEN_AI_MODEL_VERSION
+    llm_name = get_default_llm_version()[0]
     if persona.llm_model_version_override:
         llm_name = persona.llm_model_version_override
 
@@ -603,7 +603,7 @@ def compute_max_document_tokens(
 
 def compute_max_llm_input_tokens(persona: Persona) -> int:
     """Maximum tokens allows in the input to the LLM (of any type)."""
-    llm_name = GEN_AI_MODEL_VERSION
+    llm_name = get_default_llm_version()[0]
     if persona.llm_model_version_override:
         llm_name = persona.llm_model_version_override
 

diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
@@ -27,7 +27,6 @@
 from danswer.configs.constants import DISABLED_GEN_AI_MSG
 from danswer.configs.constants import MessageType
 from danswer.configs.model_configs import CHUNK_SIZE
-from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
 from danswer.db.chat import create_db_search_doc
 from danswer.db.chat import create_new_chat_message
 from danswer.db.chat import get_chat_message
@@ -48,6 +47,7 @@
 from danswer.llm.factory import get_default_llm
 from danswer.llm.interfaces import LLM
 from danswer.llm.utils import get_default_llm_tokenizer
+from danswer.llm.utils import get_default_llm_version
 from danswer.llm.utils import get_max_input_tokens
 from danswer.llm.utils import tokenizer_trim_content
 from danswer.llm.utils import translate_history_to_basemessages
@@ -445,7 +445,7 @@ def stream_chat_message_objects(
                 else default_num_chunks
             )
 
-            llm_name = GEN_AI_MODEL_VERSION
+            llm_name = get_default_llm_version()[0]
             if persona.llm_model_version_override:
                 llm_name = persona.llm_model_version_override
 

diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
@@ -48,6 +48,10 @@
 INDEX_SEPARATOR = "==="
 
 
+# Key-Value store constants
+GEN_AI_DETECTED_MODEL = "gen_ai_detected_model"
+
+
 # Messages
 DISABLED_GEN_AI_MSG = (
     "Your System Admin has disabled the Generative AI functionalities of Danswer.\n"

diff --git a/backend/danswer/configs/model_configs.py b/backend/danswer/configs/model_configs.py
@@ -83,14 +83,11 @@
 # Set GEN_AI_MODEL_PROVIDER to "gpt4all" to use gpt4all models running locally
 GEN_AI_MODEL_PROVIDER = os.environ.get("GEN_AI_MODEL_PROVIDER") or "openai"
 # If using Azure, it's the engine name, for example: Danswer
-GEN_AI_MODEL_VERSION = (
-    os.environ.get("GEN_AI_MODEL_VERSION") or "gpt-3.5-turbo-16k-0613"
-)
+GEN_AI_MODEL_VERSION = os.environ.get("GEN_AI_MODEL_VERSION")
+
 # For secondary flows like extracting filters or deciding if a chunk is useful, we don't need
 # as powerful of a model as say GPT-4 so we can use an alternative that is faster and cheaper
-FAST_GEN_AI_MODEL_VERSION = (
-    os.environ.get("FAST_GEN_AI_MODEL_VERSION") or GEN_AI_MODEL_VERSION
-)
+FAST_GEN_AI_MODEL_VERSION = os.environ.get("FAST_GEN_AI_MODEL_VERSION")
 
 # If the Generative AI model requires an API key for access, otherwise can leave blank
 GEN_AI_API_KEY = (

diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py
@@ -21,7 +21,6 @@
 from danswer.configs.danswerbot_configs import DANSWER_REACT_EMOJI
 from danswer.configs.danswerbot_configs import DISABLE_DANSWER_BOT_FILTER_DETECT
 from danswer.configs.danswerbot_configs import ENABLE_DANSWERBOT_REFLEXION
-from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
 from danswer.danswerbot.slack.blocks import build_documents_blocks
 from danswer.danswerbot.slack.blocks import build_follow_up_block
 from danswer.danswerbot.slack.blocks import build_qa_response_blocks
@@ -37,6 +36,7 @@
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.models import SlackBotConfig
 from danswer.llm.utils import check_number_of_tokens
+from danswer.llm.utils import get_default_llm_version
 from danswer.llm.utils import get_max_input_tokens
 from danswer.one_shot_answer.answer_question import get_search_answer
 from danswer.one_shot_answer.models import DirectQARequest
@@ -224,7 +224,7 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse:
         max_document_tokens: int | None = None
         max_history_tokens: int | None = None
         if len(new_message_request.messages) > 1:
-            llm_name = GEN_AI_MODEL_VERSION
+            llm_name = get_default_llm_version()[0]
             if persona and persona.llm_model_version_override:
                 llm_name = persona.llm_model_version_override
 

diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py
@@ -15,6 +15,7 @@
 from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
 from danswer.configs.model_configs import GEN_AI_TEMPERATURE
 from danswer.llm.interfaces import LLM
+from danswer.llm.utils import get_default_llm_version
 from danswer.llm.utils import message_generator_to_string_generator
 from danswer.llm.utils import should_be_verbose
 from danswer.utils.logger import setup_logger
@@ -92,7 +93,8 @@ def _get_model_str(
         return model_version
 
     # User specified something wrong, just use Danswer default
-    return GEN_AI_MODEL_VERSION
+    base, _ = get_default_llm_version()
+    return base
 
 
 class DefaultMultiLLM(LangChainChatLLM):
@@ -109,7 +111,7 @@ def __init__(
         api_key: str | None,
         timeout: int,
         model_provider: str = GEN_AI_MODEL_PROVIDER,
-        model_version: str = GEN_AI_MODEL_VERSION,
+        model_version: str | None = GEN_AI_MODEL_VERSION,
         api_base: str | None = GEN_AI_API_ENDPOINT,
         api_version: str | None = GEN_AI_API_VERSION,
         custom_llm_provider: str | None = GEN_AI_LLM_PROVIDER_TYPE,
@@ -121,6 +123,8 @@ def __init__(
         litellm.api_key = api_key or "dummy-key"
         litellm.api_version = api_version
 
+        model_version = model_version or get_default_llm_version()[0]
+
         self._llm = ChatLiteLLM(  # type: ignore
             model=model_version
             if custom_llm_provider

diff --git a/backend/danswer/llm/factory.py b/backend/danswer/llm/factory.py
@@ -1,13 +1,12 @@
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
 from danswer.configs.chat_configs import QA_TIMEOUT
-from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
 from danswer.configs.model_configs import GEN_AI_MODEL_PROVIDER
-from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
 from danswer.llm.chat_llm import DefaultMultiLLM
 from danswer.llm.custom_llm import CustomModelServer
 from danswer.llm.exceptions import GenAIDisabledException
 from danswer.llm.gpt_4_all import DanswerGPT4All
 from danswer.llm.interfaces import LLM
+from danswer.llm.utils import get_default_llm_version
 from danswer.llm.utils import get_gen_ai_api_key
 
 
@@ -26,9 +25,8 @@ def get_default_llm(
     if gen_ai_model_version_override:
         model_version = gen_ai_model_version_override
     else:
-        model_version = (
-            FAST_GEN_AI_MODEL_VERSION if use_fast_llm else GEN_AI_MODEL_VERSION
-        )
+        base, fast = get_default_llm_version()
+        model_version = fast if use_fast_llm else base
     if api_key is None:
         api_key = get_gen_ai_api_key()
 

diff --git a/backend/danswer/llm/gpt_4_all.py b/backend/danswer/llm/gpt_4_all.py
@@ -8,6 +8,7 @@
 from danswer.configs.model_configs import GEN_AI_TEMPERATURE
 from danswer.llm.interfaces import LLM
 from danswer.llm.utils import convert_lm_input_to_basic_string
+from danswer.llm.utils import get_default_llm_version
 from danswer.utils.logger import setup_logger
 
 
@@ -52,14 +53,14 @@ def requires_api_key(self) -> bool:
     def __init__(
         self,
         timeout: int,
-        model_version: str = GEN_AI_MODEL_VERSION,
+        model_version: str | None = GEN_AI_MODEL_VERSION,
         max_output_tokens: int = GEN_AI_MAX_OUTPUT_TOKENS,
         temperature: float = GEN_AI_TEMPERATURE,
     ):
         self.timeout = timeout
         self.max_output_tokens = max_output_tokens
         self.temperature = temperature
-        self.gpt4all_model = GPT4All(model_version)
+        self.gpt4all_model = GPT4All(model_version or get_default_llm_version()[0])
 
     def log_model_configs(self) -> None:
         logger.debug(

diff --git a/backend/danswer/llm/utils.py b/backend/danswer/llm/utils.py
@@ -1,6 +1,7 @@
 from collections.abc import Callable
 from collections.abc import Iterator
 from copy import copy
+from functools import lru_cache
 from typing import Any
 from typing import cast
 
@@ -19,8 +20,10 @@
 
 from danswer.configs.app_configs import LOG_LEVEL
 from danswer.configs.constants import GEN_AI_API_KEY_STORAGE_KEY
+from danswer.configs.constants import GEN_AI_DETECTED_MODEL
 from danswer.configs.constants import MessageType
 from danswer.configs.model_configs import DOC_EMBEDDING_CONTEXT_SIZE
+from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
 from danswer.configs.model_configs import GEN_AI_API_KEY
 from danswer.configs.model_configs import GEN_AI_MAX_OUTPUT_TOKENS
 from danswer.configs.model_configs import GEN_AI_MAX_TOKENS
@@ -39,6 +42,34 @@
 _LLM_TOKENIZER_ENCODE: Callable[[str], Any] | None = None
 
 
+@lru_cache()
+def get_default_llm_version() -> tuple[str, str]:
+    default_openai_model = "gpt-3.5-turbo-16k-0613"
+    if GEN_AI_MODEL_VERSION:
+        llm_version = GEN_AI_MODEL_VERSION
+    else:
+        if GEN_AI_MODEL_PROVIDER != "openai":
+            logger.warning("No LLM Model Version set")
+            # Either this value is unused or it will throw an error
+            llm_version = default_openai_model
+        else:
+            kv_store = get_dynamic_config_store()
+            try:
+                llm_version = cast(str, kv_store.load(GEN_AI_DETECTED_MODEL))
+            except ConfigNotFoundError:
+                llm_version = default_openai_model
+
+    if FAST_GEN_AI_MODEL_VERSION:
+        fast_llm_version = FAST_GEN_AI_MODEL_VERSION
+    else:
+        if GEN_AI_MODEL_PROVIDER == "openai":
+            fast_llm_version = default_openai_model
+        else:
+            fast_llm_version = llm_version
+
+    return llm_version, fast_llm_version
+
+
 def get_default_llm_tokenizer() -> Encoding:
     """Currently only supports the OpenAI default tokenizer: tiktoken"""
     global _LLM_TOKENIZER
@@ -215,8 +246,7 @@ def get_llm_max_tokens(
         # This is an override, so always return this
         return GEN_AI_MAX_TOKENS
 
-    if not model_name:
-        return 4096
+    model_name = model_name or get_default_llm_version()[0]
 
     try:
         if model_provider == "openai":
@@ -231,6 +261,7 @@ def get_max_input_tokens(
     model_provider: str = GEN_AI_MODEL_PROVIDER,
     output_tokens: int = GEN_AI_MAX_OUTPUT_TOKENS,
 ) -> int:
+    model_name = model_name or get_default_llm_version()[0]
     input_toks = (
         get_llm_max_tokens(model_name=model_name, model_provider=model_provider)
         - output_tokens

diff --git a/backend/danswer/main.py b/backend/danswer/main.py
@@ -35,10 +35,8 @@
 from danswer.configs.chat_configs import MULTILINGUAL_QUERY_EXPANSION
 from danswer.configs.constants import AuthType
 from danswer.configs.model_configs import ENABLE_RERANKING_REAL_TIME_FLOW
-from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
 from danswer.configs.model_configs import GEN_AI_API_ENDPOINT
 from danswer.configs.model_configs import GEN_AI_MODEL_PROVIDER
-from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
 from danswer.db.chat import delete_old_default_personas
 from danswer.db.connector import create_initial_default_connector
 from danswer.db.connector_credential_pair import associate_default_cc_pair
@@ -52,6 +50,7 @@
 from danswer.db.index_attempt import expire_index_attempts
 from danswer.document_index.factory import get_default_document_index
 from danswer.llm.factory import get_default_llm
+from danswer.llm.utils import get_default_llm_version
 from danswer.search.search_nlp_models import warm_up_models
 from danswer.server.danswer_api.ingestion import get_danswer_api_key
 from danswer.server.danswer_api.ingestion import router as danswer_api_router
@@ -234,11 +233,10 @@ def startup_event() -> None:
             logger.info("Generative AI Q&A disabled")
         else:
             logger.info(f"Using LLM Provider: {GEN_AI_MODEL_PROVIDER}")
-            logger.info(f"Using LLM Model Version: {GEN_AI_MODEL_VERSION}")
-            if GEN_AI_MODEL_VERSION != FAST_GEN_AI_MODEL_VERSION:
-                logger.info(
-                    f"Using Fast LLM Model Version: {FAST_GEN_AI_MODEL_VERSION}"
-                )
+            base, fast = get_default_llm_version()
+            logger.info(f"Using LLM Model Version: {base}")
+            if base != fast:
+                logger.info(f"Using Fast LLM Model Version: {fast}")
             if GEN_AI_API_ENDPOINT:
                 logger.info(f"Using LLM Endpoint: {GEN_AI_API_ENDPOINT}")
 

diff --git a/backend/danswer/server/features/persona/api.py b/backend/danswer/server/features/persona/api.py
@@ -7,7 +7,6 @@
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_user
 from danswer.configs.model_configs import GEN_AI_MODEL_PROVIDER
-from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
 from danswer.db.chat import get_persona_by_id
 from danswer.db.chat import get_personas
 from danswer.db.chat import get_prompts_by_ids
@@ -18,6 +17,7 @@
 from danswer.db.document_set import get_document_sets_by_ids
 from danswer.db.engine import get_session
 from danswer.db.models import User
+from danswer.llm.utils import get_default_llm_version
 from danswer.one_shot_answer.qa_block import build_dummy_prompt
 from danswer.server.features.persona.models import CreatePersonaRequest
 from danswer.server.features.persona.models import PersonaSnapshot
@@ -239,4 +239,4 @@ def get_default_model(
     if GEN_AI_MODEL_PROVIDER != "openai":
         return ""
 
-    return GEN_AI_MODEL_VERSION
+    return get_default_llm_version()[0]