🅰️ feat: Azure Config to Allow Different Deployments per Model (danny…

…-avila#1863) * wip: first pass for azure endpoint schema * refactor: azure config to return groupMap and modelConfigMap * wip: naming and schema changes * refactor(errorsToString): move to data-provider * feat: rename to azureGroups, add additional tests, tests all expected outcomes, return errors * feat(AppService): load Azure groups * refactor(azure): use imported types, write `mapModelToAzureConfig` * refactor: move `extractEnvVariable` to data-provider * refactor(validateAzureGroups): throw on duplicate groups or models; feat(mapModelToAzureConfig): throw if env vars not present, add tests * refactor(AppService): ensure each model is properly configured on startup * refactor: deprecate azureOpenAI environment variables in favor of librechat.yaml config * feat: use helper functions to handle and order enabled/default endpoints; initialize azureOpenAI from config file * refactor: redefine types as well as load azureOpenAI models from config file * chore(ci): fix test description naming * feat(azureOpenAI): use validated model grouping for request authentication * chore: bump data-provider following rebase * chore: bump config file version noting significant changes * feat: add title options and switch azure configs for titling and vision requests * feat: enable azure plugins from config file * fix(ci): pass tests * chore(.env.example): mark `PLUGINS_USE_AZURE` as deprecated * fix(fetchModels): early return if apiKey not passed * chore: fix azure config typing * refactor(mapModelToAzureConfig): return baseURL and headers as well as azureOptions * feat(createLLM): use `azureOpenAIBasePath` * feat(parsers): resolveHeaders * refactor(extractBaseURL): handle invalid input * feat(OpenAIClient): handle headers and baseURL for azureConfig * fix(ci): pass `OpenAIClient` tests * chore: extract env var for azureOpenAI group config, baseURL * docs: azureOpenAI config setup docs * feat: safe check of potential conflicting env vars that map to unique placeholders * fix: reset apiKey when model switches from originally requested model (vision or title) * chore: linting * docs: CONFIG_PATH notes in custom_config.md
happy-ryo · Feb 26, 2024 · 097a978 · 097a978
1 parent 7a55132
commit 097a978
Show file tree

Hide file tree

Showing 37 changed files with 2,068 additions and 396 deletions.
diff --git a/.env.example b/.env.example
@@ -66,18 +66,21 @@ ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2
 # Azure      #
 #============#
 
-# AZURE_API_KEY=
-AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4
-# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo
-# PLUGINS_USE_AZURE="true"
-
-AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
-
-# AZURE_OPENAI_API_INSTANCE_NAME=
-# AZURE_OPENAI_API_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_VERSION=
-# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=
+
+# Note: these variables are DEPRECATED
+# Use the `librechat.yaml` configuration for `azureOpenAI` instead
+# You may also continue to use them if you opt out of using the `librechat.yaml` configuration
+
+# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated
+# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 # Deprecated
+# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE # Deprecated
+# AZURE_API_KEY= # Deprecated
+# AZURE_OPENAI_API_INSTANCE_NAME= # Deprecated
+# AZURE_OPENAI_API_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_VERSION= # Deprecated
+# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated
+# PLUGINS_USE_AZURE="true" # Deprecated
 
 #============#
 # BingAI     #

diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js
@@ -1,10 +1,13 @@
 const OpenAI = require('openai');
 const { HttpsProxyAgent } = require('https-proxy-agent');
 const {
+  ImageDetail,
+  EModelEndpoint,
+  resolveHeaders,
+  ImageDetailCost,
   getResponseSender,
   validateVisionModel,
-  ImageDetailCost,
-  ImageDetail,
+  mapModelToAzureConfig,
 } = require('librechat-data-provider');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const {
@@ -665,6 +668,16 @@ class OpenAIClient extends BaseClient {
       };
     }
 
+    const { headers } = this.options;
+    if (headers && typeof headers === 'object' && !Array.isArray(headers)) {
+      configOptions.baseOptions = {
+        headers: resolveHeaders({
+          ...headers,
+          ...configOptions?.baseOptions?.headers,
+        }),
+      };
+    }
+
     if (this.options.proxy) {
       configOptions.httpAgent = new HttpsProxyAgent(this.options.proxy);
       configOptions.httpsAgent = new HttpsProxyAgent(this.options.proxy);
@@ -725,6 +738,26 @@ class OpenAIClient extends BaseClient {
       max_tokens: 16,
     };
 
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+    if (this.azure && azureConfig) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+      this.azure = azureOptions;
+      this.options.headers = resolveHeaders(headers);
+      this.options.reverseProxyUrl = baseURL ?? null;
+      this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+    }
+
     const titleChatCompletion = async () => {
       modelOptions.model = model;
 
@@ -975,6 +1008,27 @@ ${convo}
         modelOptions.max_tokens = 4000;
       }
 
+      /** @type {TAzureConfig | undefined} */
+      const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+      if (this.azure && this.isVisionModel && azureConfig) {
+        const { modelGroupMap, groupMap } = azureConfig;
+        const {
+          azureOptions,
+          baseURL,
+          headers = {},
+        } = mapModelToAzureConfig({
+          modelName: modelOptions.model,
+          modelGroupMap,
+          groupMap,
+        });
+        this.azure = azureOptions;
+        this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this);
+        opts.defaultHeaders = resolveHeaders(headers);
+        this.langchainProxy = extractBaseURL(baseURL);
+        this.apiKey = azureOptions.azureOpenAIApiKey;
+      }
+
       if (this.azure || this.options.azure) {
         // Azure does not accept `model` in the body, so we need to remove it.
         delete modelOptions.model;
@@ -1026,12 +1080,20 @@ ${convo}
           ...modelOptions,
           ...this.options.addParams,
         };
+        logger.debug('[OpenAIClient] chatCompletion: added params', {
+          addParams: this.options.addParams,
+          modelOptions,
+        });
       }
 
       if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
         this.options.dropParams.forEach((param) => {
           delete modelOptions[param];
         });
+        logger.debug('[OpenAIClient] chatCompletion: dropped params', {
+          dropParams: this.options.dropParams,
+          modelOptions,
+        });
       }
 
       let UnexpectedRoleError = false;

diff --git a/api/app/clients/llm/createLLM.js b/api/app/clients/llm/createLLM.js
@@ -55,10 +55,13 @@ function createLLM({
   }
 
   if (azure && configOptions.basePath) {
-    configOptions.basePath = constructAzureURL({
+    const azureURL = constructAzureURL({
       baseURL: configOptions.basePath,
       azure: azureOptions,
     });
+    azureOptions.azureOpenAIBasePath = azureURL.split(
+      `/${azureOptions.azureOpenAIApiDeploymentName}`,
+    )[0];
   }
 
   return new ChatOpenAI(

diff --git a/api/server/controllers/EndpointController.js b/api/server/controllers/EndpointController.js
@@ -1,4 +1,4 @@
-const { CacheKeys, EModelEndpoint } = require('librechat-data-provider');
+const { CacheKeys, EModelEndpoint, orderEndpointsConfig } = require('librechat-data-provider');
 const { loadDefaultEndpointsConfig, loadConfigEndpoints } = require('~/server/services/Config');
 const { getLogStores } = require('~/cache');
 
@@ -10,15 +10,18 @@ async function endpointController(req, res) {
     return;
   }
 
-  const defaultEndpointsConfig = await loadDefaultEndpointsConfig();
-  const customConfigEndpoints = await loadConfigEndpoints();
+  const defaultEndpointsConfig = await loadDefaultEndpointsConfig(req);
+  const customConfigEndpoints = await loadConfigEndpoints(req);
 
-  const endpointsConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints };
-  if (endpointsConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) {
-    endpointsConfig[EModelEndpoint.assistants].disableBuilder =
+  /** @type {TEndpointsConfig} */
+  const mergedConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints };
+  if (mergedConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) {
+    mergedConfig[EModelEndpoint.assistants].disableBuilder =
       req.app.locals[EModelEndpoint.assistants].disableBuilder;
   }
 
+  const endpointsConfig = orderEndpointsConfig(mergedConfig);
+
   await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig);
   res.send(JSON.stringify(endpointsConfig));
 }

diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js
@@ -1,8 +1,12 @@
 const {
+  Constants,
   FileSources,
   EModelEndpoint,
-  Constants,
   defaultSocialLogins,
+  validateAzureGroups,
+  mapModelToAzureConfig,
+  deprecatedAzureVariables,
+  conflictingAzureVariables,
 } = require('librechat-data-provider');
 const { initializeFirebase } = require('./Files/Firebase/initialize');
 const loadCustomConfig = require('./Config/loadCustomConfig');
@@ -62,6 +66,50 @@ const AppService = async (app) => {
   handleRateLimits(config?.rateLimits);
 
   const endpointLocals = {};
+
+  if (config?.endpoints?.[EModelEndpoint.azureOpenAI]) {
+    const { groups, titleModel, titleConvo, titleMethod, plugins } =
+      config.endpoints[EModelEndpoint.azureOpenAI];
+    const { isValid, modelNames, modelGroupMap, groupMap, errors } = validateAzureGroups(groups);
+
+    if (!isValid) {
+      const errorString = errors.join('\n');
+      const errorMessage = 'Invalid Azure OpenAI configuration:\n' + errorString;
+      logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    for (const modelName of modelNames) {
+      mapModelToAzureConfig({ modelName, modelGroupMap, groupMap });
+    }
+
+    endpointLocals[EModelEndpoint.azureOpenAI] = {
+      modelNames,
+      modelGroupMap,
+      groupMap,
+      titleConvo,
+      titleMethod,
+      titleModel,
+      plugins,
+    };
+
+    deprecatedAzureVariables.forEach(({ key, description }) => {
+      if (process.env[key]) {
+        logger.warn(
+          `The \`${key}\` environment variable (related to ${description}) should not be used in combination with the \`azureOpenAI\` endpoint configuration, as you will experience conflicts and errors.`,
+        );
+      }
+    });
+
+    conflictingAzureVariables.forEach(({ key }) => {
+      if (process.env[key]) {
+        logger.warn(
+          `The \`${key}\` environment variable should not be used in combination with the \`azureOpenAI\` endpoint configuration, as you may experience with the defined placeholders for mapping to the current model grouping using the same name.`,
+        );
+      }
+    });
+  }
+
   if (config?.endpoints?.[EModelEndpoint.assistants]) {
     const { disableBuilder, pollIntervalMs, timeoutMs, supportedIds, excludedIds } =
       config.endpoints[EModelEndpoint.assistants];