🚅 docs: Updated Example for LiteLLM ports and Volume mount (#2941)

* Added necessary "ports" section for it to work by default
* Added (commented out) example GCP Vertex volume mount for auth config and for ENV variable.

.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM node:18-bullseye
+
+RUN useradd -m -s /bin/bash vscode
+RUN mkdir -p /workspaces && chown -R vscode:vscode /workspaces
+WORKDIR /workspaces

.devcontainer/devcontainer.json

@@ -1,58 +1,18 @@
-// {
-// 	"name": "LibreChat_dev",
-// 	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
-// 	"dockerComposeFile": "docker-compose.yml",
-// 	// The 'service' property is the name of the service for the container that VS Code should
-// 	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
-// 	"service": "librechat",
-// 	// The 'workspaceFolder' property is the path VS Code should open by default when
-// 	// connected. Corresponds to a volume mount in .devcontainer/docker-compose.yml
-// 	"workspaceFolder": "/workspace"
-//     //,
-// 	// // Set *default* container specific settings.json values on container create.
-// 	// "settings": {},
-// 	// // Add the IDs of extensions you want installed when the container is created.
-// 	// "extensions": [],
-// 	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
-// 	// "shutdownAction": "none",
-// 	// Uncomment the next line to use 'postCreateCommand' to run commands after the container is created.
-// 	// "postCreateCommand": "uname -a",
-// 	// Comment out to connect as root instead. To add a non-root user, see: https://aka.ms/vscode-remote/containers/non-root.
-// 	// "remoteUser": "vscode"
-// }
 {
-    // "name": "LibreChat_dev",
-    "dockerComposeFile": "docker-compose.yml",
-    "service": "app",
-    // "image": "node:19-alpine",
-    // "workspaceFolder": "/workspaces",
-    "workspaceFolder": "/workspace",
-    // Set *default* container specific settings.json values on container create.
-    // "overrideCommand": true,
-    "customizations": {
-        "vscode": {
-            "extensions": [],
-            "settings": {
-                "terminal.integrated.profiles.linux": {
-                    "bash": null
-                }
-            }
+  "dockerComposeFile": "docker-compose.yml",
+  "service": "app",
+  "workspaceFolder": "/workspaces",
+  "customizations": {
+    "vscode": {
+      "extensions": [],
+      "settings": {
+        "terminal.integrated.profiles.linux": {
+          "bash": null
         }
-    },
-    "postCreateCommand": "",
-    // "workspaceMount": "src=${localWorkspaceFolder},dst=/code,type=bind,consistency=cached"
-
-    // "runArgs": [
-    //     "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined",
-    //     "-v", "/tmp/.X11-unix:/tmp/.X11-unix",
-    //     "-v", "${env:XAUTHORITY}:/root/.Xauthority:rw",
-    //     "-v", "/home/${env:USER}/.cdh:/root/.cdh",
-    //     "-e", "DISPLAY=${env:DISPLAY}",
-    //     "--name=tgw_assistant_backend_dev",
-    //     "--network=host"
-    // ],
-    // "settings": {
-    //     "terminal.integrated.shell.linux": "/bin/bash"
-    // },
-    "features": {"ghcr.io/devcontainers/features/git:1": {}}
+      }
+    }
+  },
+  "postCreateCommand": "",
+  "features": { "ghcr.io/devcontainers/features/git:1": {} },
+  "remoteUser": "vscode"
 }

.devcontainer/docker-compose.yml

@@ -1,17 +1,11 @@
-version: '3.4'
+version: "3.8"
 
 services:
   app:
-    # container_name: LibreChat_dev
-    image: node:19-bullseye
-    # Using a Dockerfile is optional, but included for completeness.
-    # build: 
-    #   context: .
-    #   dockerfile: Dockerfile
-    #   # [Optional] You can use build args to set options. e.g. 'VARIANT' below affects the image in the Dockerfile
-    #   args: 
-    #     VARIANT: buster
-    # network_mode: "host"
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
+    # restart: always
     links:
       - mongodb
       - meilisearch
@@ -21,17 +15,16 @@ services:
     - "host.docker.internal:host-gateway"
 
     volumes:
-    #   # This is where VS Code should expect to find your project's source code and the value of "workspaceFolder" in .devcontainer/devcontainer.json
-      - ..:/workspace:cached
-    #   # - /app/client/node_modules
-    #   # - ./api:/app/api
-    #   # - ./.env:/app/.env
-    #   # - ./.env.development:/app/.env.development
-    #   # - ./.env.production:/app/.env.production
-    #   # - /app/api/node_modules
-
-    #   # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details.
-    #   # - /var/run/docker.sock:/var/run/docker.sock 
+      # This is where VS Code should expect to find your project's source code and the value of "workspaceFolder" in .devcontainer/devcontainer.json
+      - ..:/workspaces:cached
+      # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details.
+      # - /var/run/docker.sock:/var/run/docker.sock 
+    environment:
+      - HOST=0.0.0.0
+      - MONGO_URI=mongodb://mongodb:27017/LibreChat
+      # - CHATGPT_REVERSE_PROXY=http://host.docker.internal:8080/api/conversation # if you are hosting your own chatgpt reverse proxy with docker
+      # - OPENAI_REVERSE_PROXY=http://host.docker.internal:8070/v1/chat/completions # if you are hosting your own chatgpt reverse proxy with docker
+      - MEILI_HOST=http://meilisearch:7700
 
     # Runs app on the same network as the service container, allows "forwardPorts" in devcontainer.json function.
     # network_mode: service:another-service
@@ -39,45 +32,34 @@ services:
     # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. 
     # (Adding the "ports" property to this file will not forward from a Codespace.)
 
-    # Uncomment the next line to use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
-    # user: vscode
-
-    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
-    # cap_add:
-    #   - SYS_PTRACE
-    # security_opt:
-    #   - seccomp:unconfined
+    # Use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
+    user: vscode
 
     # Overrides default command so things don't shut down after the process ends.
     command: /bin/sh -c "while sleep 1000; do :; done"  
 
   mongodb:
     container_name: chat-mongodb
-    # network_mode: "host"
     expose:
       - 27017
     # ports:
     #   - 27018:27017
     image: mongo
-  #   restart: always
+    # restart: always
     volumes:
       - ./data-node:/data/db
     command: mongod --noauth
   meilisearch:
     container_name: chat-meilisearch
-    image: getmeili/meilisearch:v1.0
-    # network_mode: "host"
+    image: getmeili/meilisearch:v1.5
+    # restart: always
     expose:
       - 7700
-    # ports:
-    #   - 7700:7700
-    # env_file:
-    #   - .env
+    # Uncomment this to access meilisearch from outside docker
+    # ports: 
+    #   - 7700:7700 # if exposing these ports, make sure your master key is not the default value
     environment:
-      - SEARCH=false
-      - MEILI_HOST=http://0.0.0.0:7700
-      - MEILI_HTTP_ADDR=0.0.0.0:7700
+      - MEILI_NO_ANALYTICS=true
       - MEILI_MASTER_KEY=5c71cf56d672d009e36070b5bc5e47b743535ae55c818ae3b735bb6ebfb4ba63
     volumes:
-      - ./meili_data:/meili_data
-  
+      - ./meili_data_v1.5:/meili_data

.dockerignore

@@ -1,5 +1,17 @@
+**/.circleci
+**/.editorconfig
+**/.dockerignore
+**/.git
+**/.DS_Store
+**/.vscode
 **/node_modules
-client/dist/images
+
+# Specific patterns to ignore
 data-node
-.env
-**/.env
+meili_data*
+librechat*
+Dockerfile*
+docs
+
+# Ignore all hidden files
+.*

.env.example

@@ -1,44 +1,285 @@
-##########################
-# Server configuration: 
-##########################
+#=====================================================================#
+#                       LibreChat Configuration                       #
+#=====================================================================#
+# Please refer to the reference documentation for assistance          #
+# with configuring your LibreChat environment.                        #
+#                                                                     #
+# https://www.librechat.ai/docs/configuration/dotenv                  #
+#=====================================================================#
 
-APP_TITLE=LibreChat
+#==================================================#
+#               Server Configuration               #
+#==================================================#
 
-# The server will listen to localhost:3080 by default. You can change the target IP as you want. 
-# If you want to make this server available externally, for example to share the server with others 
-# or expose this from a Docker container, set host to 0.0.0.0 or your external IP interface. 
-# Tips: Setting host to 0.0.0.0 means listening on all interfaces. It's not a real IP.  
-# Use localhost:port rather than 0.0.0.0:port to access the server. 
-# Set Node env to development if running in dev mode.
 HOST=localhost
 PORT=3080
 
-# Note: the following enables user balances, which you can add manually
-# or you will need to build out a balance accruing system for users.
-# For more info, see https://docs.librechat.ai/features/token_usage.html
+MONGO_URI=mongodb://127.0.0.1:27017/LibreChat
 
-# To manually add balances, run the following command:
-# `npm run add-balance`
+DOMAIN_CLIENT=http://localhost:3080
+DOMAIN_SERVER=http://localhost:3080
 
-# You can also specify the email and token credit amount to add, e.g.:
-# `npm run add-balance example@example.com 1000`
+NO_INDEX=true
 
-# This works well to track your own usage for personal use; 1000 credits = $0.001 (1 mill USD)
+#===============#
+# JSON Logging  #
+#===============#
 
-# Set to true to enable token credit balances for the OpenAI/Plugins endpoints
-CHECK_BALANCE=false
+# Use when process console logs in cloud deployment like GCP/AWS
+CONSOLE_JSON=false
 
-# Automated Moderation System
-# The Automated Moderation System uses a scoring mechanism to track user violations. As users commit actions
-# like excessive logins, registrations, or messaging, they accumulate violation scores. Upon reaching
-# a set threshold, the user and their IP are temporarily banned. This system ensures platform security
-# by monitoring and penalizing rapid or suspicious activities.
+#===============#
+# Debug Logging #
+#===============#
 
-BAN_VIOLATIONS=true # Whether or not to enable banning users for violations (they will still be logged)
-BAN_DURATION=1000 * 60 * 60 * 2 # how long the user and associated IP are banned for
-BAN_INTERVAL=20 # a user will be banned everytime their score reaches/crosses over the interval threshold
+DEBUG_LOGGING=true
+DEBUG_CONSOLE=false
 
-# The score for each violation
+#=============#
+# Permissions #
+#=============#
+
+# UID=1000
+# GID=1000
+
+#===============#
+# Configuration #
+#===============#
+# Use an absolute path, a relative path, or a URL
+
+# CONFIG_PATH="/alternative/path/to/librechat.yaml"
+
+#===================================================#
+#                     Endpoints                     #
+#===================================================#
+
+# ENDPOINTS=openAI,assistants,azureOpenAI,bingAI,google,gptPlugins,anthropic
+
+PROXY=
+
+#===================================#
+# Known Endpoints - librechat.yaml  #
+#===================================#
+# https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints
+
+# ANYSCALE_API_KEY=
+# APIPIE_API_KEY=
+# FIREWORKS_API_KEY=
+# GROQ_API_KEY=
+# HUGGINGFACE_TOKEN=
+# MISTRAL_API_KEY=
+# OPENROUTER_KEY=
+# PERPLEXITY_API_KEY=
+# SHUTTLEAI_API_KEY=
+# TOGETHERAI_API_KEY=
+
+#============#
+# Anthropic  #
+#============#
+
+ANTHROPIC_API_KEY=user_provided
+# ANTHROPIC_MODELS=claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
+# ANTHROPIC_REVERSE_PROXY=
+
+#============#
+# Azure      #
+#============#
+
+
+# Note: these variables are DEPRECATED
+# Use the `librechat.yaml` configuration for `azureOpenAI` instead
+# You may also continue to use them if you opt out of using the `librechat.yaml` configuration
+
+# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated
+# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 # Deprecated
+# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE # Deprecated
+# AZURE_API_KEY= # Deprecated
+# AZURE_OPENAI_API_INSTANCE_NAME= # Deprecated
+# AZURE_OPENAI_API_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_VERSION= # Deprecated
+# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated
+# PLUGINS_USE_AZURE="true" # Deprecated
+
+#============#
+# BingAI     #
+#============#
+
+BINGAI_TOKEN=user_provided
+# BINGAI_HOST=https://cn.bing.com
+
+#============#
+# Google     #
+#============#
+
+GOOGLE_KEY=user_provided
+# GOOGLE_REVERSE_PROXY=
+
+# Gemini API
+# GOOGLE_MODELS=gemini-1.5-flash-latest,gemini-1.0-pro,gemini-1.0-pro-001,gemini-1.0-pro-latest,gemini-1.0-pro-vision-latest,gemini-1.5-pro-latest,gemini-pro,gemini-pro-vision
+
+# Vertex AI
+# GOOGLE_MODELS=gemini-1.5-flash-preview-0514,gemini-1.5-pro-preview-0514,gemini-1.0-pro-vision-001,gemini-1.0-pro-002,gemini-1.0-pro-001,gemini-pro-vision,gemini-1.0-pro
+
+# Google Gemini Safety Settings
+# NOTE (Vertex AI): You do not have access to the BLOCK_NONE setting by default.
+# To use this restricted HarmBlockThreshold setting, you will need to either:
+#
+# (a) Get access through an allowlist via your Google account team
+# (b) Switch your account type to monthly invoiced billing following this instruction:
+#     https://cloud.google.com/billing/docs/how-to/invoiced-billing
+#
+# GOOGLE_SAFETY_SEXUALLY_EXPLICIT=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_HATE_SPEECH=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_HARASSMENT=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_DANGEROUS_CONTENT=BLOCK_ONLY_HIGH
+
+
+#============#
+# OpenAI     #
+#============#
+
+OPENAI_API_KEY=user_provided
+# OPENAI_MODELS=gpt-4o,gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k
+
+DEBUG_OPENAI=false
+
+# TITLE_CONVO=false
+# OPENAI_TITLE_MODEL=gpt-3.5-turbo
+
+# OPENAI_SUMMARIZE=true
+# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo
+
+# OPENAI_FORCE_PROMPT=true
+
+# OPENAI_REVERSE_PROXY=
+
+# OPENAI_ORGANIZATION=
+
+#====================#
+#   Assistants API   #
+#====================#
+
+ASSISTANTS_API_KEY=user_provided
+# ASSISTANTS_BASE_URL=
+# ASSISTANTS_MODELS=gpt-4o,gpt-3.5-turbo-0125,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-16k,gpt-3.5-turbo,gpt-4,gpt-4-0314,gpt-4-32k-0314,gpt-4-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-1106,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview
+
+#==========================#
+#   Azure Assistants API   #
+#==========================#
+
+# Note: You should map your credentials with custom variables according to your Azure OpenAI Configuration
+# The models for Azure Assistants are also determined by your Azure OpenAI configuration.
+
+# More info, including how to enable use of Assistants with Azure here:
+# https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints/azure#using-assistants-with-azure
+
+#============#
+# OpenRouter #
+#============#
+# !!!Warning: Use the variable above instead of this one. Using this one will override the OpenAI endpoint
+# OPENROUTER_API_KEY=
+
+#============#
+# Plugins    #
+#============#
+
+# PLUGIN_MODELS=gpt-4o,gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613
+
+DEBUG_PLUGINS=true
+
+CREDS_KEY=f34be427ebb29de8d88c107a71546019685ed8b241d8f2ed00c3df97ad2566f0
+CREDS_IV=e2341419ec3dd3d19b13a1a87fafcbfb
+
+# Azure AI Search
+#-----------------
+AZURE_AI_SEARCH_SERVICE_ENDPOINT=
+AZURE_AI_SEARCH_INDEX_NAME=
+AZURE_AI_SEARCH_API_KEY=
+
+AZURE_AI_SEARCH_API_VERSION=
+AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE=
+AZURE_AI_SEARCH_SEARCH_OPTION_TOP=
+AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
+
+# DALL·E
+#----------------
+# DALLE_API_KEY=
+# DALLE3_API_KEY=
+# DALLE2_API_KEY=
+# DALLE3_SYSTEM_PROMPT=
+# DALLE2_SYSTEM_PROMPT=
+# DALLE_REVERSE_PROXY=
+# DALLE3_BASEURL=
+# DALLE2_BASEURL=
+
+# DALL·E (via Azure OpenAI)
+# Note: requires some of the variables above to be set
+#----------------
+# DALLE3_AZURE_API_VERSION=
+# DALLE2_AZURE_API_VERSION=
+
+# Google
+#-----------------
+GOOGLE_SEARCH_API_KEY=
+GOOGLE_CSE_ID=
+
+# SerpAPI
+#-----------------
+SERPAPI_API_KEY=
+
+# Stable Diffusion
+#-----------------
+SD_WEBUI_URL=http://host.docker.internal:7860
+
+# Tavily
+#-----------------
+TAVILY_API_KEY=
+
+# Traversaal
+#-----------------
+TRAVERSAAL_API_KEY=
+
+# WolframAlpha
+#-----------------
+WOLFRAM_APP_ID=
+
+# Zapier
+#-----------------
+ZAPIER_NLA_API_KEY=
+
+#==================================================#
+#                      Search                      #
+#==================================================#
+
+SEARCH=true
+MEILI_NO_ANALYTICS=true
+MEILI_HOST=http://0.0.0.0:7700
+MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
+
+
+#==================================================#
+#          Speech to Text & Text to Speech         #
+#==================================================#
+
+STT_API_KEY=
+TTS_API_KEY=
+
+#===================================================#
+#                    User System                    #
+#===================================================#
+
+#========================#
+# Moderation             #
+#========================#
+
+OPENAI_MODERATION=false
+OPENAI_MODERATION_API_KEY=
+# OPENAI_MODERATION_REVERSE_PROXY=
+
+BAN_VIOLATIONS=true
+BAN_DURATION=1000 * 60 * 60 * 2
+BAN_INTERVAL=20
 
 LOGIN_VIOLATION_SCORE=1
 REGISTRATION_VIOLATION_SCORE=1
@@ -46,360 +287,132 @@ CONCURRENT_VIOLATION_SCORE=1
 MESSAGE_VIOLATION_SCORE=1
 NON_BROWSER_VIOLATION_SCORE=20
 
-# Login and registration rate limiting.
+LOGIN_MAX=7
+LOGIN_WINDOW=5
+REGISTER_MAX=5
+REGISTER_WINDOW=60
 
-LOGIN_MAX=7 # The max amount of logins allowed per IP per LOGIN_WINDOW
-LOGIN_WINDOW=5 # in minutes, determines the window of time for LOGIN_MAX logins
-REGISTER_MAX=5 # The max amount of registrations allowed per IP per REGISTER_WINDOW
-REGISTER_WINDOW=60 # in minutes, determines the window of time for REGISTER_MAX registrations
+LIMIT_CONCURRENT_MESSAGES=true
+CONCURRENT_MESSAGE_MAX=2
 
-# Message rate limiting (per user & IP)
+LIMIT_MESSAGE_IP=true
+MESSAGE_IP_MAX=40
+MESSAGE_IP_WINDOW=1
 
-LIMIT_CONCURRENT_MESSAGES=true # Whether to limit the amount of messages a user can send per request
-CONCURRENT_MESSAGE_MAX=2 # The max amount of messages a user can send per request
+LIMIT_MESSAGE_USER=false
+MESSAGE_USER_MAX=40
+MESSAGE_USER_WINDOW=1
 
-LIMIT_MESSAGE_IP=true # Whether to limit the amount of messages an IP can send per MESSAGE_IP_WINDOW
-MESSAGE_IP_MAX=40 # The max amount of messages an IP can send per MESSAGE_IP_WINDOW
-MESSAGE_IP_WINDOW=1 # in minutes, determines the window of time for MESSAGE_IP_MAX messages
+ILLEGAL_MODEL_REQ_SCORE=5
 
-# Note: You can utilize both limiters, but default is to limit by IP only.
-LIMIT_MESSAGE_USER=false # Whether to limit the amount of messages an IP can send per MESSAGE_USER_WINDOW
-MESSAGE_USER_MAX=40 # The max amount of messages an IP can send per MESSAGE_USER_WINDOW
-MESSAGE_USER_WINDOW=1 # in minutes, determines the window of time for MESSAGE_USER_MAX messages
+#========================#
+# Balance                #
+#========================#
 
-# If you have permission problems, set here the UID and GID of the user running
-# the docker compose command. The applications in the container will run with these uid/gid.
-UID=1000
-GID=1000
+CHECK_BALANCE=false
 
-# Change this to proxy any API request. 
-# It's useful if your machine has difficulty calling the original API server. 
-# PROXY=
+#========================#
+# Registration and Login #
+#========================#
 
-# Change this to your MongoDB URI if different. I recommend appending LibreChat. 
-MONGO_URI=mongodb://127.0.0.1:27018/LibreChat
-
-##########################
-# OpenAI Endpoint: 
-##########################
-
-# Access key from OpenAI platform. 
-# Leave it blank to disable this feature.
-# Set to "user_provided" to allow the user to provide their API key from the UI.
-OPENAI_API_KEY=user_provided
-
-DEBUG_OPENAI=false # Set to true to enable debug mode for the OpenAI endpoint
-
-# Identify the available models, separated by commas *without spaces*. 
-# The first will be default. 
-# Leave it blank to use internal settings. 
-# OPENAI_MODELS=gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,text-davinci-003,gpt-4,gpt-4-0314,gpt-4-0613
-
-# Titling is enabled by default when initiating a conversation.
-# Uncomment the following variable to disable this feature.
-# TITLE_CONVO=false
-
-# (Optional) The default model used for titling by is gpt-3.5-turbo-0613
-# You can change it by uncommenting the following and setting the desired model
-# Must be compatible with the OpenAI Endpoint.
-# OPENAI_TITLE_MODEL=gpt-3.5-turbo
-
-# (Optional/Experimental) Enable message summarization by uncommenting the following:
-# Note: this may affect response time when a summary is being generated.
-# OPENAI_SUMMARIZE=true
-
-# Not yet implemented: this will be a conversation option enabled by default to save users on tokens
-# We are using the ConversationSummaryBufferMemory method to summarize messages.
-# To learn more about this, see this article:
-# https://www.pinecone.io/learn/series/langchain/langchain-conversational-memory/
-
-# (Optional) The default model used for summarizing is gpt-3.5-turbo
-# You can change it by uncommenting the following and setting the desired model
-# Must be compatible with the OpenAI Endpoint.
-# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo
-
-# Reverse proxy settings for OpenAI: 
-# https://github.com/waylaidwanderer/node-chatgpt-api#using-a-reverse-proxy 
-# OPENAI_REVERSE_PROXY=
-
-# (Advanced) Sometimes when using Local LLM APIs, you may need to force the API
-# to be called with a `prompt` payload instead of a `messages` payload; to mimic the
-# a `/v1/completions` request instead of `/v1/chat/completions`
-# This may be the case for LocalAI with some models. To do so, uncomment the following:
-# OPENAI_FORCE_PROMPT=true
-
-##########################
-# OpenRouter (overrides OpenAI and Plugins Endpoints): 
-##########################
-
-# OpenRouter is a legitimate proxy service to a multitude of LLMs, both closed and open source, including:
-# OpenAI models, Anthropic models, Meta's Llama models, pygmalionai/mythalion-13b
-# and many more open source models. Newer integrations are usually discounted, too!
-
-# Note: this overrides the OpenAI and Plugins Endpoints.
-# See ./docs/install/free_ai_apis.md for more info.
-
-# OPENROUTER_API_KEY=
-
-##########################
-# AZURE Endpoint:   
-##########################
-
-# To use Azure with this project, set the following variables. These will be used to build the API URL.
-# Chat completion: 
-# `https://{AZURE_OPENAI_API_INSTANCE_NAME}.openai.azure.com/openai/deployments/{AZURE_OPENAI_API_DEPLOYMENT_NAME}/chat/completions?api-version={AZURE_OPENAI_API_VERSION}`;
-# You should also consider changing the `OPENAI_MODELS` variable above to the models available in your instance/deployment.
-# Note: I've noticed that the Azure API is much faster than the OpenAI API, so the streaming looks almost instantaneous.
-# Note "AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME" and "AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME" are optional but might be used in the future
-
-# AZURE_API_KEY=
-# AZURE_OPENAI_API_INSTANCE_NAME=
-# AZURE_OPENAI_API_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_VERSION=
-# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=
-
-# Identify the available models, separated by commas *without spaces*. 
-# The first will be default. 
-# Leave it blank to use internal settings. 
-AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4
-
-# To use Azure with the Plugins endpoint, you need the variables above, and uncomment the following variable:
-# NOTE: This may not work as expected and Azure OpenAI may not support OpenAI Functions yet
-# Omit/leave it commented to use the default OpenAI API
-
-# PLUGINS_USE_AZURE="true"
-
-##########################
-# ChatGPT Endpoint: 
-##########################
-
-# ChatGPT Browser Client (free but use at your own risk) 
-# Access token from https://chat.openai.com/api/auth/session 
-# Exposes your access token to `CHATGPT_REVERSE_PROXY`
-# Set to "user_provided" to allow the user to provide its token from the UI.
-# Leave it blank to disable this endpoint 
-CHATGPT_TOKEN=user_provided
-
-# Identify the available models, separated by commas. The first will be default. 
-# Leave it blank to use internal settings. 
-CHATGPT_MODELS=text-davinci-002-render-sha,gpt-4
-# NOTE: you can add gpt-4-plugins, gpt-4-code-interpreter, and gpt-4-browsing to the list above and use the models for these features;
-# however, the view/display portion of these features are not supported, but you can use the underlying models, which have higher token context
-# Also: text-davinci-002-render-paid is deprecated as of May 2023
-
-# Reverse proxy setting for OpenAI
-# https://github.com/waylaidwanderer/node-chatgpt-api#using-a-reverse-proxy
-# By default it will use the node-chatgpt-api recommended proxy, (it's a third party server)
-# CHATGPT_REVERSE_PROXY=<YOUR REVERSE PROXY>
-
-##########################
-# BingAI Endpoint:   
-##########################
-
-# Also used for Sydney and jailbreak
-# To get your Access token for Bing, login to https://www.bing.com
-# Use dev tools or an extension while logged into the site to copy the content of the _U cookie. 
-# If this fails, follow these instructions https://github.com/danny-avila/LibreChat/issues/370#issuecomment-1560382302 to provide the full cookie strings 
-# or check out our discord https://discord.com/channels/1086345563026489514/1143941308684177429
-# Set to "user_provided" to allow the user to provide its token from the UI.
-# Leave it blank to disable this endpoint. 
-BINGAI_TOKEN=user_provided
-
-# BingAI Host: 
-# Necessary for some people in different countries, e.g. China (https://cn.bing.com) 
-# Leave it blank to use default server. 
-# BINGAI_HOST=https://cn.bing.com 
-
-#############################
-# Plugins:
-#############################
-
-# Identify the available models, separated by commas *without spaces*. 
-# The first will be default. 
-# Leave it blank to use internal settings. 
-# PLUGIN_MODELS=gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,gpt-4,gpt-4-0314,gpt-4-0613
-
-DEBUG_PLUGINS=true # Set to false or comment out to disable debug mode for plugins
-
-# For securely storing credentials, you need a fixed key and IV. You can set them here for prod and dev environments
-# If you don't set them, the app will crash on startup.
-# You need a 32-byte key (64 characters in hex) and 16-byte IV (32 characters in hex)
-# Use this replit to generate some quickly: https://replit.com/@daavila/crypto#index.js
-# Here are some examples (THESE ARE NOT SECURE!)
-CREDS_KEY=f34be427ebb29de8d88c107a71546019685ed8b241d8f2ed00c3df97ad2566f0
-CREDS_IV=e2341419ec3dd3d19b13a1a87fafcbfb
-
-# AI-Assisted Google Search  
-# This bot supports searching google for answers to your questions with assistance from GPT! 
-# See detailed instructions here: https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md
-GOOGLE_API_KEY=
-GOOGLE_CSE_ID=
-
-# StableDiffusion WebUI
-# This bot supports StableDiffusion WebUI, using it's API to generated requested images.
-# See detailed instructions here: https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/stable_diffusion.md
-# Use "http://127.0.0.1:7860" with local install and "http://host.docker.internal:7860" for docker
-SD_WEBUI_URL=http://host.docker.internal:7860
-
-# Azure Cognitive Search
-# This plugin supports searching Azure Cognitive Search for answers to your questions.
-# See detailed instructions here: https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/azure_cognitive_search.md
-AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT=
-AZURE_COGNITIVE_SEARCH_INDEX_NAME=
-AZURE_COGNITIVE_SEARCH_API_KEY=
-
-AZURE_COGNITIVE_SEARCH_API_VERSION=
-AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE=
-AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP=
-AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT=
-
-##########################
-# PaLM (Google) Endpoint: 
-##########################
-
-# Follow the instruction here to setup:
-# https://github.com/danny-avila/LibreChat/blob/main/docs/install/apis_and_tokens.md
-
-PALM_KEY=user_provided
-
-# In case you need a reverse proxy for this endpoint:
-# GOOGLE_REVERSE_PROXY= 
-
-##########################
-# Anthropic Endpoint: 
-##########################
-# Access key from https://console.anthropic.com/
-# Leave it blank to disable this feature.
-# Set to "user_provided" to allow the user to provide their API key from the UI.
-# Note that access to claude-1 may potentially become unavailable with the release of claude-2.
-ANTHROPIC_API_KEY=user_provided
-ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2
-
-##########################
-# Proxy: To be Used by all endpoints
-##########################
-
-PROXY=
-
-##########################
-# Search: 
-##########################
-
-# ENABLING SEARCH MESSAGES/CONVOS 
-# Requires the installation of the free self-hosted Meilisearch or a paid Remote Plan (Remote not tested) 
-# The easiest setup for this is through docker-compose, which takes care of it for you. 
-SEARCH=true
-
-# HIGHLY RECOMMENDED: Disable anonymized telemetry analytics for MeiliSearch for absolute privacy.
-MEILI_NO_ANALYTICS=true
-
-# REQUIRED FOR SEARCH: MeiliSearch Host, mainly for the API server to connect to the search server. 
-# Replace '0.0.0.0' with 'meilisearch' if serving MeiliSearch with docker-compose. 
-MEILI_HOST=http://0.0.0.0:7700
-
-# REQUIRED FOR SEARCH: MeiliSearch HTTP Address, mainly for docker-compose to expose the search server. 
-# Replace '0.0.0.0' with 'meilisearch' if serving MeiliSearch with docker-compose. 
-MEILI_HTTP_ADDR=0.0.0.0:7700
-
-# REQUIRED FOR SEARCH: In production env., a secure key is needed. You can generate your own. 
-# This master key must be at least 16 bytes, composed of valid UTF-8 characters. 
-# MeiliSearch will throw an error and refuse to launch if no master key is provided, 
-# or if it is under 16 bytes. MeiliSearch will suggest a secure autogenerated master key.
-# Using docker, it seems recognized as production so use a secure key.
-# This is a ready made secure key for docker-compose, you can replace it with your own.
-MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
-
-##########################
-# User System:
-##########################
-
-# Allow Public Registration
+ALLOW_EMAIL_LOGIN=true
 ALLOW_REGISTRATION=true
-
-# Allow Social Registration
 ALLOW_SOCIAL_LOGIN=false
-
-# Allow Social Registration (WORKS ONLY for Google, Github, Discord)
 ALLOW_SOCIAL_REGISTRATION=false
 
-# JWT Secrets
-# You should use secure values. The examples given are 32-byte keys (64 characters in hex)
-# Use this replit to generate some quickly: https://replit.com/@daavila/crypto#index.js
+SESSION_EXPIRY=1000 * 60 * 15
+REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7
+
 JWT_SECRET=16f8c0ef4a5d391b26034086c628469d3f9f497f08163ab9b40137092f2909ef
 JWT_REFRESH_SECRET=eaa5191f2914e30b9387fd84e254e4ba6fc51b4654968a9b0803b456a54b8418
 
-# Google:
-# Add your Google Client ID and Secret here, you must register an app with Google Cloud to get these values
-# https://cloud.google.com/
-GOOGLE_CLIENT_ID=
-GOOGLE_CLIENT_SECRET=
-GOOGLE_CALLBACK_URL=/oauth/google/callback
+# Discord
+DISCORD_CLIENT_ID=
+DISCORD_CLIENT_SECRET=
+DISCORD_CALLBACK_URL=/oauth/discord/callback
 
-# Facebook:
-# Add your Facebook Client ID and Secret here, you must register an app with Facebook to get these values
-# https://developers.facebook.com/
+# Facebook
 FACEBOOK_CLIENT_ID=
 FACEBOOK_CLIENT_SECRET=
 FACEBOOK_CALLBACK_URL=/oauth/facebook/callback
 
-# OpenID:
-# See OpenID provider to get the below values
-# Create random string for OPENID_SESSION_SECRET
-# For Azure AD
-#   ISSUER:  https://login.microsoftonline.com/(tenant id)/v2.0/
-#   SCOPE:   openid profile email
+# GitHub
+GITHUB_CLIENT_ID=
+GITHUB_CLIENT_SECRET=
+GITHUB_CALLBACK_URL=/oauth/github/callback
+
+# Google
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
+GOOGLE_CALLBACK_URL=/oauth/google/callback
+
+# OpenID
 OPENID_CLIENT_ID=
 OPENID_CLIENT_SECRET=
 OPENID_ISSUER=
 OPENID_SESSION_SECRET=
 OPENID_SCOPE="openid profile email"
 OPENID_CALLBACK_URL=/oauth/openid/callback
-# If LABEL and URL are left empty, then the default OpenID label and logo are used.
+OPENID_REQUIRED_ROLE=
+OPENID_REQUIRED_ROLE_TOKEN_KIND=
+OPENID_REQUIRED_ROLE_PARAMETER_PATH=
+
 OPENID_BUTTON_LABEL=
 OPENID_IMAGE_URL=
 
-# Set the expiration delay for the secure cookie with the JWT token
-# Recommend session expiry to be 15 minutes
-# Delay is in millisecond e.g. 7 days is 1000*60*60*24*7
-SESSION_EXPIRY=1000 * 60 * 15
-REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7
+# LDAP
+LDAP_URL=
+LDAP_BIND_DN=
+LDAP_BIND_CREDENTIALS=
+LDAP_USER_SEARCH_BASE=
+LDAP_SEARCH_FILTER=mail={{username}}
+LDAP_CA_CERT_PATH=
 
-# Github:
-# Get the Client ID and Secret from your Discord Application
-# Add your Discord Client ID and Client Secret here:
+#========================#
+# Email Password Reset   #
+#========================#
 
-GITHUB_CLIENT_ID=your_client_id
-GITHUB_CLIENT_SECRET=your_client_secret
-GITHUB_CALLBACK_URL=/oauth/github/callback # this should be the same for everyone
+EMAIL_SERVICE=
+EMAIL_HOST=
+EMAIL_PORT=25
+EMAIL_ENCRYPTION=
+EMAIL_ENCRYPTION_HOSTNAME=
+EMAIL_ALLOW_SELFSIGNED=
+EMAIL_USERNAME=
+EMAIL_PASSWORD=
+EMAIL_FROM_NAME=
+EMAIL_FROM=noreply@librechat.ai
 
-# Discord:
-# Get the Client ID and Secret from your Discord Application
-# Add your Github Client ID and Client Secret here:
+#========================#
+# Firebase CDN           #
+#========================#
 
-DISCORD_CLIENT_ID=your_client_id
-DISCORD_CLIENT_SECRET=your_client_secret
-DISCORD_CALLBACK_URL=/oauth/discord/callback # this should be the same for everyone
+FIREBASE_API_KEY=
+FIREBASE_AUTH_DOMAIN=
+FIREBASE_PROJECT_ID=
+FIREBASE_STORAGE_BUCKET=
+FIREBASE_MESSAGING_SENDER_ID=
+FIREBASE_APP_ID=
 
-###########################
-# Application Domains
-###########################
+#===================================================#
+#                        UI                         #
+#===================================================#
 
-# Note: 
-# Server = Backend 
-# Client = Public (the client is the url you visit)
-# For the Google login to work in dev mode, you will need to change DOMAIN_SERVER to localhost:3090 or place it in .env.development
+APP_TITLE=LibreChat
+# CUSTOM_FOOTER="My custom footer"
+HELP_AND_FAQ_URL=https://librechat.ai
 
-DOMAIN_CLIENT=http://localhost:3080
-DOMAIN_SERVER=http://localhost:3080
+# SHOW_BIRTHDAY_ICON=true
 
-###########################
-# Email
-###########################
+#==================================================#
+#                      Others                      #
+#==================================================#
+#   You should leave the following commented out   #
 
-# Email is used for password reset. Note that all 4 values must be set for email to work.
-# Failing to set the 4 values will result in LibreChat using the unsecured password reset!
-EMAIL_SERVICE=   # eg. gmail
-EMAIL_USERNAME=  # eg. your email address if using gmail
-EMAIL_PASSWORD=  # eg. this is the "app password" if using gmail
-EMAIL_FROM=noreply@librechat.ai # email address for from field, it is required to set a value here even in the cases where it's not porperly working.
+# NODE_ENV=
+
+# REDIS_URI=
+# USE_REDIS=
+
+# E2E_USER_EMAIL=
+# E2E_USER_PASSWORD=

.eslintrc.js

@@ -19,6 +19,10 @@ module.exports = {
     'e2e/playwright-report/**/*',
     'packages/data-provider/types/**/*',
     'packages/data-provider/dist/**/*',
+    'packages/data-provider/test_bundle/**/*',
+    'data-node/**/*',
+    'meili_data/**/*',
+    'node_modules/**/*',
   ],
   parser: '@typescript-eslint/parser',
   parserOptions: {
@@ -61,6 +65,7 @@ module.exports = {
     'no-restricted-syntax': 'off',
     'react/prop-types': ['off'],
     'react/display-name': ['off'],
+    'no-unused-vars': ['error', { varsIgnorePattern: '^_' }],
     quotes: ['error', 'single'],
   },
   overrides: [
@@ -127,6 +132,19 @@ module.exports = {
         },
       ],
     },
+    {
+      files: './config/translations/**/*.ts',
+      parser: '@typescript-eslint/parser',
+      parserOptions: {
+        project: './config/translations/tsconfig.json',
+      },
+    },
+    {
+      files: ['./packages/data-provider/specs/**/*.ts'],
+      parserOptions: {
+        project: './packages/data-provider/tsconfig.spec.json',
+      },
+    },
   ],
   settings: {
     react: {

.github/CODE_OF_CONDUCT.md

@@ -60,7 +60,7 @@ representative at an online or offline event.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported to the community leaders responsible for enforcement here on GitHub or
-on the official [Discord Server](https://discord.gg/uDyZ5Tzhct). 
+on the official [Discord Server](https://discord.librechat.ai). 
 All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the

.github/CONTRIBUTING.md

@@ -8,7 +8,7 @@ If the feature you would like to contribute has not already received prior appro
 
 Please note that a pull request involving a feature that has not been reviewed and approved by the project maintainers may be rejected. We appreciate your understanding and cooperation.
 
-If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.gg/uDyZ5Tzhct), where you can engage with other contributors and seek guidance from the community.
+If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.librechat.ai), where you can engage with other contributors and seek guidance from the community.
 
 ## Our Standards
 

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -7,14 +7,6 @@ body:
     attributes:
       value: |
         Thanks for taking the time to fill out this bug report!
-  - type: input
-    id: contact
-    attributes:
-      label: Contact Details
-      description: How can we get in touch with you if we need more info?
-      placeholder: ex. email@example.com
-    validations:
-      required: false
   - type: textarea
     id: what-happened
     attributes:
@@ -58,7 +50,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -7,14 +7,6 @@ body:
     attributes:
       value: |
         Thank you for taking the time to fill this out!
-  - type: input
-    id: contact
-    attributes:
-      label: Contact Details
-      description: How can we contact you if we need more information?
-      placeholder: ex. email@example.com
-    validations:
-      required: false
   - type: textarea
     id: what
     attributes:
@@ -51,7 +43,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/ISSUE_TEMPLATE/QUESTION.yml

@@ -7,14 +7,6 @@ body:
     attributes:
       value: |
         Thanks for taking the time to fill this!
-  - type: input
-    id: contact
-    attributes:
-      label: Contact Details
-      description: How can we get in touch with you if we need more info?
-      placeholder: ex. email@example.com
-    validations:
-      required: false
   - type: textarea
     id: what-is-your-question
     attributes:
@@ -52,7 +44,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/SECURITY.md

@@ -12,7 +12,7 @@ When reporting a security vulnerability, you have the following options to reach
 
 - **Option 2: GitHub Issues**: You can initiate first contact via GitHub Issues. However, please note that initial contact through GitHub Issues should not include any sensitive details.
 
-- **Option 3: Discord Server**: You can join our [Discord community](https://discord.gg/5rbRxn4uME) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
+- **Option 3: Discord Server**: You can join our [Discord community](https://discord.librechat.ai) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
 
 _After the initial contact, we will establish a private communication channel for further discussion._
 
@@ -39,11 +39,11 @@ Please note that as a security-conscious community, we may not always disclose d
 
 This security policy applies to the following GitHub repository:
 
-- Repository: [LibreChat](https://github.com/danny-avila/LibreChat)
+- Repository: [LibreChat](https://github.librechat.ai)
 
 ## Contact
 
-If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.gg/NGaa9RPCft) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
+If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.librechat.ai) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
 
 ## Acknowledgments
 

.github/playwright.yml

@@ -0,0 +1,72 @@
+# name: Playwright Tests
+# on:
+#   pull_request:
+#     branches: 
+#       - main
+#       - dev
+#       - release/*
+#     paths:
+#       - 'api/**'
+#       - 'client/**'
+#       - 'packages/**'
+#       - 'e2e/**'
+# jobs:
+#   tests_e2e:
+#     name: Run Playwright tests
+#     if: github.event.pull_request.head.repo.full_name == 'danny-avila/LibreChat'
+#     timeout-minutes: 60
+#     runs-on: ubuntu-latest
+#     env:
+#       NODE_ENV: CI
+#       CI: true
+#       SEARCH: false
+#       BINGAI_TOKEN: user_provided
+#       CHATGPT_TOKEN: user_provided
+#       MONGO_URI: ${{ secrets.MONGO_URI }}
+#       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+#       E2E_USER_EMAIL: ${{ secrets.E2E_USER_EMAIL }}
+#       E2E_USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
+#       JWT_SECRET: ${{ secrets.JWT_SECRET }}
+#       JWT_REFRESH_SECRET: ${{ secrets.JWT_REFRESH_SECRET }}
+#       CREDS_KEY: ${{ secrets.CREDS_KEY }}
+#       CREDS_IV: ${{ secrets.CREDS_IV }}
+#       DOMAIN_CLIENT: ${{ secrets.DOMAIN_CLIENT }}
+#       DOMAIN_SERVER: ${{ secrets.DOMAIN_SERVER }}
+#       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 # Skip downloading during npm install
+#       PLAYWRIGHT_BROWSERS_PATH: 0 # Places binaries to node_modules/@playwright/test
+#       TITLE_CONVO: false
+#     steps:
+#       - uses: actions/checkout@v4
+#       - uses: actions/setup-node@v4
+#         with:
+#           node-version: 18
+#           cache: 'npm'
+
+#       - name: Install global dependencies
+#         run: npm ci
+
+#       # - name: Remove sharp dependency
+#       #   run: rm -rf node_modules/sharp
+
+#       # - name: Install sharp with linux dependencies
+#       #   run: cd api && SHARP_IGNORE_GLOBAL_LIBVIPS=1 npm install --arch=x64 --platform=linux --libc=glibc sharp
+
+#       - name: Build Client
+#         run: npm run frontend
+
+#       - name: Install Playwright
+#         run: |
+#           npx playwright install-deps
+#           npm install -D @playwright/test@latest
+#           npx playwright install chromium
+
+#       - name: Run Playwright tests
+#         run: npm run e2e:ci
+
+#       - name: Upload playwright report
+#         uses: actions/upload-artifact@v3
+#         if: always()
+#         with:
+#           name: playwright-report
+#           path: e2e/playwright-report/
+#           retention-days: 30

.github/pull_request_template.md

@@ -1,7 +1,10 @@
 # Pull Request Template
 
+⚠️ Before Submitting a PR, Please Review:
+- Please ensure that you have thoroughly read and understood the [Contributing Docs](https://github.com/danny-avila/LibreChat/blob/main/.github/CONTRIBUTING.md) before submitting your Pull Request.
 
-### ⚠️ Before Submitting a PR, read the [Contributing Docs](./CONTRIBUTING.md) in full!
+⚠️ Documentation Updates Notice:
+- Kindly note that documentation updates are managed in this repository: [librechat.ai](https://github.com/LibreChat-AI/librechat.ai)
 
 ## Summary
 
@@ -15,7 +18,7 @@ Please delete any irrelevant options.
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] This change requires a documentation update
-- [ ] Documentation update 
+- [ ] Translation update
 
 ## Testing
 
@@ -25,6 +28,8 @@ Please describe your test process and include instructions so that we can reprod
 
 ## Checklist
 
+Please delete any irrelevant options.
+
 - [ ] My code adheres to this project's style guidelines
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented in any complex areas of my code
@@ -33,3 +38,4 @@ Please describe your test process and include instructions so that we can reprod
 - [ ] I have written tests demonstrating that my changes are effective or that my feature works
 - [ ] Local unit tests pass with my changes
 - [ ] Any changes dependent on mine have been merged and published in downstream modules.
+- [ ] A pull request for updating the documentation has been submitted.

.github/workflows/backend-review.yml

@@ -23,9 +23,9 @@ jobs:
       BAN_INTERVAL: ${{ secrets.BAN_INTERVAL }}
       NODE_ENV: CI
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Use Node.js 20.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
@@ -35,11 +35,32 @@ jobs:
 
       - name: Install Data Provider
         run: npm run build:data-provider
+      
+      - name: Create empty auth.json file
+        run: |
+          mkdir -p api/data
+          echo '{}' > api/data/auth.json
+
+      - name: Check for Circular dependency in rollup
+        working-directory: ./packages/data-provider
+        run: |
+          output=$(npm run rollup:api)
+          echo "$output"
+          if echo "$output" | grep -q "Circular dependency"; then
+            echo "Error: Circular dependency detected!"
+            exit 1
+          fi
+
+      - name: Prepare .env.test file
+        run: cp api/test/.env.test.example api/test/.env.test
 
       - name: Run unit tests
         run: cd api && npm run test:ci
 
+      - name: Run librechat-data-provider unit tests
+        run: cd packages/data-provider && npm run test:ci
+
       - name: Run linters
         uses: wearerequired/lint-action@v2
         with:
-          eslint: true
+          eslint: true

.github/workflows/container.yml

@@ -1,52 +0,0 @@
-name: Docker Compose Build on Tag
-
-# The workflow is triggered when a tag is pushed
-on:
-  push:
-    tags:
-      - "*"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v1
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Run docker-compose build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-          docker-compose build
-          docker build -f Dockerfile.multi --target api-build -t librechat-api .
-
-      # Get Tag Name
-      - name: Get Tag Name
-        id: tag_name
-        run: echo "TAG_NAME=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
-
-      # Tag it properly before push to github
-      - name: tag image and push
-        run: |
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:latest

.github/workflows/data-provider.yml

@@ -11,8 +11,8 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: 16
       - run: cd packages/data-provider && npm ci
@@ -22,8 +22,8 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: 16
           registry-url: 'https://registry.npmjs.org'

.github/workflows/deploy.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # checkout the repo
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@main
+        uses: actions/checkout@v4
 
       - name: 'Login via Azure CLI'
         uses: azure/login@v1

.github/workflows/dev-images.yml

@@ -8,19 +8,33 @@ on:
     paths:
       - 'api/**'
       - 'client/**'
+      - 'packages/**'
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-dev-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat-dev
 
     steps:
       # Check out the repository
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v1
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       # Log in to GitHub Container Registry
       - name: Log in to GitHub Container Registry
@@ -30,22 +44,29 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Build Docker images
-      - name: Build Docker images
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
         run: |
           cp .env.example .env
-          docker build -f Dockerfile.multi --target api-build -t librechat-dev-api .
-          docker build -f Dockerfile -t librechat-dev .
 
-      # Tag and push the images to GitHub Container Registry
-      - name: Tag and push images
-        run: |
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.sha }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.sha }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/frontend-review.yml

@@ -1,11 +1,6 @@
-#github action to run unit tests for frontend with jest
 name: Frontend Unit Tests
+
 on:
-  # push:
-  #   branches: 
-  #     - main
-  #     - dev
-  #     - release/*
   pull_request:
     branches: 
       - main
@@ -14,15 +9,16 @@ on:
     paths:
       - 'client/**'
       - 'packages/**'
+
 jobs:
-  tests_frontend:
-    name: Run frontend unit tests
+  tests_frontend_ubuntu:
+    name: Run frontend unit tests on Ubuntu
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Use Node.js 20.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
@@ -35,4 +31,26 @@ jobs:
 
       - name: Run unit tests
         run: npm run test:ci --verbose
-        working-directory: client
+        working-directory: client
+
+  tests_frontend_windows:
+    name: Run frontend unit tests on Windows
+    timeout-minutes: 60
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build Client
+        run: npm run frontend:ci
+
+      - name: Run unit tests
+        run: npm run test:ci --verbose
+        working-directory: client

.github/workflows/generate_embeddings.yml

@@ -0,0 +1,20 @@
+name: 'generate_embeddings'
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: supabase/embeddings-generator@v0.0.5
+        with:
+          supabase-url: ${{ secrets.SUPABASE_URL }}
+          supabase-service-role-key: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
+          openai-key: ${{ secrets.OPENAI_DOC_EMBEDDINGS_KEY }}
+          docs-root-path: 'docs'

.github/workflows/main-image-workflow.yml

@@ -0,0 +1,69 @@
+name: Docker Compose Build Latest Main Image Tag (Manual Dispatch)
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Fetch tags and set the latest tag
+        run: |
+          git fetch --tags
+          echo "LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)" >> $GITHUB_ENV
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/mkdocs.yaml

@@ -1,24 +0,0 @@
-name: mkdocs
-on:
-  push:
-    branches:
-      - main
-permissions:
-  contents: write
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.x
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
-      - uses: actions/cache@v3
-        with:
-          key: mkdocs-material-${{ env.cache_id }}
-          path: .cache
-          restore-keys: |
-            mkdocs-material-
-      - run: pip install mkdocs-material 
-      - run: mkdocs gh-deploy --force

.github/workflows/playwright.yml

@@ -1,72 +0,0 @@
-name: Playwright Tests
-on:
-  pull_request:
-    branches: 
-      - main
-      - dev
-      - release/*
-    paths:
-      - 'api/**'
-      - 'client/**'
-      - 'packages/**'
-      - 'e2e/**'
-jobs:
-  tests_e2e:
-    name: Run Playwright tests
-    if: github.event.pull_request.head.repo.full_name == 'danny-avila/LibreChat'
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    env:
-      NODE_ENV: CI
-      CI: true
-      SEARCH: false
-      BINGAI_TOKEN: user_provided
-      CHATGPT_TOKEN: user_provided
-      MONGO_URI: ${{ secrets.MONGO_URI }}
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      E2E_USER_EMAIL: ${{ secrets.E2E_USER_EMAIL }}
-      E2E_USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
-      JWT_SECRET: ${{ secrets.JWT_SECRET }}
-      JWT_REFRESH_SECRET: ${{ secrets.JWT_REFRESH_SECRET }}
-      CREDS_KEY: ${{ secrets.CREDS_KEY }}
-      CREDS_IV: ${{ secrets.CREDS_IV }}
-      DOMAIN_CLIENT: ${{ secrets.DOMAIN_CLIENT }}
-      DOMAIN_SERVER: ${{ secrets.DOMAIN_SERVER }}
-      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 # Skip downloading during npm install
-      PLAYWRIGHT_BROWSERS_PATH: 0 # Places binaries to node_modules/@playwright/test
-      TITLE_CONVO: false
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18
-          cache: 'npm'
-
-      - name: Install global dependencies
-        run: npm ci
-
-      # - name: Remove sharp dependency
-      #   run: rm -rf node_modules/sharp
-
-      # - name: Install sharp with linux dependencies
-      #   run: cd api && SHARP_IGNORE_GLOBAL_LIBVIPS=1 npm install --arch=x64 --platform=linux --libc=glibc sharp
-
-      - name: Build Client
-        run: npm run frontend
-
-      - name: Install Playwright
-        run: |
-          npx playwright install-deps
-          npm install -D @playwright/test@latest
-          npx playwright install chromium
-
-      - name: Run Playwright tests
-        run: npm run e2e:ci
-
-      - name: Upload playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: e2e/playwright-report/
-          retention-days: 30

.github/workflows/tag-images.yml

@@ -0,0 +1,67 @@
+name: Docker Images Build on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      # Check out the repository
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.gitignore

@@ -2,7 +2,7 @@
 
 # Logs
 data-node
-meili_data
+meili_data*
 data/
 logs
 *.log
@@ -21,6 +21,10 @@ coverage
 # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
 .grunt
 
+# translation services
+config/translations/stores/*
+client/src/localization/languages/*_missing_keys.json
+
 # Compiled Dirs (http://nodejs.org/api/addons.html)
 build/
 dist/
@@ -40,7 +44,7 @@ meili_data/
 api/node_modules/
 client/node_modules/
 bower_components/
-types/
+*.d.ts
 
 # Floobits
 .floo
@@ -48,6 +52,10 @@ types/
 .floo
 .flooignore
 
+#config file
+librechat.yaml
+librechat.yml
+
 # Environment
 .npmrc
 .env*
@@ -65,11 +73,20 @@ src/style - official.css
 /playwright/.cache/
 .DS_Store
 *.code-workspace
+.idx
+monospace.json
 .idea
+*.iml
 *.pem
 config.local.ts
 **/storageState.json
 junit.xml
+**/.venv/
+**/venv/
+
+# docker override file
+docker-compose.override.yaml
+docker-compose.override.yml
 
 # meilisearch
 meilisearch
@@ -78,4 +95,12 @@ data.ms/*
 auth.json
 
 /packages/ux-shared/
-/images
+/images
+
+!client/src/components/Nav/SettingsTabs/Data/
+
+# User uploads
+uploads/
+
+# owner
+release/

.husky/pre-commit

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env sh 
 set -e
 . "$(dirname -- "$0")/_/husky.sh"
 [ -n "$CI" ] && exit 0

Dockerfile

@@ -1,21 +1,32 @@
-# Base node image
-FROM node:19-alpine AS node
+# v0.7.2
 
-COPY . /app
+# Base node image
+FROM node:20-alpine AS node
+
+RUN apk --no-cache add curl
+
+RUN mkdir -p /app && chown node:node /app
 WORKDIR /app
 
-# Install call deps - Install curl for health check
-RUN apk --no-cache add curl && \
-    # We want to inherit env from the container, not the file
-    # This will preserve any existing env file if it's already in souce
-    # otherwise it will create a new one
-    touch .env && \
-    # Build deps in seperate 
-    npm ci
+USER node
 
-# React client build
-ENV NODE_OPTIONS="--max-old-space-size=2048"
-RUN npm run frontend
+COPY --chown=node:node . .
+
+RUN \
+    # Allow mounting of these files, which have no default
+    touch .env ; \
+    # Create directories for the volumes to inherit the correct permissions
+    mkdir -p /app/client/public/images /app/api/logs ; \
+    npm config set fetch-retry-maxtimeout 600000 ; \
+    npm config set fetch-retries 5 ; \
+    npm config set fetch-retry-mintimeout 15000 ; \
+    npm install --no-audit; \
+    # React client build
+    NODE_OPTIONS="--max-old-space-size=2048" npm run frontend; \
+    npm prune --production; \
+    npm cache clean --force
+
+RUN mkdir -p /app/client/public/images /app/api/logs
 
 # Node API setup
 EXPOSE 3080

Dockerfile.multi

@@ -1,34 +1,37 @@
+# v0.7.2
+
 # Build API, Client and Data Provider
-FROM node:19-alpine AS base
+FROM node:20-alpine AS base
 
-WORKDIR /app
-COPY config/loader.js ./config/
-RUN npm install dotenv
-
-WORKDIR /app/api
-COPY api/package*.json ./
-COPY api/ ./
-RUN npm install
+# Build data-provider
+FROM base AS data-provider-build
+WORKDIR /app/packages/data-provider
+COPY ./packages/data-provider ./
+RUN npm install; npm cache clean --force
+RUN npm run build
+RUN npm prune --production
 
 # React client build
 FROM base AS client-build
 WORKDIR /app/client
+COPY ./client/package*.json ./
+# Copy data-provider to client's node_modules
+COPY --from=data-provider-build /app/packages/data-provider/ /app/client/node_modules/librechat-data-provider/
+RUN npm install; npm cache clean --force
 COPY ./client/ ./
-
-WORKDIR /app/packages/data-provider
-COPY ./packages/data-provider ./
-RUN npm install
-RUN npm run build
-RUN mkdir -p /app/client/node_modules/librechat-data-provider/
-RUN cp -R /app/packages/data-provider/* /app/client/node_modules/librechat-data-provider/
-
-WORKDIR /app/client
-RUN npm install
 ENV NODE_OPTIONS="--max-old-space-size=2048"
 RUN npm run build
 
 # Node API setup
 FROM base AS api-build
+WORKDIR /app/api
+COPY api/package*.json ./
+COPY api/ ./
+# Copy helper scripts
+COPY config/ ./
+# Copy data-provider to API's node_modules
+COPY --from=data-provider-build /app/packages/data-provider/ /app/api/node_modules/librechat-data-provider/
+RUN npm install --include prod; npm cache clean --force
 COPY --from=client-build /app/client/dist /app/client/dist
 EXPOSE 3080
 ENV HOST=0.0.0.0

LICENSE

@@ -1,8 +1,6 @@
-# MIT License
+MIT License
 
-Copyright (c) 2023 Danny Avila
-
----
+Copyright (c) 2024 LibreChat
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -14,8 +12,6 @@ furnished to do so, subject to the following conditions:
 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 
-##
-
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -23,7 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-
----
-
-## [Go Back to ReadMe](../README.md)

README.md

@@ -1,14 +1,14 @@
 <p align="center">
-  <a href="https://docs.librechat.ai">
-    <img src="docs/assets/LibreChat.svg" height="256">
-  </a>
-  <a href="https://docs.librechat.ai">
-    <h1 align="center">LibreChat</h1>
+  <a href="https://librechat.ai">
+    <img src="client/public/assets/logo.svg" height="256">
   </a>
+  <h1 align="center">
+    <a href="https://librechat.ai">LibreChat</a>
+  </h1>
 </p>
 
 <p align="center">
-  <a href="https://discord.gg/NGaa9RPCft"> 
+  <a href="https://discord.librechat.ai"> 
     <img
       src="https://img.shields.io/discord/1086345563026489514?label=&logo=discord&style=for-the-badge&logoWidth=20&logoColor=white&labelColor=000000&color=blueviolet">
   </a>
@@ -20,145 +20,119 @@
     <img
       src="https://img.shields.io/badge/DOCS-blue.svg?style=for-the-badge&logo=read-the-docs&logoColor=white&labelColor=000000&logoWidth=20">
   </a>
-  <a aria-label="Sponsors" href="#sponsors">
+  <a aria-label="Sponsors" href="https://github.com/sponsors/danny-avila">
     <img
       src="https://img.shields.io/badge/SPONSORS-brightgreen.svg?style=for-the-badge&logo=github-sponsors&logoColor=white&labelColor=000000&logoWidth=20">
   </a>
 </p>
 
-## All-In-One AI Conversations with LibreChat ##
+<p align="center">
+<a href="https://railway.app/template/b5k2mn?referralCode=myKrVZ">
+  <img src="https://railway.app/button.svg" alt="Deploy on Railway" height="30">
+</a>
+<a href="https://zeabur.com/templates/0X2ZY8">
+  <img src="https://zeabur.com/button.svg" alt="Deploy on Zeabur" height="30"/>
+</a>
+<a href="https://template.cloud.sealos.io/deploy?templateName=librechat">
+  <img src="https://raw.githubusercontent.com/labring-actions/templates/main/Deploy-on-Sealos.svg" alt="Deploy on Sealos" height="30">
+</a>
+</p>
+
+# 📃 Features
+
+- 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and latest updates
+- 🤖 AI model selection:
+  - OpenAI, Azure OpenAI, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins, Assistants API (including Azure Assistants)
+- ✅ Compatible across both **[Remote & Local AI services](https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints):**
+  - groq, Ollama, Cohere, Mistral AI, Apple MLX, koboldcpp, OpenRouter, together.ai, Perplexity, ShuttleAI, and more
+- 💾 Create, Save, & Share Custom Presets
+- 🔀 Switch between AI Endpoints and Presets, mid-chat
+- 🔄 Edit, Resubmit, and Continue Messages with Conversation branching
+- 🌿 Fork Messages & Conversations for Advanced Context control
+- 💬 Multimodal Chat:
+    - Upload and analyze images with Claude 3, GPT-4 (including `gpt-4o`), and Gemini Vision 📸
+    - Chat with Files using Custom Endpoints, OpenAI, Azure, Anthropic, & Google. 🗃️
+    - Advanced Agents with Files, Code Interpreter, Tools, and API Actions 🔦
+      - Available through the [OpenAI Assistants API](https://platform.openai.com/docs/assistants/overview) 🌤️
+      - Non-OpenAI Agents in Active Development 🚧
+- 🌎 Multilingual UI:
+  - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
+  - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands, עברית
+- 🎨 Customizable Dropdown & Interface: Adapts to both power users and newcomers.
+- 📥 Import Conversations from LibreChat, ChatGPT, Chatbot UI
+- 📤 Export conversations as screenshots, markdown, text, json.
+- 🔍 Search all messages/conversations
+- 🔌 Plugins, including web access, image generation with DALL-E-3 and more
+- 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
+- ⚙️ Configure Proxy, Reverse Proxy, Docker, & many Deployment options:
+  - Use completely local or deploy on the cloud
+- 📖 Completely Open-Source & Built in Public
+- 🧑‍🤝‍🧑 Community-driven development, support, and feedback
+
+[For a thorough review of our features, see our docs here](https://docs.librechat.ai/) 📚
+
+## 🪶 All-In-One AI Conversations with LibreChat
+
 LibreChat brings together the future of assistant AIs with the revolutionary technology of OpenAI's ChatGPT. Celebrating the original styling, LibreChat gives you the ability to integrate multiple AI models. It also integrates and enhances original client features such as conversation and message search, prompt templates and plugins.
 
 With LibreChat, you no longer need to opt for ChatGPT Plus and can instead use free or pay-per-call APIs. We welcome contributions, cloning, and forking to enhance the capabilities of this advanced chatbot platform.
-  
-<!-- https://github.com/danny-avila/LibreChat/assets/110412045/c1eb0c0f-41f6-4335-b982-84b278b53d59 -->
 
-[![Watch the video](https://img.youtube.com/vi/pNIOs1ovsXw/maxresdefault.jpg)](https://youtu.be/pNIOs1ovsXw)
+[![Watch the video](https://img.youtube.com/vi/YLVUW5UP9N0/maxresdefault.jpg)](https://www.youtube.com/watch?v=YLVUW5UP9N0)
 Click on the thumbnail to open the video☝️
 
-# Features
-- Response streaming identical to ChatGPT through server-sent events
-- UI from original ChatGPT, including Dark mode
-- AI model selection: OpenAI API, BingAI, ChatGPT Browser, PaLM2, Anthropic (Claude), Plugins
-- Create, Save, & Share custom presets - [More info on prompt presets here](https://github.com/danny-avila/LibreChat/releases/tag/v0.3.0)
-- Edit and Resubmit messages with conversation branching
-- Search all messages/conversations - [More info here](https://github.com/danny-avila/LibreChat/releases/tag/v0.1.0)
-- Plugins now available (including web access, image generation and more)
+---
+
+## 🌐 Resources
+
+**GitHub Repo:**
+  - **RAG API:** [github.com/danny-avila/rag_api](https://github.com/danny-avila/rag_api)
+  - **Website:** [github.com/LibreChat-AI/librechat.ai](https://github.com/LibreChat-AI/librechat.ai)
+
+**Other:**
+  - **Website:** [librechat.ai](https://librechat.ai)
+  - **Documentation:** [docs.librechat.ai](https://docs.librechat.ai)
+  - **Blog:** [blog.librechat.ai](https://docs.librechat.ai)
 
 ---
 
-## ⚠️ [Breaking Changes](docs/general_info/breaking_changes.md) ⚠️
+## 📝 Changelog
 
-**Please read this before updating from a previous version**
+Keep up with the latest updates by visiting the releases page and notes:
+- [Releases](https://github.com/danny-avila/LibreChat/releases)
+- [Changelog](https://www.librechat.ai/changelog) 
+
+**⚠️ Please consult the [changelog](https://www.librechat.ai/changelog) for breaking changes before updating.**
 
 ---
 
-## Changelog 
-Keep up with the latest updates by visiting the releases page - [Releases](https://github.com/danny-avila/LibreChat/releases)
+## ⭐ Star History
+
+<p align="center">
+  <a href="https://star-history.com/#danny-avila/LibreChat&Date">
+    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
+  </a>
+</p>
+<p align="center">
+  <a href="https://trendshift.io/repositories/4685" target="_blank" style="padding: 10px;">
+    <img src="https://trendshift.io/api/badge/repositories/4685" alt="danny-avila%2FLibreChat | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
+  </a>
+  <a href="https://runacap.com/ross-index/q1-24/" target="_blank" rel="noopener" style="margin-left: 20px;">
+    <img style="width: 260px; height: 56px" src="https://runacap.com/wp-content/uploads/2024/04/ROSS_badge_white_Q1_2024.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q1 2024 | Runa Capital" width="260" height="56"/>
+  </a>
+</p>
 
 ---
 
-<h1>Table of Contents</h1>
+## ✨ Contributions
 
-<details open>
-  <summary><strong>Getting Started</strong></summary>
-
-  * Installation
-    * [Docker Compose Install🐳](docs/install/docker_compose_install.md)
-    * [Linux Install🐧](docs/install/linux_install.md)
-    * [Mac Install🍎](docs/install/mac_install.md)
-    * [Windows Install💙](docs/install/windows_install.md)
-  * Configuration
-    * [APIs and Tokens](docs/install/apis_and_tokens.md)
-    * [User Auth System](docs/install/user_auth_system.md)
-    * [Online MongoDB Database](docs/install/mongodb.md)
-    * [Default Language](docs/install/default_language.md)
-</details>
-
-<details>
-  <summary><strong>General Information</strong></summary>
-
-  * [Code of Conduct](.github/CODE_OF_CONDUCT.md)
-  * [Project Origin](docs/general_info/project_origin.md)
-  * [Multilingual Information](docs/general_info/multilingual_information.md)
-  * [Tech Stack](docs/general_info/tech_stack.md)   
-</details>
-
-<details>
-  <summary><strong>Features</strong></summary>
-
-  * **Plugins**
-    * [Introduction](docs/features/plugins/introduction.md)
-    * [Google](docs/features/plugins/google_search.md)
-    * [Stable Diffusion](docs/features/plugins/stable_diffusion.md)
-    * [Wolfram](docs/features/plugins/wolfram.md)
-    * [Make Your Own Plugin](docs/features/plugins/make_your_own.md)
-    * [Using official ChatGPT Plugins](docs/features/plugins/chatgpt_plugins_openapi.md)
-
-  
-  * [Automated Moderation](docs/features/mod_system.md)
-  * [Third-Party Tools](docs/features/third_party.md)
-  * [Proxy](docs/features/proxy.md)
-  * [Bing Jailbreak](docs/features/bing_jailbreak.md)
-  * [Token Usage](docs/features/token_usage.md)
-</details>
-
-<details>
-  <summary><strong>Cloud Deployment</strong></summary>
-
-  * [DigitalOcean](docs/deployment/digitalocean.md)
-  * [Azure](docs/deployment/azure-terraform.md)
-  * [Linode](docs/deployment/linode.md)
-  * [Cloudflare](docs/deployment/cloudflare.md)
-  * [Ngrok](docs/deployment/ngrok.md)
-  * [HuggingFace](docs/deployment/huggingface.md)
-  * [Render](docs/deployment/render.md)
-  * [Meilisearch in Render](docs/deployment/meilisearch_in_render.md)
-  * [Hetzner](docs/deployment/hetzner_ubuntu.md)
-  * [Heroku](docs/deployment/heroku.md)
-</details>
-
-<details>
-  <summary><strong>Contributions</strong></summary>
-  
-  * [Contributor Guidelines](.github/CONTRIBUTING.md)
-  * [Documentation Guidelines](docs/contributions/documentation_guidelines.md)
-  * [Contribute a Translation](docs/contributions/translation_contribution.md)
-  * [Code Standards and Conventions](docs/contributions/coding_conventions.md)
-  * [Testing](docs/contributions/testing.md)
-  * [Security](.github/SECURITY.md)
-  * [Project Roadmap](https://github.com/users/danny-avila/projects/2)
-</details>
+Contributions, suggestions, bug reports and fixes are welcome!
 
+For new features, components, or extensions, please open an issue and discuss before sending a PR.
 
 ---
 
-## Star History
+## 💖 This project exists in its current state thanks to all the people who contribute
 
-<a href="https://star-history.com/#danny-avila/LibreChat&Date">
-  <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
-</a>
-
----
-
-## Sponsors
-
-  Sponsored by <a href="https://github.com/mjtechguy"><b>@mjtechguy</b></a>, <a href="https://github.com/SphaeroX"><b>@SphaeroX</b></a>, <a href="https://github.com/DavidDev1334"><b>@DavidDev1334</b></a>, <a href="https://github.com/fuegovic"><b>@fuegovic</b></a>, <a href="https://github.com/Pharrcyde"><b>@Pharrcyde</b></a> 
-  
----
-
-## Contributors
-Contributions and suggestions bug reports and fixes are welcome!
-Please read the documentation before you do!
-
----
-
-For new features, components, or extensions, please open an issue and discuss before sending a PR. 
-
-- Join the [Discord community](https://discord.gg/uDyZ5Tzhct)
-
-This project exists in its current state thanks to all the people who contribute
----
 <a href="https://github.com/danny-avila/LibreChat/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=danny-avila/LibreChat" />
 </a>

api/app/bingai.js

@@ -1,6 +1,8 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
-const { getUserKey, checkUserKeyExpiry } = require('../server/services/UserService');
+const { EModelEndpoint } = require('librechat-data-provider');
+const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
+const { logger } = require('~/config');
 
 const askBing = async ({
   text,
@@ -22,10 +24,7 @@ const askBing = async ({
 
   let key = null;
   if (expiresAt && isUserProvided) {
-    checkUserKeyExpiry(
-      expiresAt,
-      'Your BingAI Cookies have expired. Please provide your cookies again.',
-    );
+    checkUserKeyExpiry(expiresAt, EModelEndpoint.bingAI);
     key = await getUserKey({ userId, name: 'bingAI' });
   }
 
@@ -100,7 +99,7 @@ const askBing = async ({
     }
   }
 
-  console.log('bing options', options);
+  logger.debug('bing options', options);
 
   const res = await bingAIClient.sendMessage(text, options);
 

api/app/chatgpt-browser.js

@@ -1,5 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
+const { Constants, EModelEndpoint } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('../server/services/UserService');
 
 const browserClient = async ({
@@ -17,10 +18,7 @@ const browserClient = async ({
 
   let key = null;
   if (expiresAt && isUserProvided) {
-    checkUserKeyExpiry(
-      expiresAt,
-      'Your ChatGPT Access Token has expired. Please provide your token again.',
-    );
+    checkUserKeyExpiry(expiresAt, EModelEndpoint.chatGPTBrowser);
     key = await getUserKey({ userId, name: 'chatGPTBrowser' });
   }
 
@@ -48,7 +46,7 @@ const browserClient = async ({
     options = { ...options, parentMessageId, conversationId };
   }
 
-  if (parentMessageId === '00000000-0000-0000-0000-000000000000') {
+  if (parentMessageId === Constants.NO_PARENT) {
     delete options.conversationId;
   }
 

api/app/clients/AnthropicClient.js

@@ -1,20 +1,43 @@
-// const { Agent, ProxyAgent } = require('undici');
-const BaseClient = require('./BaseClient');
-const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const Anthropic = require('@anthropic-ai/sdk');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
+const {
+  getResponseSender,
+  EModelEndpoint,
+  validateVisionModel,
+} = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images/encode');
+const {
+  truncateText,
+  formatMessage,
+  titleFunctionPrompt,
+  parseParamFromPrompt,
+  createContextHandlers,
+} = require('./prompts');
+const spendTokens = require('~/models/spendTokens');
+const { getModelMaxTokens } = require('~/utils');
+const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
 
 const HUMAN_PROMPT = '\n\nHuman:';
 const AI_PROMPT = '\n\nAssistant:';
 
 const tokenizersCache = {};
 
+/** Helper function to introduce a delay before retrying */
+function delayBeforeRetry(attempts, baseDelay = 1000) {
+  return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts));
+}
+
 class AnthropicClient extends BaseClient {
-  constructor(apiKey, options = {}, cacheOptions = {}) {
-    super(apiKey, options, cacheOptions);
+  constructor(apiKey, options = {}) {
+    super(apiKey, options);
     this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY;
-    this.sender = 'Anthropic';
     this.userLabel = HUMAN_PROMPT;
     this.assistantLabel = AI_PROMPT;
+    this.contextStrategy = options.contextStrategy
+      ? options.contextStrategy.toLowerCase()
+      : 'discard';
     this.setOptions(options);
   }
 
@@ -40,13 +63,22 @@ class AnthropicClient extends BaseClient {
       ...modelOptions,
       // set some good defaults (check for undefined in some cases because they may be 0)
       model: modelOptions.model || 'claude-1',
-      temperature: typeof modelOptions.temperature === 'undefined' ? 0.7 : modelOptions.temperature, // 0 - 1, 0.7 is recommended
+      temperature: typeof modelOptions.temperature === 'undefined' ? 1 : modelOptions.temperature, // 0 - 1, 1 is default
       topP: typeof modelOptions.topP === 'undefined' ? 0.7 : modelOptions.topP, // 0 - 1, default: 0.7
       topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40
       stop: modelOptions.stop, // no stop method for now
     };
 
-    this.maxContextTokens = this.options.maxContextTokens || 99999;
+    this.isClaude3 = this.modelOptions.model.includes('claude-3');
+    this.useMessages = this.isClaude3 || !!this.options.attachments;
+
+    this.defaultVisionModel = this.options.visionModel ?? 'claude-3-sonnet-20240229';
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
+
+    this.maxContextTokens =
+      this.options.maxContextTokens ??
+      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ??
+      100000;
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500;
     this.maxPromptTokens =
       this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens;
@@ -59,6 +91,14 @@ class AnthropicClient extends BaseClient {
       );
     }
 
+    this.sender =
+      this.options.sender ??
+      getResponseSender({
+        model: this.modelOptions.model,
+        endpoint: EModelEndpoint.anthropic,
+        modelLabel: this.options.modelLabel,
+      });
+
     this.startToken = '||>';
     this.endToken = '';
     this.gptEncoder = this.constructor.getTokenizer('cl100k_base');
@@ -77,17 +117,95 @@ class AnthropicClient extends BaseClient {
     return this;
   }
 
+  /**
+   * Get the initialized Anthropic client.
+   * @returns {Anthropic} The Anthropic client instance.
+   */
   getClient() {
-    if (this.options.reverseProxyUrl) {
-      return new Anthropic({
-        apiKey: this.apiKey,
-        baseURL: this.options.reverseProxyUrl,
-      });
-    } else {
-      return new Anthropic({
-        apiKey: this.apiKey,
-      });
+    /** @type {Anthropic.default.RequestOptions} */
+    const options = {
+      fetch: this.fetch,
+      apiKey: this.apiKey,
+    };
+
+    if (this.options.proxy) {
+      options.httpAgent = new HttpsProxyAgent(this.options.proxy);
     }
+
+    if (this.options.reverseProxyUrl) {
+      options.baseURL = this.options.reverseProxyUrl;
+    }
+
+    return new Anthropic(options);
+  }
+
+  getTokenCountForResponse(response) {
+    return this.getTokenCountForMessage({
+      role: 'assistant',
+      content: response.text,
+    });
+  }
+
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
+   * - Sets `this.isVisionModel` to `true` if vision request.
+   * - Deletes `this.modelOptions.stop` if vision request.
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.anthropic];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    const visionModelAvailable = availableModels?.includes(this.defaultVisionModel);
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      visionModelAvailable &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+  }
+
+  /**
+   * Calculate the token cost in tokens for an image based on its dimensions and detail level.
+   *
+   * For reference, see: https://docs.anthropic.com/claude/docs/vision#image-costs
+   *
+   * @param {Object} image - The image object.
+   * @param {number} image.width - The width of the image.
+   * @param {number} image.height - The height of the image.
+   * @returns {number} The calculated token cost measured by tokens.
+   *
+   */
+  calculateImageTokenCost({ width, height }) {
+    return Math.ceil((width * height) / 750);
+  }
+
+  async addImageURLs(message, attachments) {
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      EModelEndpoint.anthropic,
+    );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
+  async recordTokenUsage({ promptTokens, completionTokens, model, context = 'message' }) {
+    await spendTokens(
+      {
+        context,
+        user: this.user,
+        conversationId: this.conversationId,
+        model: model ?? this.modelOptions.model,
+        endpointTokenConfig: this.options.endpointTokenConfig,
+      },
+      { promptTokens, completionTokens },
+    );
   }
 
   async buildMessages(messages, parentMessageId) {
@@ -95,32 +213,148 @@ class AnthropicClient extends BaseClient {
       messages,
       parentMessageId,
     });
-    if (this.options.debug) {
-      console.debug('AnthropicClient: orderedMessages', orderedMessages, parentMessageId);
+
+    logger.debug('[AnthropicClient] orderedMessages', { orderedMessages, parentMessageId });
+
+    if (this.options.attachments) {
+      const attachments = await this.options.attachments;
+      const images = attachments.filter((file) => file.type.includes('image'));
+
+      if (images.length && !this.isVisionModel) {
+        throw new Error('Images are only supported with the Claude 3 family of models');
+      }
+
+      const latestMessage = orderedMessages[orderedMessages.length - 1];
+
+      if (this.message_file_map) {
+        this.message_file_map[latestMessage.messageId] = attachments;
+      } else {
+        this.message_file_map = {
+          [latestMessage.messageId]: attachments,
+        };
+      }
+
+      const files = await this.addImageURLs(latestMessage, attachments);
+
+      this.options.attachments = files;
     }
 
-    const formattedMessages = orderedMessages.map((message) => ({
-      author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
-      content: message?.content ?? message.text,
-    }));
+    if (this.message_file_map) {
+      this.contextHandlers = createContextHandlers(
+        this.options.req,
+        orderedMessages[orderedMessages.length - 1].text,
+      );
+    }
+
+    const formattedMessages = orderedMessages.map((message, i) => {
+      const formattedMessage = this.useMessages
+        ? formatMessage({
+          message,
+          endpoint: EModelEndpoint.anthropic,
+        })
+        : {
+          author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
+          content: message?.content ?? message.text,
+        };
+
+      const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+      /* If tokens were never counted, or, is a Vision request and the message has files, count again */
+      if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {
+        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
+      }
+
+      /* If message has files, calculate image token cost */
+      if (this.message_file_map && this.message_file_map[message.messageId]) {
+        const attachments = this.message_file_map[message.messageId];
+        for (const file of attachments) {
+          if (file.embedded) {
+            this.contextHandlers?.processFile(file);
+            continue;
+          }
+
+          orderedMessages[i].tokenCount += this.calculateImageTokenCost({
+            width: file.width,
+            height: file.height,
+          });
+        }
+      }
+
+      formattedMessage.tokenCount = orderedMessages[i].tokenCount;
+      return formattedMessage;
+    });
+
+    if (this.contextHandlers) {
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + (this.options.promptPrefix ?? '');
+    }
+
+    let { context: messagesInWindow, remainingContextTokens } =
+      await this.getMessagesWithinTokenLimit(formattedMessages);
+
+    const tokenCountMap = orderedMessages
+      .slice(orderedMessages.length - messagesInWindow.length)
+      .reduce((map, message, index) => {
+        const { messageId } = message;
+        if (!messageId) {
+          return map;
+        }
+
+        map[messageId] = orderedMessages[index].tokenCount;
+        return map;
+      }, {});
+
+    logger.debug('[AnthropicClient]', {
+      messagesInWindow: messagesInWindow.length,
+      remainingContextTokens,
+    });
 
     let lastAuthor = '';
     let groupedMessages = [];
 
-    for (let message of formattedMessages) {
+    for (let i = 0; i < messagesInWindow.length; i++) {
+      const message = messagesInWindow[i];
+      const author = message.role ?? message.author;
       // If last author is not same as current author, add to new group
-      if (lastAuthor !== message.author) {
-        groupedMessages.push({
-          author: message.author,
+      if (lastAuthor !== author) {
+        const newMessage = {
           content: [message.content],
-        });
-        lastAuthor = message.author;
+        };
+
+        if (message.role) {
+          newMessage.role = message.role;
+        } else {
+          newMessage.author = message.author;
+        }
+
+        groupedMessages.push(newMessage);
+        lastAuthor = author;
         // If same author, append content to the last group
       } else {
         groupedMessages[groupedMessages.length - 1].content.push(message.content);
       }
     }
 
+    groupedMessages = groupedMessages.map((msg, i) => {
+      const isLast = i === groupedMessages.length - 1;
+      if (msg.content.length === 1) {
+        const content = msg.content[0];
+        return {
+          ...msg,
+          // reason: final assistant content cannot end with trailing whitespace
+          content:
+            isLast && this.useMessages && msg.role === 'assistant' && typeof content === 'string'
+              ? content?.trim()
+              : content,
+        };
+      }
+
+      if (!this.useMessages && msg.tokenCount) {
+        delete msg.tokenCount;
+      }
+
+      return msg;
+    });
+
     let identityPrefix = '';
     if (this.options.userLabel) {
       identityPrefix = `\nHuman's name: ${this.options.userLabel}`;
@@ -146,9 +380,10 @@ class AnthropicClient extends BaseClient {
     // Prompt AI to respond, empty if last message was from AI
     let isEdited = lastAuthor === this.assistantLabel;
     const promptSuffix = isEdited ? '' : `${promptPrefix}${this.assistantLabel}\n`;
-    let currentTokenCount = isEdited
-      ? this.getTokenCount(promptPrefix)
-      : this.getTokenCount(promptSuffix);
+    let currentTokenCount =
+      isEdited || this.useMessages
+        ? this.getTokenCount(promptPrefix)
+        : this.getTokenCount(promptSuffix);
 
     let promptBody = '';
     const maxTokenCount = this.maxPromptTokens;
@@ -216,7 +451,69 @@ class AnthropicClient extends BaseClient {
       return true;
     };
 
-    await buildPromptBody();
+    const messagesPayload = [];
+    const buildMessagesPayload = async () => {
+      let canContinue = true;
+
+      if (promptPrefix) {
+        this.systemMessage = promptPrefix;
+      }
+
+      while (currentTokenCount < maxTokenCount && groupedMessages.length > 0 && canContinue) {
+        const message = groupedMessages.pop();
+
+        let tokenCountForMessage = message.tokenCount ?? this.getTokenCountForMessage(message);
+
+        const newTokenCount = currentTokenCount + tokenCountForMessage;
+        const exceededMaxCount = newTokenCount > maxTokenCount;
+
+        if (exceededMaxCount && messagesPayload.length === 0) {
+          throw new Error(
+            `Prompt is too long. Max token count is ${maxTokenCount}, but prompt is ${newTokenCount} tokens long.`,
+          );
+        } else if (exceededMaxCount) {
+          canContinue = false;
+          break;
+        }
+
+        delete message.tokenCount;
+        messagesPayload.unshift(message);
+        currentTokenCount = newTokenCount;
+
+        // Switch off isEdited after using it once
+        if (isEdited && message.role === 'assistant') {
+          isEdited = false;
+        }
+
+        // Wait for next tick to avoid blocking the event loop
+        await new Promise((resolve) => setImmediate(resolve));
+      }
+    };
+
+    const processTokens = () => {
+      // Add 2 tokens for metadata after all messages have been counted.
+      currentTokenCount += 2;
+
+      // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
+      this.modelOptions.maxOutputTokens = Math.min(
+        this.maxContextTokens - currentTokenCount,
+        this.maxResponseTokens,
+      );
+    };
+
+    if (this.modelOptions.model.startsWith('claude-3')) {
+      await buildMessagesPayload();
+      processTokens();
+      return {
+        prompt: messagesPayload,
+        context: messagesInWindow,
+        promptTokens: currentTokenCount,
+        tokenCountMap,
+      };
+    } else {
+      await buildPromptBody();
+      processTokens();
+    }
 
     if (nextMessage.remove) {
       promptBody = promptBody.replace(nextMessage.messageString, '');
@@ -226,20 +523,24 @@ class AnthropicClient extends BaseClient {
 
     let prompt = `${promptBody}${promptSuffix}`;
 
-    // Add 2 tokens for metadata after all messages have been counted.
-    currentTokenCount += 2;
-
-    // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
-    this.modelOptions.maxOutputTokens = Math.min(
-      this.maxContextTokens - currentTokenCount,
-      this.maxResponseTokens,
-    );
-
-    return { prompt, context };
+    return { prompt, context, promptTokens: currentTokenCount, tokenCountMap };
   }
 
   getCompletion() {
-    console.log('AnthropicClient doesn\'t use getCompletion (all handled in sendCompletion)');
+    logger.debug('AnthropicClient doesn\'t use getCompletion (all handled in sendCompletion)');
+  }
+
+  /**
+   * Creates a message or completion response using the Anthropic client.
+   * @param {Anthropic} client - The Anthropic client instance.
+   * @param {Anthropic.default.MessageCreateParams | Anthropic.default.CompletionCreateParams} options - The options for the message or completion.
+   * @param {boolean} useMessages - Whether to use messages or completions. Defaults to `this.useMessages`.
+   * @returns {Promise<Anthropic.default.Message | Anthropic.default.Completion>} The response from the Anthropic client.
+   */
+  async createResponse(client, options, useMessages) {
+    return useMessages ?? this.useMessages
+      ? await client.messages.create(options)
+      : await client.completions.create(options);
   }
 
   async sendCompletion(payload, { onProgress, abortController }) {
@@ -254,12 +555,7 @@ class AnthropicClient extends BaseClient {
       modelOptions.stream = true;
     }
 
-    const { debug } = this.options;
-    if (debug) {
-      console.debug();
-      console.debug(modelOptions);
-      console.debug();
-    }
+    logger.debug('modelOptions', { modelOptions });
 
     const client = this.getClient();
     const metadata = {
@@ -276,61 +572,107 @@ class AnthropicClient extends BaseClient {
       topP: top_p,
       topK: top_k,
     } = this.modelOptions;
+
     const requestOptions = {
-      prompt: payload,
       model,
       stream: stream || true,
-      max_tokens_to_sample: maxOutputTokens || 1500,
       stop_sequences,
       temperature,
       metadata,
       top_p,
       top_k,
     };
-    if (this.options.debug) {
-      console.log('AnthropicClient: requestOptions');
-      console.dir(requestOptions, { depth: null });
-    }
-    const response = await client.completions.create(requestOptions);
 
-    signal.addEventListener('abort', () => {
-      if (this.options.debug) {
-        console.log('AnthropicClient: message aborted!');
-      }
-      response.controller.abort();
-    });
-
-    for await (const completion of response) {
-      if (this.options.debug) {
-        // Uncomment to debug message stream
-        // console.debug(completion);
-      }
-      text += completion.completion;
-      onProgress(completion.completion);
+    if (this.useMessages) {
+      requestOptions.messages = payload;
+      requestOptions.max_tokens = maxOutputTokens || 1500;
+    } else {
+      requestOptions.prompt = payload;
+      requestOptions.max_tokens_to_sample = maxOutputTokens || 1500;
     }
 
-    signal.removeEventListener('abort', () => {
-      if (this.options.debug) {
-        console.log('AnthropicClient: message aborted!');
+    if (this.systemMessage) {
+      requestOptions.system = this.systemMessage;
+    }
+
+    logger.debug('[AnthropicClient]', { ...requestOptions });
+
+    const handleChunk = (currentChunk) => {
+      if (currentChunk) {
+        text += currentChunk;
+        onProgress(currentChunk);
       }
-      response.controller.abort();
-    });
+    };
+
+    const maxRetries = 3;
+    async function processResponse() {
+      let attempts = 0;
+
+      while (attempts < maxRetries) {
+        let response;
+        try {
+          response = await this.createResponse(client, requestOptions);
+
+          signal.addEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+
+          for await (const completion of response) {
+            // Handle each completion as before
+            if (completion?.delta?.text) {
+              handleChunk(completion.delta.text);
+            } else if (completion.completion) {
+              handleChunk(completion.completion);
+            }
+          }
+
+          // Successful processing, exit loop
+          break;
+        } catch (error) {
+          attempts += 1;
+          logger.warn(
+            `User: ${this.user} | Anthropic Request ${attempts} failed: ${error.message}`,
+          );
+
+          if (attempts < maxRetries) {
+            await delayBeforeRetry(attempts, 350);
+          } else {
+            throw new Error(`Operation failed after ${maxRetries} attempts: ${error.message}`);
+          }
+        } finally {
+          signal.removeEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+        }
+      }
+    }
+
+    await processResponse.bind(this)();
 
     return text.trim();
   }
 
   getSaveOptions() {
     return {
+      maxContextTokens: this.options.maxContextTokens,
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
+      resendFiles: this.options.resendFiles,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
 
   getBuildMessagesOptions() {
-    if (this.options.debug) {
-      console.log('AnthropicClient doesn\'t use getBuildMessagesOptions');
-    }
+    logger.debug('AnthropicClient doesn\'t use getBuildMessagesOptions');
   }
 
   static getTokenizer(encoding, isModelName = false, extendSpecialTokens = {}) {
@@ -350,6 +692,78 @@ class AnthropicClient extends BaseClient {
   getTokenCount(text) {
     return this.gptEncoder.encode(text, 'all').length;
   }
+
+  /**
+   * Generates a concise title for a conversation based on the user's input text and response.
+   * Involves sending a chat completion request with specific instructions for title generation.
+   *
+   * This function capitlizes on [Anthropic's function calling training](https://docs.anthropic.com/claude/docs/functions-external-tools).
+   *
+   * @param {Object} params - The parameters for the conversation title generation.
+   * @param {string} params.text - The user's input.
+   * @param {string} [params.responseText=''] - The AI's immediate response to the user.
+   *
+   * @returns {Promise<string | 'New Chat'>} A promise that resolves to the generated conversation title.
+   *                            In case of failure, it will return the default title, "New Chat".
+   */
+  async titleConvo({ text, responseText = '' }) {
+    let title = 'New Chat';
+    const convo = `<initial_message>
+  ${truncateText(text)}
+  </initial_message>
+  <response>
+  ${JSON.stringify(truncateText(responseText))}
+  </response>`;
+
+    const { ANTHROPIC_TITLE_MODEL } = process.env ?? {};
+    const model = this.options.titleModel ?? ANTHROPIC_TITLE_MODEL ?? 'claude-3-haiku-20240307';
+    const system = titleFunctionPrompt;
+
+    const titleChatCompletion = async () => {
+      const content = `<conversation_context>
+  ${convo}
+  </conversation_context>
+  
+  Please generate a title for this conversation.`;
+
+      const titleMessage = { role: 'user', content };
+      const requestOptions = {
+        model,
+        temperature: 0.3,
+        max_tokens: 1024,
+        system,
+        stop_sequences: ['\n\nHuman:', '\n\nAssistant', '</function_calls>'],
+        messages: [titleMessage],
+      };
+
+      try {
+        const response = await this.createResponse(this.getClient(), requestOptions, true);
+        let promptTokens = response?.usage?.input_tokens;
+        let completionTokens = response?.usage?.output_tokens;
+        if (!promptTokens) {
+          promptTokens = this.getTokenCountForMessage(titleMessage);
+          promptTokens += this.getTokenCountForMessage({ role: 'system', content: system });
+        }
+        if (!completionTokens) {
+          completionTokens = this.getTokenCountForMessage(response.content[0]);
+        }
+        await this.recordTokenUsage({
+          model,
+          promptTokens,
+          completionTokens,
+          context: 'title',
+        });
+        const text = response.content[0].text;
+        title = parseParamFromPrompt(text, 'title');
+      } catch (e) {
+        logger.error('[AnthropicClient] There was an issue generating the title', e);
+      }
+    };
+
+    await titleChatCompletion();
+    logger.debug('[AnthropicClient] Convo Title: ' + title);
+    return title;
+  }
 }
 
 module.exports = AnthropicClient;

api/app/clients/BaseClient.js

@@ -1,8 +1,12 @@
 const crypto = require('crypto');
+const fetch = require('node-fetch');
+const { supportsBalanceCheck, Constants } = require('librechat-data-provider');
+const { getConvo, getMessages, saveMessage, updateMessage, saveConvo } = require('~/models');
+const { addSpaceIfNeeded, isEnabled } = require('~/server/utils');
+const checkBalance = require('~/models/checkBalance');
+const { getFiles } = require('~/models/File');
 const TextStream = require('./TextStream');
-const { getConvo, getMessages, saveMessage, updateMessage, saveConvo } = require('../../models');
-const { addSpaceIfNeeded, isEnabled } = require('../../server/utils');
-const checkBalance = require('../../models/checkBalance');
+const { logger } = require('~/config');
 
 class BaseClient {
   constructor(apiKey, options = {}) {
@@ -14,13 +18,14 @@ class BaseClient {
       month: 'long',
       day: 'numeric',
     });
+    this.fetch = this.fetch.bind(this);
   }
 
   setOptions() {
     throw new Error('Method \'setOptions\' must be implemented.');
   }
 
-  getCompletion() {
+  async getCompletion() {
     throw new Error('Method \'getCompletion\' must be implemented.');
   }
 
@@ -41,15 +46,30 @@ class BaseClient {
   }
 
   async getTokenCountForResponse(response) {
-    if (this.options.debug) {
-      console.debug('`recordTokenUsage` not implemented.', response);
-    }
+    logger.debug('`[BaseClient] recordTokenUsage` not implemented.', response);
   }
 
   async recordTokenUsage({ promptTokens, completionTokens }) {
-    if (this.options.debug) {
-      console.debug('`recordTokenUsage` not implemented.', { promptTokens, completionTokens });
+    logger.debug('`[BaseClient] recordTokenUsage` not implemented.', {
+      promptTokens,
+      completionTokens,
+    });
+  }
+
+  /**
+   * Makes an HTTP request and logs the process.
+   *
+   * @param {RequestInfo} url - The URL to make the request to. Can be a string or a Request object.
+   * @param {RequestInit} [init] - Optional init options for the request.
+   * @returns {Promise<Response>} - A promise that resolves to the response of the fetch request.
+   */
+  async fetch(_url, init) {
+    let url = _url;
+    if (this.options.directEndpoint) {
+      url = this.options.reverseProxyUrl;
     }
+    logger.debug(`Making request to ${url}`);
+    return await fetch(url, init);
   }
 
   getBuildMessagesOptions() {
@@ -62,7 +82,7 @@ class BaseClient {
   }
 
   async setMessageOptions(opts = {}) {
-    if (opts && typeof opts === 'object') {
+    if (opts && opts.replaceOptions) {
       this.setOptions(opts);
     }
 
@@ -72,7 +92,7 @@ class BaseClient {
     const saveOptions = this.getSaveOptions();
     this.abortController = opts.abortController ?? new AbortController();
     const conversationId = opts.conversationId ?? crypto.randomUUID();
-    const parentMessageId = opts.parentMessageId ?? '00000000-0000-0000-0000-000000000000';
+    const parentMessageId = opts.parentMessageId ?? Constants.NO_PARENT;
     const userMessageId = opts.overrideParentMessageId ?? crypto.randomUUID();
     let responseMessageId = opts.responseMessageId ?? crypto.randomUUID();
     let head = isEdited ? responseMessageId : parentMessageId;
@@ -194,14 +214,14 @@ class BaseClient {
       const update = {};
 
       if (messageId === tokenCountMap.summaryMessage?.messageId) {
-        this.options.debug && console.debug(`Adding summary props to ${messageId}.`);
+        logger.debug(`[BaseClient] Adding summary props to ${messageId}.`);
 
         update.summary = tokenCountMap.summaryMessage.content;
         update.summaryTokenCount = tokenCountMap.summaryMessage.tokenCount;
       }
 
       if (message.tokenCount && !update.summaryTokenCount) {
-        this.options.debug && console.debug(`Skipping ${messageId}: already had a token count.`);
+        logger.debug(`[BaseClient] Skipping ${messageId}: already had a token count.`);
         continue;
       }
 
@@ -278,19 +298,17 @@ class BaseClient {
     if (instructions) {
       ({ tokenCount, ..._instructions } = instructions);
     }
-    this.options.debug && _instructions && console.debug('instructions tokenCount', tokenCount);
+    _instructions && logger.debug('[BaseClient] instructions tokenCount: ' + tokenCount);
     let payload = this.addInstructions(formattedMessages, _instructions);
     let orderedWithInstructions = this.addInstructions(orderedMessages, instructions);
 
     let { context, remainingContextTokens, messagesToRefine, summaryIndex } =
       await this.getMessagesWithinTokenLimit(orderedWithInstructions);
 
-    this.options.debug &&
-      console.debug(
-        'remainingContextTokens, this.maxContextTokens (1/2)',
-        remainingContextTokens,
-        this.maxContextTokens,
-      );
+    logger.debug('[BaseClient] Context Count (1/2)', {
+      remainingContextTokens,
+      maxContextTokens: this.maxContextTokens,
+    });
 
     let summaryMessage;
     let summaryTokenCount;
@@ -308,10 +326,9 @@ class BaseClient {
 
     if (diff > 0) {
       payload = payload.slice(diff);
-      this.options.debug &&
-        console.debug(
-          `Difference between original payload (${length}) and context (${context.length}): ${diff}`,
-        );
+      logger.debug(
+        `[BaseClient] Difference between original payload (${length}) and context (${context.length}): ${diff}`,
+      );
     }
 
     const latestMessage = orderedWithInstructions[orderedWithInstructions.length - 1];
@@ -338,12 +355,10 @@ class BaseClient {
     // Make sure to only continue summarization logic if the summary message was generated
     shouldSummarize = summaryMessage && shouldSummarize;
 
-    this.options.debug &&
-      console.debug(
-        'remainingContextTokens, this.maxContextTokens (2/2)',
-        remainingContextTokens,
-        this.maxContextTokens,
-      );
+    logger.debug('[BaseClient] Context Count (2/2)', {
+      remainingContextTokens,
+      maxContextTokens: this.maxContextTokens,
+    });
 
     let tokenCountMap = orderedWithInstructions.reduce((map, message, index) => {
       const { messageId } = message;
@@ -361,19 +376,13 @@ class BaseClient {
 
     const promptTokens = this.maxContextTokens - remainingContextTokens;
 
-    if (this.options.debug) {
-      console.debug('<-------------------------PAYLOAD/TOKEN COUNT MAP------------------------->');
-      console.debug('Payload:', payload);
-      console.debug('Token Count Map:', tokenCountMap);
-      console.debug(
-        'Prompt Tokens',
-        promptTokens,
-        'remainingContextTokens',
-        remainingContextTokens,
-        'this.maxContextTokens',
-        this.maxContextTokens,
-      );
-    }
+    logger.debug('[BaseClient] tokenCountMap:', tokenCountMap);
+    logger.debug('[BaseClient]', {
+      promptTokens,
+      remainingContextTokens,
+      payloadSize: payload.length,
+      maxContextTokens: this.maxContextTokens,
+    });
 
     return { payload, tokenCountMap, promptTokens, messages: orderedWithInstructions };
   }
@@ -382,6 +391,14 @@ class BaseClient {
     const { user, head, isEdited, conversationId, responseMessageId, saveOptions, userMessage } =
       await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     const { generation = '' } = opts;
 
     // It's not necessary to push to currentMessages
@@ -417,14 +434,14 @@ class BaseClient {
       // this only matters when buildMessages is utilizing the parentMessageId, and may vary on implementation
       isEdited ? head : userMessage.messageId,
       this.getBuildMessagesOptions(opts),
+      opts,
     );
 
     if (tokenCountMap) {
-      console.dir(tokenCountMap, { depth: null });
+      logger.debug('[BaseClient] tokenCountMap', tokenCountMap);
       if (tokenCountMap[userMessage.messageId]) {
         userMessage.tokenCount = tokenCountMap[userMessage.messageId];
-        console.log('userMessage.tokenCount', userMessage.tokenCount);
-        console.log('userMessage', userMessage);
+        logger.debug('[BaseClient] userMessage', userMessage);
       }
 
       this.handleTokenCountMap(tokenCountMap);
@@ -434,7 +451,10 @@ class BaseClient {
       await this.saveMessageToDatabase(userMessage, saveOptions, user);
     }
 
-    if (isEnabled(process.env.CHECK_BALANCE)) {
+    if (
+      isEnabled(process.env.CHECK_BALANCE) &&
+      supportsBalanceCheck[this.options.endpointType ?? this.options.endpoint]
+    ) {
       await checkBalance({
         req: this.options.req,
         res: this.options.res,
@@ -442,13 +462,16 @@ class BaseClient {
           user: this.user,
           tokenType: 'prompt',
           amount: promptTokens,
-          debug: this.options.debug,
           model: this.modelOptions.model,
+          endpoint: this.options.endpoint,
+          endpointTokenConfig: this.options.endpointTokenConfig,
         },
       });
     }
 
     const completion = await this.sendCompletion(payload, opts);
+    this.abortController.requestCompleted = true;
+
     const responseMessage = {
       messageId: responseMessageId,
       conversationId,
@@ -459,6 +482,9 @@ class BaseClient {
       sender: this.sender,
       text: addSpaceIfNeeded(generation) + completion,
       promptTokens,
+      iconURL: this.options.iconURL,
+      endpoint: this.options.endpoint,
+      ...(this.metadata ?? {}),
     };
 
     if (
@@ -481,9 +507,7 @@ class BaseClient {
   }
 
   async loadHistory(conversationId, parentMessageId = null) {
-    if (this.options.debug) {
-      console.debug('Loading history for conversation', conversationId, parentMessageId);
-    }
+    logger.debug('[BaseClient] Loading history:', { conversationId, parentMessageId });
 
     const messages = (await getMessages({ conversationId })) ?? [];
 
@@ -496,37 +520,56 @@ class BaseClient {
       mapMethod = this.getMessageMapMethod();
     }
 
-    const orderedMessages = this.constructor.getMessagesForConversation({
+    let _messages = this.constructor.getMessagesForConversation({
       messages,
       parentMessageId,
       mapMethod,
     });
 
+    _messages = await this.addPreviousAttachments(_messages);
+
     if (!this.shouldSummarize) {
-      return orderedMessages;
+      return _messages;
     }
 
     // Find the latest message with a 'summary' property
-    for (let i = orderedMessages.length - 1; i >= 0; i--) {
-      if (orderedMessages[i]?.summary) {
-        this.previous_summary = orderedMessages[i];
+    for (let i = _messages.length - 1; i >= 0; i--) {
+      if (_messages[i]?.summary) {
+        this.previous_summary = _messages[i];
         break;
       }
     }
 
-    if (this.options.debug && this.previous_summary) {
+    if (this.previous_summary) {
       const { messageId, summary, tokenCount, summaryTokenCount } = this.previous_summary;
-      console.debug('Previous summary:', { messageId, summary, tokenCount, summaryTokenCount });
+      logger.debug('[BaseClient] Previous summary:', {
+        messageId,
+        summary,
+        tokenCount,
+        summaryTokenCount,
+      });
     }
 
-    return orderedMessages;
+    return _messages;
   }
 
+  /**
+   * Save a message to the database.
+   * @param {TMessage} message
+   * @param {Partial<TConversation>} endpointOptions
+   * @param {string | null} user
+   */
   async saveMessageToDatabase(message, endpointOptions, user = null) {
-    await saveMessage({ ...message, user, unfinished: false, cancelled: false });
+    await saveMessage({
+      ...message,
+      endpoint: this.options.endpoint,
+      unfinished: false,
+      user,
+    });
     await saveConvo(user, {
       conversationId: message.conversationId,
       endpoint: this.options.endpoint,
+      endpointType: this.options.endpointType,
       ...endpointOptions,
     });
   }
@@ -548,15 +591,15 @@ class BaseClient {
    *
    * Each message object should have an 'id' or 'messageId' property and may have a 'parentMessageId' property.
    * The 'parentMessageId' is the ID of the message that the current message is a reply to.
-   * If 'parentMessageId' is not present, null, or is '00000000-0000-0000-0000-000000000000',
+   * If 'parentMessageId' is not present, null, or is Constants.NO_PARENT,
    * the message is considered a root message.
    *
    * @param {Object} options - The options for the function.
-   * @param {Array} options.messages - An array of message objects. Each object should have either an 'id' or 'messageId' property, and may have a 'parentMessageId' property.
+   * @param {TMessage[]} options.messages - An array of message objects. Each object should have either an 'id' or 'messageId' property, and may have a 'parentMessageId' property.
    * @param {string} options.parentMessageId - The ID of the parent message to start the traversal from.
    * @param {Function} [options.mapMethod] - An optional function to map over the ordered messages. If provided, it will be applied to each message in the resulting array.
    * @param {boolean} [options.summary=false] - If set to true, the traversal modifies messages with 'summary' and 'summaryTokenCount' properties and stops at the message with a 'summary' property.
-   * @returns {Array} An array containing the messages in the order they should be displayed, starting with the most recent message with a 'summary' property if the 'summary' option is true, and ending with the message identified by 'parentMessageId'.
+   * @returns {TMessage[]} An array containing the messages in the order they should be displayed, starting with the most recent message with a 'summary' property if the 'summary' option is true, and ending with the message identified by 'parentMessageId'.
    */
   static getMessagesForConversation({
     messages,
@@ -603,9 +646,7 @@ class BaseClient {
       }
 
       currentMessageId =
-        message.parentMessageId === '00000000-0000-0000-0000-000000000000'
-          ? null
-          : message.parentMessageId;
+        message.parentMessageId === Constants.NO_PARENT ? null : message.parentMessageId;
     }
 
     orderedMessages.reverse();
@@ -624,6 +665,11 @@ class BaseClient {
    * An additional 3 tokens need to be added for assistant label priming after all messages have been counted.
    * In our implementation, this is accounted for in the getMessagesWithinTokenLimit method.
    *
+   * The content parts example was adapted from the following example:
+   * https://github.com/openai/openai-cookbook/pull/881/files
+   *
+   * Note: image token calculation is to be done elsewhere where we have access to the image metadata
+   *
    * @param {Object} message
    */
   getTokenCountForMessage(message) {
@@ -636,14 +682,34 @@ class BaseClient {
       tokensPerName = -1;
     }
 
+    const processValue = (value) => {
+      if (Array.isArray(value)) {
+        for (let item of value) {
+          if (!item || !item.type || item.type === 'image_url') {
+            continue;
+          }
+
+          const nestedValue = item[item.type];
+
+          if (!nestedValue) {
+            continue;
+          }
+
+          processValue(nestedValue);
+        }
+      } else {
+        numTokens += this.getTokenCount(value);
+      }
+    };
+
     let numTokens = tokensPerMessage;
     for (let [key, value] of Object.entries(message)) {
-      numTokens += this.getTokenCount(value);
+      processValue(value);
+
       if (key === 'name') {
         numTokens += tokensPerName;
       }
     }
-
     return numTokens;
   }
 
@@ -654,6 +720,54 @@ class BaseClient {
 
     return await this.sendCompletion(payload, opts);
   }
+
+  /**
+   *
+   * @param {TMessage[]} _messages
+   * @returns {Promise<TMessage[]>}
+   */
+  async addPreviousAttachments(_messages) {
+    if (!this.options.resendFiles) {
+      return _messages;
+    }
+
+    /**
+     *
+     * @param {TMessage} message
+     */
+    const processMessage = async (message) => {
+      if (!this.message_file_map) {
+        /** @type {Record<string, MongoFile[]> */
+        this.message_file_map = {};
+      }
+
+      const fileIds = message.files.map((file) => file.file_id);
+      const files = await getFiles({
+        file_id: { $in: fileIds },
+      });
+
+      await this.addImageURLs(message, files);
+
+      this.message_file_map[message.messageId] = files;
+      return message;
+    };
+
+    const promises = [];
+
+    for (const message of _messages) {
+      if (!message.files) {
+        promises.push(message);
+        continue;
+      }
+
+      promises.push(processMessage(message));
+    }
+
+    const messages = await Promise.all(promises);
+
+    this.checkVisionRequest(Object.values(this.message_file_map ?? {}).flat());
+    return messages;
+  }
 }
 
 module.exports = BaseClient;

api/app/clients/ChatGPTClient.js

@@ -1,9 +1,19 @@
-const crypto = require('crypto');
 const Keyv = require('keyv');
+const crypto = require('crypto');
+const {
+  EModelEndpoint,
+  resolveHeaders,
+  CohereConstants,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
+const { CohereClient } = require('cohere-ai');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
+const { createCoherePayload } = require('./llm');
 const { Agent, ProxyAgent } = require('undici');
 const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
+const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils');
 
 const CHATGPT_MODEL = 'gpt-3.5-turbo';
 const tokenizersCache = {};
@@ -140,11 +150,13 @@ class ChatGPTClient extends BaseClient {
     return tokenizer;
   }
 
-  async getCompletion(input, onProgress, abortController = null) {
+  /** @type {getCompletion} */
+  async getCompletion(input, onProgress, onTokenProgress, abortController = null) {
     if (!abortController) {
       abortController = new AbortController();
     }
-    const modelOptions = { ...this.modelOptions };
+
+    let modelOptions = { ...this.modelOptions };
     if (typeof onProgress === 'function') {
       modelOptions.stream = true;
     }
@@ -159,50 +171,176 @@ class ChatGPTClient extends BaseClient {
     }
 
     const { debug } = this.options;
-    const url = this.completionsUrl;
+    let baseURL = this.completionsUrl;
     if (debug) {
       console.debug();
-      console.debug(url);
+      console.debug(baseURL);
       console.debug(modelOptions);
       console.debug();
     }
+
     const opts = {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify(modelOptions),
       dispatcher: new Agent({
         bodyTimeout: 0,
         headersTimeout: 0,
       }),
     };
 
-    if (this.apiKey && this.options.azure) {
-      opts.headers['api-key'] = this.apiKey;
+    if (this.isVisionModel) {
+      modelOptions.max_tokens = 4000;
+    }
+
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const isAzure = this.azure || this.options.azure;
+    if (
+      (isAzure && this.isVisionModel && azureConfig) ||
+      (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    ) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+      opts.headers = resolveHeaders(headers);
+      this.langchainProxy = extractBaseURL(baseURL);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+      this.azure = !serverless && azureOptions;
+      this.azureEndpoint =
+        !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+    }
+
+    if (this.options.headers) {
+      opts.headers = { ...opts.headers, ...this.options.headers };
+    }
+
+    if (isAzure) {
+      // Azure does not accept `model` in the body, so we need to remove it.
+      delete modelOptions.model;
+
+      baseURL = this.langchainProxy
+        ? constructAzureURL({
+          baseURL: this.langchainProxy,
+          azureOptions: this.azure,
+        })
+        : this.azureEndpoint.split(/(?<!\/)\/(chat|completion)\//)[0];
+
+      if (this.options.forcePrompt) {
+        baseURL += '/completions';
+      } else {
+        baseURL += '/chat/completions';
+      }
+
+      opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
+      opts.headers = { ...opts.headers, 'api-key': this.apiKey };
     } else if (this.apiKey) {
       opts.headers.Authorization = `Bearer ${this.apiKey}`;
     }
 
+    if (process.env.OPENAI_ORGANIZATION) {
+      opts.headers['OpenAI-Organization'] = process.env.OPENAI_ORGANIZATION;
+    }
+
     if (this.useOpenRouter) {
       opts.headers['HTTP-Referer'] = 'https://librechat.ai';
       opts.headers['X-Title'] = 'LibreChat';
     }
 
-    if (this.options.headers) {
-      opts.headers = { ...opts.headers, ...this.options.headers };
-    }
-
     if (this.options.proxy) {
       opts.dispatcher = new ProxyAgent(this.options.proxy);
     }
 
+    /* hacky fixes for Mistral AI API:
+      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
+      - If there is only one message and it's a system message, change the role to user
+      */
+    if (baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+      const { messages } = modelOptions;
+
+      const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+      if (systemMessageIndex > 0) {
+        const [systemMessage] = messages.splice(systemMessageIndex, 1);
+        messages.unshift(systemMessage);
+      }
+
+      modelOptions.messages = messages;
+
+      if (messages.length === 1 && messages[0].role === 'system') {
+        modelOptions.messages[0].role = 'user';
+      }
+    }
+
+    if (this.options.addParams && typeof this.options.addParams === 'object') {
+      modelOptions = {
+        ...modelOptions,
+        ...this.options.addParams,
+      };
+      logger.debug('[ChatGPTClient] chatCompletion: added params', {
+        addParams: this.options.addParams,
+        modelOptions,
+      });
+    }
+
+    if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
+      this.options.dropParams.forEach((param) => {
+        delete modelOptions[param];
+      });
+      logger.debug('[ChatGPTClient] chatCompletion: dropped params', {
+        dropParams: this.options.dropParams,
+        modelOptions,
+      });
+    }
+
+    if (baseURL.startsWith(CohereConstants.API_URL)) {
+      const payload = createCoherePayload({ modelOptions });
+      return await this.cohereChatCompletion({ payload, onTokenProgress });
+    }
+
+    if (baseURL.includes('v1') && !baseURL.includes('/completions') && !this.isChatCompletion) {
+      baseURL = baseURL.split('v1')[0] + 'v1/completions';
+    } else if (
+      baseURL.includes('v1') &&
+      !baseURL.includes('/chat/completions') &&
+      this.isChatCompletion
+    ) {
+      baseURL = baseURL.split('v1')[0] + 'v1/chat/completions';
+    }
+
+    const BASE_URL = new URL(baseURL);
+    if (opts.defaultQuery) {
+      Object.entries(opts.defaultQuery).forEach(([key, value]) => {
+        BASE_URL.searchParams.append(key, value);
+      });
+      delete opts.defaultQuery;
+    }
+
+    const completionsURL = BASE_URL.toString();
+    opts.body = JSON.stringify(modelOptions);
+
     if (modelOptions.stream) {
       // eslint-disable-next-line no-async-promise-executor
       return new Promise(async (resolve, reject) => {
         try {
           let done = false;
-          await fetchEventSource(url, {
+          await fetchEventSource(completionsURL, {
             ...opts,
             signal: abortController.signal,
             async onopen(response) {
@@ -230,7 +368,6 @@ class ChatGPTClient extends BaseClient {
               // workaround for private API not sending [DONE] event
               if (!done) {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
               }
             },
@@ -243,14 +380,13 @@ class ChatGPTClient extends BaseClient {
             },
             onmessage(message) {
               if (debug) {
-                // console.debug(message);
+                console.debug(message);
               }
               if (!message.data || message.event === 'ping') {
                 return;
               }
               if (message.data === '[DONE]') {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
                 done = true;
                 return;
@@ -263,7 +399,7 @@ class ChatGPTClient extends BaseClient {
         }
       });
     }
-    const response = await fetch(url, {
+    const response = await fetch(completionsURL, {
       ...opts,
       signal: abortController.signal,
     });
@@ -281,6 +417,43 @@ class ChatGPTClient extends BaseClient {
     return response.json();
   }
 
+  /** @type {cohereChatCompletion} */
+  async cohereChatCompletion({ payload, onTokenProgress }) {
+    const cohere = new CohereClient({
+      token: this.apiKey,
+      environment: this.completionsUrl,
+    });
+
+    if (!payload.stream) {
+      const chatResponse = await cohere.chat(payload);
+      return chatResponse.text;
+    }
+
+    const chatStream = await cohere.chatStream(payload);
+    let reply = '';
+    for await (const message of chatStream) {
+      if (!message) {
+        continue;
+      }
+
+      if (message.eventType === 'text-generation' && message.text) {
+        onTokenProgress(message.text);
+        reply += message.text;
+      }
+      /*
+      Cohere API Chinese Unicode character replacement hotfix.
+      Should be un-commented when the following issue is resolved:
+      https://github.com/cohere-ai/cohere-typescript/issues/151
+
+      else if (message.eventType === 'stream-end' && message.response) {
+        reply = message.response.text;
+      }
+      */
+    }
+
+    return reply;
+  }
+
   async generateTitle(userMessage, botMessage) {
     const instructionsPayload = {
       role: 'system',
@@ -548,7 +721,7 @@ ${botMessage.message}
     if (isChatGptModel) {
       return { prompt: [instructionsPayload, messagePayload], context };
     }
-    return { prompt, context };
+    return { prompt, context, promptTokens: currentTokenCount };
   }
 
   getTokenCount(text) {

api/app/clients/GoogleClient.js

@@ -1,23 +1,64 @@
-const BaseClient = require('./BaseClient');
 const { google } = require('googleapis');
 const { Agent, ProxyAgent } = require('undici');
+const { ChatVertexAI } = require('@langchain/google-vertexai');
+const { ChatGoogleGenerativeAI } = require('@langchain/google-genai');
+const { GoogleGenerativeAI: GenAI } = require('@google/generative-ai');
+const { GoogleVertexAI } = require('@langchain/community/llms/googlevertexai');
+const { ChatGoogleVertexAI } = require('langchain/chat_models/googlevertexai');
+const { AIMessage, HumanMessage, SystemMessage } = require('langchain/schema');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
+const {
+  validateVisionModel,
+  getResponseSender,
+  endpointSettings,
+  EModelEndpoint,
+  VisionModes,
+  AuthKeys,
+} = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images');
+const { formatMessage, createContextHandlers } = require('./prompts');
+const { getModelMaxTokens } = require('~/utils');
+const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
 
+const loc = 'us-central1';
+const publisher = 'google';
+const endpointPrefix = `https://${loc}-aiplatform.googleapis.com`;
+// const apiEndpoint = loc + '-aiplatform.googleapis.com';
 const tokenizersCache = {};
 
+const settings = endpointSettings[EModelEndpoint.google];
+
 class GoogleClient extends BaseClient {
   constructor(credentials, options = {}) {
     super('apiKey', options);
-    this.client_email = credentials.client_email;
-    this.project_id = credentials.project_id;
-    this.private_key = credentials.private_key;
-    this.sender = 'PaLM2';
+    let creds = {};
+
+    if (typeof credentials === 'string') {
+      creds = JSON.parse(credentials);
+    } else if (credentials) {
+      creds = credentials;
+    }
+
+    const serviceKey = creds[AuthKeys.GOOGLE_SERVICE_KEY] ?? {};
+    this.serviceKey =
+      serviceKey && typeof serviceKey === 'string' ? JSON.parse(serviceKey) : serviceKey ?? {};
+    this.client_email = this.serviceKey.client_email;
+    this.private_key = this.serviceKey.private_key;
+    this.project_id = this.serviceKey.project_id;
+    this.access_token = null;
+
+    this.apiKey = creds[AuthKeys.GOOGLE_API_KEY];
+
+    if (options.skipSetOptions) {
+      return;
+    }
     this.setOptions(options);
   }
 
-  /* Google/PaLM2 specific methods */
+  /* Google specific methods */
   constructUrl() {
-    return `https://us-central1-aiplatform.googleapis.com/v1/projects/${this.project_id}/locations/us-central1/publishers/google/models/${this.modelOptions.model}:predict`;
+    return `${endpointPrefix}/v1/projects/${this.project_id}/locations/${loc}/publishers/${publisher}/models/${this.modelOptions.model}:serverStreamingPredict`;
   }
 
   async getClient() {
@@ -26,8 +67,7 @@ class GoogleClient extends BaseClient {
 
     jwtClient.authorize((err) => {
       if (err) {
-        console.error('Error: jwtClient failed to authorize');
-        console.error(err.message);
+        logger.error('jwtClient failed to authorize', err);
         throw err;
       }
     });
@@ -35,6 +75,22 @@ class GoogleClient extends BaseClient {
     return jwtClient;
   }
 
+  async getAccessToken() {
+    const scopes = ['https://www.googleapis.com/auth/cloud-platform'];
+    const jwtClient = new google.auth.JWT(this.client_email, null, this.private_key, scopes);
+
+    return new Promise((resolve, reject) => {
+      jwtClient.authorize((err, tokens) => {
+        if (err) {
+          logger.error('jwtClient failed to authorize', err);
+          reject(err);
+        } else {
+          resolve(tokens.access_token);
+        }
+      });
+    });
+  }
+
   /* Required Client methods */
   setOptions(options) {
     if (this.options && !this.options.replaceOptions) {
@@ -53,30 +109,47 @@ class GoogleClient extends BaseClient {
       this.options = options;
     }
 
-    this.options.examples = this.options.examples.filter(
-      (obj) => obj.input.content !== '' && obj.output.content !== '',
-    );
+    this.options.examples = (this.options.examples ?? [])
+      .filter((ex) => ex)
+      .filter((obj) => obj.input.content !== '' && obj.output.content !== '');
 
     const modelOptions = this.options.modelOptions || {};
     this.modelOptions = {
       ...modelOptions,
       // set some good defaults (check for undefined in some cases because they may be 0)
-      model: modelOptions.model || 'chat-bison',
-      temperature: typeof modelOptions.temperature === 'undefined' ? 0.2 : modelOptions.temperature, // 0 - 1, 0.2 is recommended
-      topP: typeof modelOptions.topP === 'undefined' ? 0.95 : modelOptions.topP, // 0 - 1, default: 0.95
-      topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40
+      model: modelOptions.model || settings.model.default,
+      temperature:
+        typeof modelOptions.temperature === 'undefined'
+          ? settings.temperature.default
+          : modelOptions.temperature,
+      topP: typeof modelOptions.topP === 'undefined' ? settings.topP.default : modelOptions.topP,
+      topK: typeof modelOptions.topK === 'undefined' ? settings.topK.default : modelOptions.topK,
       // stop: modelOptions.stop // no stop method for now
     };
 
-    this.isChatModel = this.modelOptions.model.startsWith('chat-');
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
+
+    /** @type {boolean} Whether using a "GenerativeAI" Model */
+    this.isGenerativeModel = this.modelOptions.model.includes('gemini');
+    const { isGenerativeModel } = this;
+    this.isChatModel = !isGenerativeModel && this.modelOptions.model.includes('chat');
     const { isChatModel } = this;
-    this.isTextModel = this.modelOptions.model.startsWith('text-');
+    this.isTextModel =
+      !isGenerativeModel && !isChatModel && /code|text/.test(this.modelOptions.model);
     const { isTextModel } = this;
 
-    this.maxContextTokens = this.options.maxContextTokens || (isTextModel ? 8000 : 4096);
+    this.maxContextTokens =
+      this.options.maxContextTokens ??
+      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google);
+
     // The max prompt tokens is determined by the max context tokens minus the max response tokens.
     // Earlier messages will be dropped until the prompt is within the limit.
-    this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1024;
+    this.maxResponseTokens = this.modelOptions.maxOutputTokens || settings.maxOutputTokens.default;
+
+    if (this.maxContextTokens > 32000) {
+      this.maxContextTokens = this.maxContextTokens - this.maxResponseTokens;
+    }
+
     this.maxPromptTokens =
       this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens;
 
@@ -88,10 +161,18 @@ class GoogleClient extends BaseClient {
       );
     }
 
+    this.sender =
+      this.options.sender ??
+      getResponseSender({
+        model: this.modelOptions.model,
+        endpoint: EModelEndpoint.google,
+        modelLabel: this.options.modelLabel,
+      });
+
     this.userLabel = this.options.userLabel || 'User';
     this.modelLabel = this.options.modelLabel || 'Assistant';
 
-    if (isChatModel) {
+    if (isChatModel || isGenerativeModel) {
       // Use these faux tokens to help the AI understand the context since we are building the chat log ourselves.
       // Trying to use "<|im_start|>" causes the AI to still generate "<" or "<|" at the end sometimes for some reason,
       // without tripping the stop sequences, so I'm using "||>" instead.
@@ -99,8 +180,8 @@ class GoogleClient extends BaseClient {
       this.endToken = '';
       this.gptEncoder = this.constructor.getTokenizer('cl100k_base');
     } else if (isTextModel) {
-      this.startToken = '<|im_start|>';
-      this.endToken = '<|im_end|>';
+      this.startToken = '||>';
+      this.endToken = '';
       this.gptEncoder = this.constructor.getTokenizer('text-davinci-003', true, {
         '<|im_start|>': 100264,
         '<|im_end|>': 100265,
@@ -138,22 +219,180 @@ class GoogleClient extends BaseClient {
     return this;
   }
 
-  getMessageMapMethod() {
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    /* Validation vision request */
+    this.defaultVisionModel = this.options.visionModel ?? 'gemini-pro-vision';
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.google];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      availableModels?.includes(this.defaultVisionModel) &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+
+    if (this.isVisionModel && !attachments && this.modelOptions.model.includes('gemini-pro')) {
+      this.modelOptions.model = 'gemini-pro';
+      this.isVisionModel = false;
+    }
+  }
+
+  formatMessages() {
     return ((message) => ({
       author: message?.author ?? (message.isCreatedByUser ? this.userLabel : this.modelLabel),
       content: message?.content ?? message.text,
     })).bind(this);
   }
 
-  buildMessages(messages = []) {
-    const formattedMessages = messages.map(this.getMessageMapMethod());
+  /**
+   * Formats messages for generative AI
+   * @param {TMessage[]} messages
+   * @returns
+   */
+  async formatGenerativeMessages(messages) {
+    const formattedMessages = [];
+    const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    const files = await this.addImageURLs(latestMessage, attachments, VisionModes.generative);
+    this.options.attachments = files;
+    messages[messages.length - 1] = latestMessage;
+
+    for (const _message of messages) {
+      const role = _message.isCreatedByUser ? this.userLabel : this.modelLabel;
+      const parts = [];
+      parts.push({ text: _message.text });
+      if (!_message.image_urls?.length) {
+        formattedMessages.push({ role, parts });
+        continue;
+      }
+
+      for (const images of _message.image_urls) {
+        if (images.inlineData) {
+          parts.push({ inlineData: images.inlineData });
+        }
+      }
+
+      formattedMessages.push({ role, parts });
+    }
+
+    return formattedMessages;
+  }
+
+  /**
+   *
+   * Adds image URLs to the message object and returns the files
+   *
+   * @param {TMessage[]} messages
+   * @param {MongoFile[]} files
+   * @returns {Promise<MongoFile[]>}
+   */
+  async addImageURLs(message, attachments, mode = '') {
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      EModelEndpoint.google,
+      mode,
+    );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
+  /**
+   * Builds the augmented prompt for attachments
+   * TODO: Add File API Support
+   * @param {TMessage[]} messages
+   */
+  async buildAugmentedPrompt(messages = []) {
+    const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    this.contextHandlers = createContextHandlers(this.options.req, latestMessage.text);
+
+    if (this.contextHandlers) {
+      for (const file of attachments) {
+        if (file.embedded) {
+          this.contextHandlers?.processFile(file);
+          continue;
+        }
+      }
+
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + this.options.promptPrefix;
+    }
+  }
+
+  async buildVisionMessages(messages = [], parentMessageId) {
+    const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    await this.buildAugmentedPrompt(messages);
+
+    const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId);
+
+    const files = await this.addImageURLs(latestMessage, attachments);
+
+    this.options.attachments = files;
+
+    latestMessage.text = prompt;
+
+    const payload = {
+      instances: [
+        {
+          messages: [new HumanMessage(formatMessage({ message: latestMessage }))],
+        },
+      ],
+      parameters: this.modelOptions,
+    };
+    return { prompt: payload };
+  }
+
+  /** @param {TMessage[]} [messages=[]]  */
+  async buildGenerativeMessages(messages = []) {
+    this.userLabel = 'user';
+    this.modelLabel = 'model';
+    const promises = [];
+    promises.push(await this.formatGenerativeMessages(messages));
+    promises.push(this.buildAugmentedPrompt(messages));
+    const [formattedMessages] = await Promise.all(promises);
+    return { prompt: formattedMessages };
+  }
+
+  async buildMessages(messages = [], parentMessageId) {
+    if (!this.isGenerativeModel && !this.project_id) {
+      throw new Error(
+        '[GoogleClient] a Service Account JSON Key is required for PaLM 2 and Codey models (Vertex AI)',
+      );
+    }
+
+    if (!this.project_id && this.modelOptions.model.includes('1.5')) {
+      return await this.buildGenerativeMessages(messages);
+    }
+
+    if (this.options.attachments && this.isGenerativeModel) {
+      return this.buildVisionMessages(messages, parentMessageId);
+    }
+
+    if (this.isTextModel) {
+      return this.buildMessagesPrompt(messages, parentMessageId);
+    }
+
     let payload = {
       instances: [
         {
-          messages: formattedMessages,
+          messages: messages
+            .map(this.formatMessages())
+            .map((msg) => ({ ...msg, role: msg.author === 'User' ? 'user' : 'assistant' }))
+            .map((message) => formatMessage({ message, langChain: true })),
         },
       ],
-      parameters: this.options.modelOptions,
+      parameters: this.modelOptions,
     };
 
     if (this.options.promptPrefix) {
@@ -164,34 +403,171 @@ class GoogleClient extends BaseClient {
       payload.instances[0].examples = this.options.examples;
     }
 
-    /* TO-DO: text model needs more context since it can't process an array of messages */
-    if (this.isTextModel) {
-      payload.instances = [
-        {
-          prompt: messages[messages.length - 1].content,
-        },
-      ];
-    }
-
-    if (this.options.debug) {
-      console.debug('GoogleClient buildMessages');
-      console.dir(payload, { depth: null });
-    }
+    logger.debug('[GoogleClient] buildMessages', payload);
 
     return { prompt: payload };
   }
 
-  async getCompletion(payload, abortController = null) {
+  async buildMessagesPrompt(messages, parentMessageId) {
+    const orderedMessages = this.constructor.getMessagesForConversation({
+      messages,
+      parentMessageId,
+    });
+
+    logger.debug('[GoogleClient]', {
+      orderedMessages,
+      parentMessageId,
+    });
+
+    const formattedMessages = orderedMessages.map((message) => ({
+      author: message.isCreatedByUser ? this.userLabel : this.modelLabel,
+      content: message?.content ?? message.text,
+    }));
+
+    let lastAuthor = '';
+    let groupedMessages = [];
+
+    for (let message of formattedMessages) {
+      // If last author is not same as current author, add to new group
+      if (lastAuthor !== message.author) {
+        groupedMessages.push({
+          author: message.author,
+          content: [message.content],
+        });
+        lastAuthor = message.author;
+        // If same author, append content to the last group
+      } else {
+        groupedMessages[groupedMessages.length - 1].content.push(message.content);
+      }
+    }
+
+    let identityPrefix = '';
+    if (this.options.userLabel) {
+      identityPrefix = `\nHuman's name: ${this.options.userLabel}`;
+    }
+
+    if (this.options.modelLabel) {
+      identityPrefix = `${identityPrefix}\nYou are ${this.options.modelLabel}`;
+    }
+
+    let promptPrefix = (this.options.promptPrefix || '').trim();
+    if (promptPrefix) {
+      // If the prompt prefix doesn't end with the end token, add it.
+      if (!promptPrefix.endsWith(`${this.endToken}`)) {
+        promptPrefix = `${promptPrefix.trim()}${this.endToken}\n\n`;
+      }
+      promptPrefix = `\nContext:\n${promptPrefix}`;
+    }
+
+    if (identityPrefix) {
+      promptPrefix = `${identityPrefix}${promptPrefix}`;
+    }
+
+    // Prompt AI to respond, empty if last message was from AI
+    let isEdited = lastAuthor === this.modelLabel;
+    const promptSuffix = isEdited ? '' : `${promptPrefix}\n\n${this.modelLabel}:\n`;
+    let currentTokenCount = isEdited
+      ? this.getTokenCount(promptPrefix)
+      : this.getTokenCount(promptSuffix);
+
+    let promptBody = '';
+    const maxTokenCount = this.maxPromptTokens;
+
+    const context = [];
+
+    // Iterate backwards through the messages, adding them to the prompt until we reach the max token count.
+    // Do this within a recursive async function so that it doesn't block the event loop for too long.
+    // Also, remove the next message when the message that puts us over the token limit is created by the user.
+    // Otherwise, remove only the exceeding message. This is due to Anthropic's strict payload rule to start with "Human:".
+    const nextMessage = {
+      remove: false,
+      tokenCount: 0,
+      messageString: '',
+    };
+
+    const buildPromptBody = async () => {
+      if (currentTokenCount < maxTokenCount && groupedMessages.length > 0) {
+        const message = groupedMessages.pop();
+        const isCreatedByUser = message.author === this.userLabel;
+        // Use promptPrefix if message is edited assistant'
+        const messagePrefix =
+          isCreatedByUser || !isEdited
+            ? `\n\n${message.author}:`
+            : `${promptPrefix}\n\n${message.author}:`;
+        const messageString = `${messagePrefix}\n${message.content}${this.endToken}\n`;
+        let newPromptBody = `${messageString}${promptBody}`;
+
+        context.unshift(message);
+
+        const tokenCountForMessage = this.getTokenCount(messageString);
+        const newTokenCount = currentTokenCount + tokenCountForMessage;
+
+        if (!isCreatedByUser) {
+          nextMessage.messageString = messageString;
+          nextMessage.tokenCount = tokenCountForMessage;
+        }
+
+        if (newTokenCount > maxTokenCount) {
+          if (!promptBody) {
+            // This is the first message, so we can't add it. Just throw an error.
+            throw new Error(
+              `Prompt is too long. Max token count is ${maxTokenCount}, but prompt is ${newTokenCount} tokens long.`,
+            );
+          }
+
+          // Otherwise, ths message would put us over the token limit, so don't add it.
+          // if created by user, remove next message, otherwise remove only this message
+          if (isCreatedByUser) {
+            nextMessage.remove = true;
+          }
+
+          return false;
+        }
+        promptBody = newPromptBody;
+        currentTokenCount = newTokenCount;
+
+        // Switch off isEdited after using it for the first time
+        if (isEdited) {
+          isEdited = false;
+        }
+
+        // wait for next tick to avoid blocking the event loop
+        await new Promise((resolve) => setImmediate(resolve));
+        return buildPromptBody();
+      }
+      return true;
+    };
+
+    await buildPromptBody();
+
+    if (nextMessage.remove) {
+      promptBody = promptBody.replace(nextMessage.messageString, '');
+      currentTokenCount -= nextMessage.tokenCount;
+      context.shift();
+    }
+
+    let prompt = `${promptBody}${promptSuffix}`.trim();
+
+    // Add 2 tokens for metadata after all messages have been counted.
+    currentTokenCount += 2;
+
+    // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
+    this.modelOptions.maxOutputTokens = Math.min(
+      this.maxContextTokens - currentTokenCount,
+      this.maxResponseTokens,
+    );
+
+    return { prompt, context };
+  }
+
+  async _getCompletion(payload, abortController = null) {
     if (!abortController) {
       abortController = new AbortController();
     }
     const { debug } = this.options;
     const url = this.completionsUrl;
     if (debug) {
-      console.debug();
-      console.debug(url);
-      console.debug(this.modelOptions);
-      console.debug();
+      logger.debug('GoogleClient _getCompletion', { url, payload });
     }
     const opts = {
       method: 'POST',
@@ -208,51 +584,184 @@ class GoogleClient extends BaseClient {
 
     const client = await this.getClient();
     const res = await client.request({ url, method: 'POST', data: payload });
-    console.dir(res.data, { depth: null });
+    logger.debug('GoogleClient _getCompletion', { res });
     return res.data;
   }
 
+  createLLM(clientOptions) {
+    const model = clientOptions.modelName ?? clientOptions.model;
+    if (this.project_id && this.isTextModel) {
+      return new GoogleVertexAI(clientOptions);
+    } else if (this.project_id && this.isChatModel) {
+      return new ChatGoogleVertexAI(clientOptions);
+    } else if (this.project_id) {
+      return new ChatVertexAI(clientOptions);
+    } else if (model.includes('1.5')) {
+      return new GenAI(this.apiKey).getGenerativeModel(
+        {
+          ...clientOptions,
+          model,
+        },
+        { apiVersion: 'v1beta' },
+      );
+    }
+
+    return new ChatGoogleGenerativeAI({ ...clientOptions, apiKey: this.apiKey });
+  }
+
+  async getCompletion(_payload, options = {}) {
+    const { onProgress, abortController } = options;
+    const { parameters, instances } = _payload;
+    const { messages: _messages, context, examples: _examples } = instances?.[0] ?? {};
+
+    let examples;
+
+    let clientOptions = { ...parameters, maxRetries: 2 };
+
+    if (this.project_id) {
+      clientOptions['authOptions'] = {
+        credentials: {
+          ...this.serviceKey,
+        },
+        projectId: this.project_id,
+      };
+    }
+
+    if (!parameters) {
+      clientOptions = { ...clientOptions, ...this.modelOptions };
+    }
+
+    if (this.isGenerativeModel && !this.project_id) {
+      clientOptions.modelName = clientOptions.model;
+      delete clientOptions.model;
+    }
+
+    if (_examples && _examples.length) {
+      examples = _examples
+        .map((ex) => {
+          const { input, output } = ex;
+          if (!input || !output) {
+            return undefined;
+          }
+          return {
+            input: new HumanMessage(input.content),
+            output: new AIMessage(output.content),
+          };
+        })
+        .filter((ex) => ex);
+
+      clientOptions.examples = examples;
+    }
+
+    const model = this.createLLM(clientOptions);
+
+    let reply = '';
+    const messages = this.isTextModel ? _payload.trim() : _messages;
+
+    if (!this.isVisionModel && context && messages?.length > 0) {
+      messages.unshift(new SystemMessage(context));
+    }
+
+    const modelName = clientOptions.modelName ?? clientOptions.model ?? '';
+    if (modelName?.includes('1.5') && !this.project_id) {
+      /** @type {GenerativeModel} */
+      const client = model;
+      const requestOptions = {
+        contents: _payload,
+      };
+
+      if (this.options?.promptPrefix?.length) {
+        requestOptions.systemInstruction = {
+          parts: [
+            {
+              text: this.options.promptPrefix,
+            },
+          ],
+        };
+      }
+
+      const safetySettings = _payload.safetySettings;
+      requestOptions.safetySettings = safetySettings;
+
+      const delay = modelName.includes('flash') ? 8 : 14;
+      const result = await client.generateContentStream(requestOptions);
+      for await (const chunk of result.stream) {
+        const chunkText = chunk.text();
+        await this.generateTextStream(chunkText, onProgress, {
+          delay,
+        });
+        reply += chunkText;
+      }
+      return reply;
+    }
+
+    const safetySettings = _payload.safetySettings;
+    const stream = await model.stream(messages, {
+      signal: abortController.signal,
+      timeout: 7000,
+      safetySettings: safetySettings,
+    });
+
+    let delay = this.isGenerativeModel ? 12 : 8;
+    if (modelName.includes('flash')) {
+      delay = 5;
+    }
+    for await (const chunk of stream) {
+      const chunkText = chunk?.content ?? chunk;
+      await this.generateTextStream(chunkText, onProgress, {
+        delay,
+      });
+      reply += chunkText;
+    }
+
+    return reply;
+  }
+
   getSaveOptions() {
     return {
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
 
   getBuildMessagesOptions() {
-    // console.log('GoogleClient doesn\'t use getBuildMessagesOptions');
+    // logger.debug('GoogleClient doesn\'t use getBuildMessagesOptions');
   }
 
   async sendCompletion(payload, opts = {}) {
-    console.log('GoogleClient: sendcompletion', payload, opts);
+    const modelName = payload.parameters?.model;
+
+    if (modelName && modelName.toLowerCase().includes('gemini')) {
+      const safetySettings = [
+        {
+          category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+          threshold:
+            process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_HATE_SPEECH',
+          threshold: process.env.GOOGLE_SAFETY_HATE_SPEECH || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_HARASSMENT',
+          threshold: process.env.GOOGLE_SAFETY_HARASSMENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+          threshold:
+            process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+      ];
+
+      payload.safetySettings = safetySettings;
+    }
+
     let reply = '';
-    let blocked = false;
-    try {
-      const result = await this.getCompletion(payload, opts.abortController);
-      blocked = result?.predictions?.[0]?.safetyAttributes?.blocked;
-      reply =
-        result?.predictions?.[0]?.candidates?.[0]?.content ||
-        result?.predictions?.[0]?.content ||
-        '';
-      if (blocked === true) {
-        reply = `Google blocked a proper response to your message:\n${JSON.stringify(
-          result.predictions[0].safetyAttributes,
-        )}${reply.length > 0 ? `\nAI Response:\n${reply}` : ''}`;
-      }
-      if (this.options.debug) {
-        console.debug('result');
-        console.debug(result);
-      }
-    } catch (err) {
-      console.error('Error: failed to send completion to Google');
-      console.error(err.message);
-    }
-
-    if (!blocked) {
-      await this.generateTextStream(reply, opts.onProgress, { delay: 0.5 });
-    }
-
+    reply = await this.getCompletion(payload, opts);
     return reply.trim();
   }
 

api/app/clients/OllamaClient.js

@@ -0,0 +1,154 @@
+const { z } = require('zod');
+const axios = require('axios');
+const { Ollama } = require('ollama');
+const { deriveBaseURL } = require('~/utils');
+const { logger } = require('~/config');
+
+const ollamaPayloadSchema = z.object({
+  mirostat: z.number().optional(),
+  mirostat_eta: z.number().optional(),
+  mirostat_tau: z.number().optional(),
+  num_ctx: z.number().optional(),
+  repeat_last_n: z.number().optional(),
+  repeat_penalty: z.number().optional(),
+  temperature: z.number().optional(),
+  seed: z.number().nullable().optional(),
+  stop: z.array(z.string()).optional(),
+  tfs_z: z.number().optional(),
+  num_predict: z.number().optional(),
+  top_k: z.number().optional(),
+  top_p: z.number().optional(),
+  stream: z.optional(z.boolean()),
+  model: z.string(),
+});
+
+/**
+ * @param {string} imageUrl
+ * @returns {string}
+ * @throws {Error}
+ */
+const getValidBase64 = (imageUrl) => {
+  const parts = imageUrl.split(';base64,');
+
+  if (parts.length === 2) {
+    return parts[1];
+  } else {
+    logger.error('Invalid or no Base64 string found in URL.');
+  }
+};
+
+class OllamaClient {
+  constructor(options = {}) {
+    const host = deriveBaseURL(options.baseURL ?? 'http://localhost:11434');
+    /** @type {Ollama} */
+    this.client = new Ollama({ host });
+  }
+
+  /**
+   * Fetches Ollama models from the specified base API path.
+   * @param {string} baseURL
+   * @returns {Promise<string[]>} The Ollama models.
+   */
+  static async fetchModels(baseURL) {
+    let models = [];
+    if (!baseURL) {
+      return models;
+    }
+    try {
+      const ollamaEndpoint = deriveBaseURL(baseURL);
+      /** @type {Promise<AxiosResponse<OllamaListResponse>>} */
+      const response = await axios.get(`${ollamaEndpoint}/api/tags`);
+      models = response.data.models.map((tag) => tag.name);
+      return models;
+    } catch (error) {
+      const logMessage =
+        'Failed to fetch models from Ollama API. If you are not using Ollama directly, and instead, through some aggregator or reverse proxy that handles fetching via OpenAI spec, ensure the name of the endpoint doesn\'t start with `ollama` (case-insensitive).';
+      logger.error(logMessage, error);
+      return [];
+    }
+  }
+
+  /**
+   * @param {ChatCompletionMessage[]} messages
+   * @returns {OllamaMessage[]}
+   */
+  static formatOpenAIMessages(messages) {
+    const ollamaMessages = [];
+
+    for (const message of messages) {
+      if (typeof message.content === 'string') {
+        ollamaMessages.push({
+          role: message.role,
+          content: message.content,
+        });
+        continue;
+      }
+
+      let aggregatedText = '';
+      let imageUrls = [];
+
+      for (const content of message.content) {
+        if (content.type === 'text') {
+          aggregatedText += content.text + ' ';
+        } else if (content.type === 'image_url') {
+          imageUrls.push(getValidBase64(content.image_url.url));
+        }
+      }
+
+      const ollamaMessage = {
+        role: message.role,
+        content: aggregatedText.trim(),
+      };
+
+      if (imageUrls.length > 0) {
+        ollamaMessage.images = imageUrls;
+      }
+
+      ollamaMessages.push(ollamaMessage);
+    }
+
+    return ollamaMessages;
+  }
+
+  /***
+   * @param {Object} params
+   * @param {ChatCompletionPayload} params.payload
+   * @param {onTokenProgress} params.onProgress
+   * @param {AbortController} params.abortController
+   */
+  async chatCompletion({ payload, onProgress, abortController = null }) {
+    let intermediateReply = '';
+
+    const parameters = ollamaPayloadSchema.parse(payload);
+    const messages = OllamaClient.formatOpenAIMessages(payload.messages);
+
+    if (parameters.stream) {
+      const stream = await this.client.chat({
+        messages,
+        ...parameters,
+      });
+
+      for await (const chunk of stream) {
+        const token = chunk.message.content;
+        intermediateReply += token;
+        onProgress(token);
+        if (abortController.signal.aborted) {
+          stream.controller.abort();
+          break;
+        }
+      }
+    }
+    // TODO: regular completion
+    else {
+      // const generation = await this.client.generate(payload);
+    }
+
+    return intermediateReply;
+  }
+  catch(err) {
+    logger.error('[OllamaClient.chatCompletion]', err);
+    throw err;
+  }
+}
+
+module.exports = { OllamaClient, ollamaPayloadSchema };

api/app/clients/PluginsClient.js

@@ -3,11 +3,15 @@ const { CallbackManager } = require('langchain/callbacks');
 const { BufferMemory, ChatMessageHistory } = require('langchain/memory');
 const { initializeCustomAgent, initializeFunctionsAgent } = require('./agents');
 const { addImages, buildErrorInput, buildPromptPrefix } = require('./output_parsers');
-const checkBalance = require('../../models/checkBalance');
+const { processFileURL } = require('~/server/services/Files/process');
+const { EModelEndpoint } = require('librechat-data-provider');
 const { formatLangChainMessages } = require('./prompts');
-const { isEnabled } = require('../../server/utils');
+const checkBalance = require('~/models/checkBalance');
 const { SelfReflectionTool } = require('./tools');
+const { isEnabled } = require('~/server/utils');
+const { extractBaseURL } = require('~/utils');
 const { loadTools } = require('./tools/util');
+const { logger } = require('~/config');
 
 class PluginsClient extends OpenAIClient {
   constructor(apiKey, options = {}) {
@@ -27,18 +31,10 @@ class PluginsClient extends OpenAIClient {
 
     super.setOptions(options);
 
-    if (this.functionsAgent && this.agentOptions.model && !this.useOpenRouter) {
-      this.agentOptions.model = this.getFunctionModelName(this.agentOptions.model);
-    }
-
     this.isGpt3 = this.modelOptions?.model?.includes('gpt-3');
 
     if (this.options.reverseProxyUrl) {
-      this.langchainProxy = this.options.reverseProxyUrl.match(/.*v1/)?.[0];
-      !this.langchainProxy &&
-        console.warn(`The reverse proxy URL ${this.options.reverseProxyUrl} is not valid for Plugins.
-The url must follow OpenAI specs, for example: https://localhost:8080/v1/chat/completions
-If your reverse proxy is compatible to OpenAI specs in every other way, it may still work without plugins enabled.`);
+      this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl);
     }
   }
 
@@ -46,8 +42,12 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     return {
       chatGptLabel: this.options.chatGptLabel,
       promptPrefix: this.options.promptPrefix,
+      tools: this.options.tools,
       ...this.modelOptions,
       agentOptions: this.agentOptions,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
     };
   }
 
@@ -56,7 +56,9 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
   }
 
   getFunctionModelName(input) {
-    if (input.includes('gpt-3.5-turbo')) {
+    if (/-(?!0314)\d{4}/.test(input)) {
+      return input;
+    } else if (input.includes('gpt-3.5-turbo')) {
       return 'gpt-3.5-turbo';
     } else if (input.includes('gpt-4')) {
       return 'gpt-4';
@@ -85,17 +87,15 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       initialMessageCount: this.currentMessages.length + 1,
     });
 
-    if (this.options.debug) {
-      console.debug(
-        `<-----Agent Model: ${model.modelName} | Temp: ${model.temperature} | Functions: ${this.functionsAgent}----->`,
-      );
-    }
+    logger.debug(
+      `[PluginsClient] Agent Model: ${model.modelName} | Temp: ${model.temperature} | Functions: ${this.functionsAgent}`,
+    );
 
     // Map Messages to Langchain format
     const pastMessages = formatLangChainMessages(this.currentMessages.slice(0, -1), {
       userName: this.options?.name,
     });
-    this.options.debug && console.debug('pastMessages: ', pastMessages);
+    logger.debug('[PluginsClient] pastMessages: ' + pastMessages.length);
 
     // TODO: use readOnly memory, TokenBufferMemory? (both unavailable in LangChainJS)
     const memory = new BufferMemory({
@@ -113,7 +113,8 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
         signal: this.abortController.signal,
         openAIApiKey: this.openAIApiKey,
         conversationId: this.conversationId,
-        debug: this.options?.debug,
+        fileStrategy: this.options.req.app.locals.fileStrategy,
+        processFileURL,
         message,
       },
     });
@@ -124,19 +125,16 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       return;
     }
 
-    if (this.options.debug) {
-      console.debug('Requested Tools');
-      console.debug(this.options.tools);
-      console.debug('Loaded Tools');
-      console.debug(this.tools.map((tool) => tool.name));
-    }
+    logger.debug('[PluginsClient] Requested Tools', this.options.tools);
+    logger.debug(
+      '[PluginsClient] Loaded Tools',
+      this.tools.map((tool) => tool.name),
+    );
 
     const handleAction = (action, runId, callback = null) => {
       this.saveLatestAction(action);
 
-      if (this.options.debug) {
-        console.debug('Latest Agent Action ', this.actions[this.actions.length - 1]);
-      }
+      logger.debug('[PluginsClient] Latest Agent Action ', this.actions[this.actions.length - 1]);
 
       if (typeof callback === 'function') {
         callback(action, runId);
@@ -150,9 +148,11 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       signal,
       pastMessages,
       tools: this.tools,
-      currentDateString: this.currentDateString,
       verbose: this.options.debug,
       returnIntermediateSteps: true,
+      customName: this.options.chatGptLabel,
+      currentDateString: this.currentDateString,
+      customInstructions: this.options.promptPrefix,
       callbackManager: CallbackManager.fromHandlers({
         async handleAgentAction(action, runId) {
           handleAction(action, runId, onAgentAction);
@@ -165,9 +165,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       }),
     });
 
-    if (this.options.debug) {
-      console.debug('Loaded agent.');
-    }
+    logger.debug('[PluginsClient] Loaded agent.');
   }
 
   async executorCall(message, { signal, stream, onToolStart, onToolEnd }) {
@@ -183,12 +181,10 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       });
       const input = attempts > 1 ? errorInput : message;
 
-      if (this.options.debug) {
-        console.debug(`Attempt ${attempts} of ${maxAttempts}`);
-      }
+      logger.debug(`[PluginsClient] Attempt ${attempts} of ${maxAttempts}`);
 
-      if (this.options.debug && errorMessage.length > 0) {
-        console.debug('Caught error, input:', input);
+      if (errorMessage.length > 0) {
+        logger.debug('[PluginsClient] Caught error, input: ' + JSON.stringify(input));
       }
 
       try {
@@ -211,10 +207,10 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
         ]);
         break; // Exit the loop if the function call is successful
       } catch (err) {
-        console.error(err);
+        logger.error('[PluginsClient] executorCall error:', err);
         if (attempts === maxAttempts) {
           const { run } = this.runManager.getRunByConversationId(this.conversationId);
-          const defaultOutput = `Encountered an error while attempting to respond. Error: ${err.message}`;
+          const defaultOutput = `Encountered an error while attempting to respond: ${err.message}`;
           this.result.output = run && run.error ? run.error : defaultOutput;
           this.result.errorMessage = run && run.error ? run.error : err.message;
           this.result.intermediateSteps = this.actions;
@@ -226,8 +222,11 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
 
   async handleResponseMessage(responseMessage, saveOptions, user) {
     const { output, errorMessage, ...result } = this.result;
-    this.options.debug &&
-      console.debug('[handleResponseMessage] Output:', { output, errorMessage, ...result });
+    logger.debug('[PluginsClient][handleResponseMessage] Output:', {
+      output,
+      errorMessage,
+      ...result,
+    });
     const { error } = responseMessage;
     if (!error) {
       responseMessage.tokenCount = this.getTokenCountForResponse(responseMessage);
@@ -251,7 +250,8 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       this.setOptions(opts);
       return super.sendMessage(message, opts);
     }
-    this.options.debug && console.log('Plugins sendMessage', message, opts);
+
+    logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts });
     const {
       user,
       isEdited,
@@ -265,6 +265,14 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       onToolEnd,
     } = await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     this.currentMessages.push(userMessage);
 
     let {
@@ -281,10 +289,10 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     );
 
     if (tokenCountMap) {
-      console.dir(tokenCountMap, { depth: null });
+      logger.debug('[PluginsClient] tokenCountMap', { tokenCountMap });
       if (tokenCountMap[userMessage.messageId]) {
         userMessage.tokenCount = tokenCountMap[userMessage.messageId];
-        console.log('userMessage.tokenCount', userMessage.tokenCount);
+        logger.debug('[PluginsClient] userMessage.tokenCount', userMessage.tokenCount);
       }
       this.handleTokenCountMap(tokenCountMap);
     }
@@ -305,11 +313,14 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
           amount: promptTokens,
           debug: this.options.debug,
           model: this.modelOptions.model,
+          endpoint: EModelEndpoint.openAI,
         },
       });
     }
 
     const responseMessage = {
+      endpoint: EModelEndpoint.gptPlugins,
+      iconURL: this.options.iconURL,
       messageId: responseMessageId,
       conversationId,
       parentMessageId: userMessage.messageId,
@@ -357,6 +368,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       const trimmedPartial = opts.getPartialText().replaceAll(':::plugin:::\n', '');
       responseMessage.text =
         trimmedPartial.length === 0 ? `${partialText}${this.result.output}` : partialText;
+      addImages(this.result.intermediateSteps, responseMessage);
       await this.generateTextStream(this.result.output, opts.onProgress, { delay: 5 });
       return await this.handleResponseMessage(responseMessage, saveOptions, user);
     }
@@ -368,10 +380,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       return await this.handleResponseMessage(responseMessage, saveOptions, user);
     }
 
-    if (this.options.debug) {
-      console.debug('Plugins completion phase: this.result');
-      console.debug(this.result);
-    }
+    logger.debug('[PluginsClient] Completion phase: this.result', this.result);
 
     const promptPrefix = buildPromptPrefix({
       result: this.result,
@@ -379,28 +388,20 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       functionsAgent: this.functionsAgent,
     });
 
-    if (this.options.debug) {
-      console.debug('Plugins: promptPrefix');
-      console.debug(promptPrefix);
-    }
+    logger.debug('[PluginsClient]', { promptPrefix });
 
     payload = await this.buildCompletionPrompt({
       messages: this.currentMessages,
       promptPrefix,
     });
 
-    if (this.options.debug) {
-      console.debug('buildCompletionPrompt Payload');
-      console.debug(payload);
-    }
+    logger.debug('[PluginsClient] buildCompletionPrompt Payload', payload);
     responseMessage.text = await this.sendCompletion(payload, opts);
     return await this.handleResponseMessage(responseMessage, saveOptions, user);
   }
 
   async buildCompletionPrompt({ messages, promptPrefix: _promptPrefix }) {
-    if (this.options.debug) {
-      console.debug('buildCompletionPrompt messages', messages);
-    }
+    logger.debug('[PluginsClient] buildCompletionPrompt messages', messages);
 
     const orderedMessages = messages;
     let promptPrefix = _promptPrefix.trim();

api/app/clients/TextStream.js

@@ -1,4 +1,5 @@
 const { Readable } = require('stream');
+const { logger } = require('~/config');
 
 class TextStream extends Readable {
   constructor(text, options = {}) {
@@ -38,7 +39,7 @@ class TextStream extends Readable {
       });
 
       this.on('end', () => {
-        // console.log('Stream ended');
+        // logger.debug('[processTextStream] Stream ended');
         resolve();
       });
 
@@ -50,7 +51,7 @@ class TextStream extends Readable {
     try {
       await streamPromise;
     } catch (err) {
-      console.error('Error processing text stream:', err);
+      logger.error('[processTextStream] Error in text stream:', err);
       // Handle the error appropriately, e.g., return an error message or throw an error
     }
   }

api/app/clients/agents/CustomAgent/initializeCustomAgent.js

@@ -13,10 +13,18 @@ const initializeCustomAgent = async ({
   tools,
   model,
   pastMessages,
+  customName,
+  customInstructions,
   currentDateString,
   ...rest
 }) => {
   let prompt = CustomAgent.createPrompt(tools, { currentDateString, model: model.modelName });
+  if (customName) {
+    prompt = `You are "${customName}".\n${prompt}`;
+  }
+  if (customInstructions) {
+    prompt = `${prompt}\n${customInstructions}`;
+  }
 
   const chatPrompt = ChatPromptTemplate.fromMessages([
     new SystemMessagePromptTemplate(prompt),

api/app/clients/agents/CustomAgent/outputParser.js

@@ -1,4 +1,5 @@
 const { ZeroShotAgentOutputParser } = require('langchain/agents');
+const { logger } = require('~/config');
 
 class CustomOutputParser extends ZeroShotAgentOutputParser {
   constructor(fields) {
@@ -64,9 +65,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
     const match = this.actionValues.exec(text); // old v2
 
     if (!match) {
-      console.log(
-        '\n\n<----------------------HIT NO MATCH PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT NO MATCH PARSING ERROR---------------------->\n\n' +
+          match,
       );
       const thoughts = text.replace(/[tT]hought:/, '').split('\n');
       // return {
@@ -84,9 +85,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
     let selectedTool = match?.[1].trim().toLowerCase();
 
     if (match && selectedTool === 'n/a') {
-      console.log(
-        '\n\n<----------------------HIT N/A PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT N/A PARSING ERROR---------------------->\n\n' +
+          match,
       );
       return {
         tool: 'self-reflection',
@@ -97,25 +98,25 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
 
     let toolIsValid = this.checkIfValidTool(selectedTool);
     if (match && !toolIsValid) {
-      console.log(
-        '\n\n<----------------Tool invalid: Re-assigning Selected Tool---------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------[CustomOutputParser] Tool invalid: Re-assigning Selected Tool---------------->\n\n' +
+          match,
       );
       selectedTool = this.getValidTool(selectedTool);
     }
 
     if (match && !selectedTool) {
-      console.log(
-        '\n\n<----------------------HIT INVALID TOOL PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT INVALID TOOL PARSING ERROR---------------------->\n\n' +
+          match,
       );
       selectedTool = 'self-reflection';
     }
 
     if (match && !match[2]) {
-      console.log(
-        '\n\n<----------------------HIT NO ACTION INPUT PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT NO ACTION INPUT PARSING ERROR---------------------->\n\n' +
+          match,
       );
 
       // In case there is no action input, let's double-check if there is an action input in 'text' variable
@@ -139,7 +140,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
     }
 
     if (match && selectedTool.length > this.longestToolName.length) {
-      console.log('\n\n<----------------------HIT LONG PARSING ERROR---------------------->\n\n');
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT LONG PARSING ERROR---------------------->\n\n',
+      );
 
       let action, input, thought;
       let firstIndex = Infinity;
@@ -156,9 +159,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
       // In case there is no action input, let's double-check if there is an action input in 'text' variable
       const actionInputMatch = this.actionInputRegex.exec(text);
       if (action && actionInputMatch) {
-        console.log(
-          '\n\n<------Matched Action Input in Long Parsing Error------>\n\n',
-          actionInputMatch,
+        logger.debug(
+          '\n\n<------[CustomOutputParser] Matched Action Input in Long Parsing Error------>\n\n' +
+            actionInputMatch,
         );
         return {
           tool: action,
@@ -185,15 +188,14 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
 
         const inputMatch = this.actionValues.exec(returnValues.log); //new
         if (inputMatch) {
-          console.log('inputMatch');
-          console.dir(inputMatch, { depth: null });
+          logger.debug('[CustomOutputParser] inputMatch', inputMatch);
           returnValues.toolInput = inputMatch[1].replaceAll('"', '').trim();
           returnValues.log = returnValues.log.replace(this.actionValues, '');
         }
 
         return returnValues;
       } else {
-        console.log('No valid tool mentioned.', this.tools, text);
+        logger.debug('[CustomOutputParser] No valid tool mentioned.', this.tools, text);
         return {
           tool: 'self-reflection',
           toolInput: 'Hypothetical actions: \n"' + text + '"\n',
@@ -202,8 +204,8 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
       }
 
       // if (action && input) {
-      //   console.log('Action:', action);
-      //   console.log('Input:', input);
+      //   logger.debug('Action:', action);
+      //   logger.debug('Input:', input);
       // }
     }
 

api/app/clients/agents/Functions/FunctionsAgent.js

@@ -7,6 +7,8 @@ const {
   SystemMessagePromptTemplate,
   HumanMessagePromptTemplate,
 } = require('langchain/prompts');
+const { logger } = require('~/config');
+
 const PREFIX = 'You are a helpful AI assistant.';
 
 function parseOutput(message) {
@@ -112,7 +114,7 @@ class FunctionsAgent extends Agent {
       valuesForLLM,
       callbackManager,
     );
-    console.log('message', message);
+    logger.debug('[FunctionsAgent] plan message', message);
     return parseOutput(message);
   }
 }

api/app/clients/agents/Functions/initializeFunctionsAgent.js

@@ -10,6 +10,8 @@ const initializeFunctionsAgent = async ({
   tools,
   model,
   pastMessages,
+  customName,
+  customInstructions,
   currentDateString,
   ...rest
 }) => {
@@ -24,7 +26,13 @@ const initializeFunctionsAgent = async ({
     returnMessages: true,
   });
 
-  const prefix = addToolDescriptions(`Current Date: ${currentDateString}\n${PREFIX}`, tools);
+  let prefix = addToolDescriptions(`Current Date: ${currentDateString}\n${PREFIX}`, tools);
+  if (customName) {
+    prefix = `You are "${customName}".\n${prefix}`;
+  }
+  if (customInstructions) {
+    prefix = `${prefix}\n${customInstructions}`;
+  }
 
   return await initializeAgentExecutorWithOptions(tools, model, {
     agentType: 'openai-functions',

api/app/clients/callbacks/createStartHandler.js

@@ -1,7 +1,9 @@
 const { promptTokensEstimate } = require('openai-chat-tokens');
-const checkBalance = require('../../../models/checkBalance');
-const { isEnabled } = require('../../../server/utils');
-const { formatFromLangChain } = require('../prompts');
+const { EModelEndpoint, supportsBalanceCheck } = require('librechat-data-provider');
+const { formatFromLangChain } = require('~/app/clients/prompts');
+const checkBalance = require('~/models/checkBalance');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
 
 const createStartHandler = ({
   context,
@@ -15,9 +17,15 @@ const createStartHandler = ({
     const { model, functions, function_call } = invocation_params;
     const messages = _messages[0].map(formatFromLangChain);
 
-    if (manager.debug) {
-      console.log(`handleChatModelStart: ${context}`);
-      console.dir({ model, functions, function_call }, { depth: null });
+    logger.debug(`[createStartHandler] handleChatModelStart: ${context}`, {
+      model,
+      function_call,
+    });
+
+    if (context !== 'title') {
+      logger.debug(`[createStartHandler] handleChatModelStart: ${context}`, {
+        functions,
+      });
     }
 
     const payload = { messages };
@@ -34,13 +42,15 @@ const createStartHandler = ({
     }
 
     prelimPromptTokens += promptTokensEstimate(payload);
-    if (manager.debug) {
-      console.log('Prelim Prompt Tokens & Token Buffer', prelimPromptTokens, tokenBuffer);
-    }
+    logger.debug('[createStartHandler]', {
+      prelimPromptTokens,
+      tokenBuffer,
+    });
     prelimPromptTokens += tokenBuffer;
 
     try {
-      if (isEnabled(process.env.CHECK_BALANCE)) {
+      // TODO: if plugins extends to non-OpenAI models, this will need to be updated
+      if (isEnabled(process.env.CHECK_BALANCE) && supportsBalanceCheck[EModelEndpoint.openAI]) {
         const generations =
           initialMessageCount && messages.length > initialMessageCount
             ? messages.slice(initialMessageCount)
@@ -55,11 +65,12 @@ const createStartHandler = ({
             debug: manager.debug,
             generations,
             model,
+            endpoint: EModelEndpoint.openAI,
           },
         });
       }
     } catch (err) {
-      console.error(`[${context}] checkBalance error`, err);
+      logger.error(`[createStartHandler][${context}] checkBalance error`, err);
       manager.abortController.abort();
       if (context === 'summary' || context === 'plugins') {
         manager.addRun(runId, { conversationId, error: err.message });

api/app/clients/chains/runTitleChain.js

@@ -1,6 +1,7 @@
 const { z } = require('zod');
 const { langPrompt, createTitlePrompt, escapeBraces, getSnippet } = require('../prompts');
 const { createStructuredOutputChainFromZod } = require('langchain/chains/openai_functions');
+const { logger } = require('~/config');
 
 const langSchema = z.object({
   language: z.string().describe('The language of the input text (full noun, no abbreviations).'),
@@ -30,8 +31,7 @@ const runTitleChain = async ({ llm, text, convo, signal, callbacks }) => {
   try {
     snippet = getSnippet(text);
   } catch (e) {
-    console.log('Error getting snippet of text for titleChain');
-    console.log(e);
+    logger.error('[runTitleChain] Error getting snippet of text for titleChain', e);
   }
   const languageChain = createLanguageChain({ llm, callbacks });
   const titleChain = createTitleChain({ llm, callbacks, convo: escapeBraces(convo) });

api/app/clients/llm/RunManager.js

@@ -1,5 +1,6 @@
-const { createStartHandler } = require('../callbacks');
-const spendTokens = require('../../../models/spendTokens');
+const { createStartHandler } = require('~/app/clients/callbacks');
+const spendTokens = require('~/models/spendTokens');
+const { logger } = require('~/config');
 
 class RunManager {
   constructor(fields) {
@@ -35,7 +36,7 @@ class RunManager {
     if (this.runs.has(runId)) {
       this.runs.delete(runId);
     } else {
-      console.error(`Run with ID ${runId} does not exist.`);
+      logger.error(`[api/app/clients/llm/RunManager] Run with ID ${runId} does not exist.`);
     }
   }
 
@@ -57,10 +58,19 @@ class RunManager {
       {
         handleChatModelStart: createStartHandler({ ...metadata, manager: this }),
         handleLLMEnd: async (output, runId, _parentRunId) => {
-          if (this.debug) {
-            console.log(`handleLLMEnd: ${JSON.stringify(metadata)}`);
-            console.dir({ output, runId, _parentRunId }, { depth: null });
+          const { llmOutput, ..._output } = output;
+          logger.debug(`[RunManager] handleLLMEnd: ${JSON.stringify(metadata)}`, {
+            runId,
+            _parentRunId,
+            llmOutput,
+          });
+
+          if (metadata.context !== 'title') {
+            logger.debug('[RunManager] handleLLMEnd:', {
+              output: _output,
+            });
           }
+
           const { tokenUsage } = output.llmOutput;
           const run = this.getRunById(runId);
           this.removeRun(runId);
@@ -74,8 +84,7 @@ class RunManager {
           await spendTokens(txData, tokenUsage);
         },
         handleLLMError: async (err) => {
-          this.debug && console.log(`handleLLMError: ${JSON.stringify(metadata)}`);
-          this.debug && console.error(err);
+          logger.error(`[RunManager] handleLLMError: ${JSON.stringify(metadata)}`, err);
           if (metadata.context === 'title') {
             return;
           } else if (metadata.context === 'plugins') {

api/app/clients/llm/createCoherePayload.js

@@ -0,0 +1,85 @@
+const { CohereConstants } = require('librechat-data-provider');
+const { titleInstruction } = require('../prompts/titlePrompts');
+
+// Mapping OpenAI roles to Cohere roles
+const roleMap = {
+  user: CohereConstants.ROLE_USER,
+  assistant: CohereConstants.ROLE_CHATBOT,
+  system: CohereConstants.ROLE_SYSTEM, // Recognize and map the system role explicitly
+};
+
+/**
+ * Adjusts an OpenAI ChatCompletionPayload to conform with Cohere's expected chat payload format.
+ * Now includes handling for "system" roles explicitly mentioned.
+ *
+ * @param {Object} options - Object containing the model options.
+ * @param {ChatCompletionPayload} options.modelOptions - The OpenAI model payload options.
+ * @returns {CohereChatStreamRequest} Cohere-compatible chat API payload.
+ */
+function createCoherePayload({ modelOptions }) {
+  /** @type {string | undefined} */
+  let preamble;
+  let latestUserMessageContent = '';
+  const {
+    stream,
+    stop,
+    top_p,
+    temperature,
+    frequency_penalty,
+    presence_penalty,
+    max_tokens,
+    messages,
+    model,
+    ...rest
+  } = modelOptions;
+
+  // Filter out the latest user message and transform remaining messages to Cohere's chat_history format
+  let chatHistory = messages.reduce((acc, message, index, arr) => {
+    const isLastUserMessage = index === arr.length - 1 && message.role === 'user';
+
+    const messageContent =
+      typeof message.content === 'string'
+        ? message.content
+        : message.content.map((part) => (part.type === 'text' ? part.text : '')).join(' ');
+
+    if (isLastUserMessage) {
+      latestUserMessageContent = messageContent;
+    } else {
+      acc.push({
+        role: roleMap[message.role] || CohereConstants.ROLE_USER,
+        message: messageContent,
+      });
+    }
+
+    return acc;
+  }, []);
+
+  if (
+    chatHistory.length === 1 &&
+    chatHistory[0].role === CohereConstants.ROLE_SYSTEM &&
+    !latestUserMessageContent.length
+  ) {
+    const message = chatHistory[0].message;
+    latestUserMessageContent = message.includes(titleInstruction)
+      ? CohereConstants.TITLE_MESSAGE
+      : '.';
+    preamble = message;
+  }
+
+  return {
+    message: latestUserMessageContent,
+    model: model,
+    chatHistory,
+    stream: stream ?? false,
+    temperature: temperature,
+    frequencyPenalty: frequency_penalty,
+    presencePenalty: presence_penalty,
+    maxTokens: max_tokens,
+    stopSequences: stop,
+    preamble,
+    p: top_p,
+    ...rest,
+  };
+}
+
+module.exports = createCoherePayload;

api/app/clients/llm/createLLM.js

@@ -1,5 +1,28 @@
 const { ChatOpenAI } = require('langchain/chat_models/openai');
+const { sanitizeModelName, constructAzureURL } = require('~/utils');
+const { isEnabled } = require('~/server/utils');
 
+/**
+ * Creates a new instance of a language model (LLM) for chat interactions.
+ *
+ * @param {Object} options - The options for creating the LLM.
+ * @param {ModelOptions} options.modelOptions - The options specific to the model, including modelName, temperature, presence_penalty, frequency_penalty, and other model-related settings.
+ * @param {ConfigOptions} options.configOptions - Configuration options for the API requests, including proxy settings and custom headers.
+ * @param {Callbacks} options.callbacks - Callback functions for managing the lifecycle of the LLM, including token buffers, context, and initial message count.
+ * @param {boolean} [options.streaming=false] - Determines if the LLM should operate in streaming mode.
+ * @param {string} options.openAIApiKey - The API key for OpenAI, used for authentication.
+ * @param {AzureOptions} [options.azure={}] - Optional Azure-specific configurations. If provided, Azure configurations take precedence over OpenAI configurations.
+ *
+ * @returns {ChatOpenAI} An instance of the ChatOpenAI class, configured with the provided options.
+ *
+ * @example
+ * const llm = createLLM({
+ *   modelOptions: { modelName: 'gpt-3.5-turbo', temperature: 0.2 },
+ *   configOptions: { basePath: 'https://example.api/path' },
+ *   callbacks: { onMessage: handleMessage },
+ *   openAIApiKey: 'your-api-key'
+ * });
+ */
 function createLLM({
   modelOptions,
   configOptions,
@@ -13,22 +36,42 @@ function createLLM({
     apiKey: openAIApiKey,
   };
 
+  /**  @type {AzureOptions} */
+  let azureOptions = {};
   if (azure) {
+    const useModelName = isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME);
+
     credentials = {};
     configuration = {};
+    azureOptions = azure;
+
+    azureOptions.azureOpenAIApiDeploymentName = useModelName
+      ? sanitizeModelName(modelOptions.modelName)
+      : azureOptions.azureOpenAIApiDeploymentName;
   }
 
-  // console.debug('createLLM: configOptions');
-  // console.debug(configOptions);
+  if (azure && process.env.AZURE_OPENAI_DEFAULT_MODEL) {
+    modelOptions.modelName = process.env.AZURE_OPENAI_DEFAULT_MODEL;
+  }
+
+  if (azure && configOptions.basePath) {
+    const azureURL = constructAzureURL({
+      baseURL: configOptions.basePath,
+      azureOptions,
+    });
+    azureOptions.azureOpenAIBasePath = azureURL.split(
+      `/${azureOptions.azureOpenAIApiDeploymentName}`,
+    )[0];
+  }
 
   return new ChatOpenAI(
     {
       streaming,
-      verbose: true,
       credentials,
       configuration,
-      ...azure,
+      ...azureOptions,
       ...modelOptions,
+      ...credentials,
       callbacks,
     },
     configOptions,

api/app/clients/llm/index.js

@@ -1,7 +1,9 @@
 const createLLM = require('./createLLM');
 const RunManager = require('./RunManager');
+const createCoherePayload = require('./createCoherePayload');
 
 module.exports = {
   createLLM,
   RunManager,
+  createCoherePayload,
 };

api/app/clients/memory/summaryBuffer.js

@@ -1,6 +1,7 @@
 const { ConversationSummaryBufferMemory, ChatMessageHistory } = require('langchain/memory');
 const { formatLangChainMessages, SUMMARY_PROMPT } = require('../prompts');
 const { predictNewSummary } = require('../chains');
+const { logger } = require('~/config');
 
 const createSummaryBufferMemory = ({ llm, prompt, messages, ...rest }) => {
   const chatHistory = new ChatMessageHistory(messages);
@@ -22,9 +23,8 @@ const summaryBuffer = async ({
   prompt = SUMMARY_PROMPT,
   signal,
 }) => {
-  if (debug && previous_summary) {
-    console.log('<-----------PREVIOUS SUMMARY----------->\n\n');
-    console.log(previous_summary);
+  if (previous_summary) {
+    logger.debug('[summaryBuffer]', { previous_summary });
   }
 
   const formattedMessages = formatLangChainMessages(context, formatOptions);
@@ -46,8 +46,7 @@ const summaryBuffer = async ({
   const messages = await chatPromptMemory.chatHistory.getMessages();
 
   if (debug) {
-    console.log('<-----------SUMMARY BUFFER MESSAGES----------->\n\n');
-    console.log(JSON.stringify(messages));
+    logger.debug('[summaryBuffer]', { summary_buffer_messages: messages.length });
   }
 
   const predictSummary = await predictNewSummary({
@@ -58,8 +57,7 @@ const summaryBuffer = async ({
   });
 
   if (debug) {
-    console.log('<-----------SUMMARY----------->\n\n');
-    console.log(JSON.stringify(predictSummary));
+    logger.debug('[summaryBuffer]', { summary: predictSummary });
   }
 
   return { role: 'system', content: predictSummary };

api/app/clients/output_parsers/addImages.js

@@ -1,26 +1,71 @@
+const { logger } = require('~/config');
+
+/**
+ * The `addImages` function corrects any erroneous image URLs in the `responseMessage.text`
+ * and appends image observations from `intermediateSteps` if they are not already present.
+ *
+ * @function
+ * @module addImages
+ *
+ * @param {Array.<Object>} intermediateSteps - An array of objects, each containing an observation.
+ * @param {Object} responseMessage - An object containing the text property which might have image URLs.
+ *
+ * @property {string} intermediateSteps[].observation - The observation string which might contain an image markdown.
+ * @property {string} responseMessage.text - The text which might contain image URLs.
+ *
+ * @example
+ *
+ * const intermediateSteps = [
+ *   { observation: '![desc](/images/test.png)' }
+ * ];
+ * const responseMessage = { text: 'Some text with ![desc](sandbox:/images/test.png)' };
+ *
+ * addImages(intermediateSteps, responseMessage);
+ *
+ * logger.debug(responseMessage.text);
+ * // Outputs: 'Some text with ![desc](/images/test.png)\n![desc](/images/test.png)'
+ *
+ * @returns {void}
+ */
 function addImages(intermediateSteps, responseMessage) {
   if (!intermediateSteps || !responseMessage) {
     return;
   }
 
+  // Correct any erroneous URLs in the responseMessage.text first
   intermediateSteps.forEach((step) => {
     const { observation } = step;
     if (!observation || !observation.includes('![')) {
       return;
     }
 
-    // Extract the image file path from the observation
-    const observedImagePath = observation.match(/\(\/images\/.*\.\w*\)/g)[0];
+    const match = observation.match(/\/images\/.*\.\w*/);
+    if (!match) {
+      return;
+    }
+    const essentialImagePath = match[0];
 
-    // Check if the responseMessage already includes the image file path
-    if (!responseMessage.text.includes(observedImagePath)) {
-      // If the image file path is not found, append the whole observation
-      responseMessage.text += '\n' + observation;
-      if (this.options.debug) {
-        console.debug('added image from intermediateSteps');
+    const regex = /!\[.*?\]\((.*?)\)/g;
+    let matchErroneous;
+    while ((matchErroneous = regex.exec(responseMessage.text)) !== null) {
+      if (matchErroneous[1] && !matchErroneous[1].startsWith('/images/')) {
+        responseMessage.text = responseMessage.text.replace(matchErroneous[1], essentialImagePath);
       }
     }
   });
+
+  // Now, check if the responseMessage already includes the correct image file path and append if not
+  intermediateSteps.forEach((step) => {
+    const { observation } = step;
+    if (!observation || !observation.includes('![')) {
+      return;
+    }
+    const observedImagePath = observation.match(/!\[.*\]\([^)]*\)/g);
+    if (observedImagePath && !responseMessage.text.includes(observedImagePath[0])) {
+      responseMessage.text += '\n' + observation;
+      logger.debug('[addImages] added image from intermediateSteps:', observation);
+    }
+  });
 }
 
 module.exports = addImages;

api/app/clients/output_parsers/addImages.spec.js

@@ -0,0 +1,84 @@
+let addImages = require('./addImages');
+
+describe('addImages', () => {
+  let intermediateSteps;
+  let responseMessage;
+  let options;
+
+  beforeEach(() => {
+    intermediateSteps = [];
+    responseMessage = { text: '' };
+    options = { debug: false };
+    this.options = options;
+    addImages = addImages.bind(this);
+  });
+
+  it('should handle null or undefined parameters', () => {
+    addImages(null, responseMessage);
+    expect(responseMessage.text).toBe('');
+
+    addImages(intermediateSteps, null);
+    expect(responseMessage.text).toBe('');
+
+    addImages(null, null);
+    expect(responseMessage.text).toBe('');
+  });
+
+  it('should append correct image markdown if not present in responseMessage', () => {
+    intermediateSteps.push({ observation: '![desc](/images/test.png)' });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe('\n![desc](/images/test.png)');
+  });
+
+  it('should not append image markdown if already present in responseMessage', () => {
+    responseMessage.text = '![desc](/images/test.png)';
+    intermediateSteps.push({ observation: '![desc](/images/test.png)' });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe('![desc](/images/test.png)');
+  });
+
+  it('should correct and append image markdown with erroneous URL', () => {
+    responseMessage.text = '![desc](sandbox:/images/test.png)';
+    intermediateSteps.push({ observation: '![desc](/images/test.png)' });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe('![desc](/images/test.png)');
+  });
+
+  it('should correct multiple erroneous URLs in responseMessage', () => {
+    responseMessage.text =
+      '![desc1](sandbox:/images/test1.png) ![desc2](version:/images/test2.png)';
+    intermediateSteps.push({ observation: '![desc1](/images/test1.png)' });
+    intermediateSteps.push({ observation: '![desc2](/images/test2.png)' });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe('![desc1](/images/test1.png) ![desc2](/images/test2.png)');
+  });
+
+  it('should not append non-image markdown observations', () => {
+    intermediateSteps.push({ observation: '[desc](/images/test.png)' });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe('');
+  });
+
+  it('should handle multiple observations', () => {
+    intermediateSteps.push({ observation: '![desc1](/images/test1.png)' });
+    intermediateSteps.push({ observation: '![desc2](/images/test2.png)' });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe('\n![desc1](/images/test1.png)\n![desc2](/images/test2.png)');
+  });
+
+  it('should not append if observation does not contain image markdown', () => {
+    intermediateSteps.push({ observation: 'This is a test observation without image markdown.' });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe('');
+  });
+
+  it('should append correctly from a real scenario', () => {
+    responseMessage.text =
+      'Here is the generated image based on your request. It depicts a surreal landscape filled with floating musical notes. The style is impressionistic, with vibrant sunset hues dominating the scene. At the center, there\'s a silhouette of a grand piano, adding a dreamy emotion to the overall image. This could serve as a unique and creative music album cover. Would you like to make any changes or generate another image?';
+    const originalText = responseMessage.text;
+    const imageMarkdown = '![generated image](/images/img-RnVWaYo2Yg4x3e0isICiMuf5.png)';
+    intermediateSteps.push({ observation: imageMarkdown });
+    addImages(intermediateSteps, responseMessage);
+    expect(responseMessage.text).toBe(`${originalText}\n${imageMarkdown}`);
+  });
+});

api/app/clients/prompts/createContextHandlers.js

@@ -0,0 +1,159 @@
+const axios = require('axios');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
+
+const footer = `Use the context as your learned knowledge to better answer the user.
+
+In your response, remember to follow these guidelines:
+- If you don't know the answer, simply say that you don't know.
+- If you are unsure how to answer, ask for clarification.
+- Avoid mentioning that you obtained the information from the context.
+
+Answer appropriately in the user's language.
+`;
+
+function createContextHandlers(req, userMessageContent) {
+  if (!process.env.RAG_API_URL) {
+    return;
+  }
+
+  const queryPromises = [];
+  const processedFiles = [];
+  const processedIds = new Set();
+  const jwtToken = req.headers.authorization.split(' ')[1];
+  const useFullContext = isEnabled(process.env.RAG_USE_FULL_CONTEXT);
+
+  const query = async (file) => {
+    if (useFullContext) {
+      return axios.get(`${process.env.RAG_API_URL}/documents/${file.file_id}/context`, {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+        },
+      });
+    }
+
+    return axios.post(
+      `${process.env.RAG_API_URL}/query`,
+      {
+        file_id: file.file_id,
+        query: userMessageContent,
+        k: 4,
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+  };
+
+  const processFile = async (file) => {
+    if (file.embedded && !processedIds.has(file.file_id)) {
+      try {
+        const promise = query(file);
+        queryPromises.push(promise);
+        processedFiles.push(file);
+        processedIds.add(file.file_id);
+      } catch (error) {
+        logger.error(`Error processing file ${file.filename}:`, error);
+      }
+    }
+  };
+
+  const createContext = async () => {
+    try {
+      if (!queryPromises.length || !processedFiles.length) {
+        return '';
+      }
+
+      const oneFile = processedFiles.length === 1;
+      const header = `The user has attached ${oneFile ? 'a' : processedFiles.length} file${
+        !oneFile ? 's' : ''
+      } to the conversation:`;
+
+      const files = `${
+        oneFile
+          ? ''
+          : `
+      <files>`
+      }${processedFiles
+        .map(
+          (file) => `
+              <file>
+                <filename>${file.filename}</filename>
+                <type>${file.type}</type>
+              </file>`,
+        )
+        .join('')}${
+        oneFile
+          ? ''
+          : `
+        </files>`
+      }`;
+
+      const resolvedQueries = await Promise.all(queryPromises);
+
+      const context = resolvedQueries
+        .map((queryResult, index) => {
+          const file = processedFiles[index];
+          let contextItems = queryResult.data;
+
+          const generateContext = (currentContext) =>
+            `
+          <file>
+            <filename>${file.filename}</filename>
+            <context>${currentContext}
+            </context>
+          </file>`;
+
+          if (useFullContext) {
+            return generateContext(`\n${contextItems}`);
+          }
+
+          contextItems = queryResult.data
+            .map((item) => {
+              const pageContent = item[0].page_content;
+              return `
+            <contextItem>
+              <![CDATA[${pageContent?.trim()}]]>
+            </contextItem>`;
+            })
+            .join('');
+
+          return generateContext(contextItems);
+        })
+        .join('');
+
+      if (useFullContext) {
+        const prompt = `${header}
+          ${context}
+          ${footer}`;
+
+        return prompt;
+      }
+
+      const prompt = `${header}
+        ${files}
+
+        A semantic search was executed with the user's message as the query, retrieving the following context inside <context></context> XML tags.
+
+        <context>${context}
+        </context>
+
+        ${footer}`;
+
+      return prompt;
+    } catch (error) {
+      logger.error('Error creating context:', error);
+      throw error;
+    }
+  };
+
+  return {
+    processFile,
+    createContext,
+  };
+}
+
+module.exports = createContextHandlers;

api/app/clients/prompts/createVisionPrompt.js

@@ -0,0 +1,34 @@
+/**
+ * Generates a prompt instructing the user to describe an image in detail, tailored to different types of visual content.
+ * @param {boolean} pluralized - Whether to pluralize the prompt for multiple images.
+ * @returns {string} - The generated vision prompt.
+ */
+const createVisionPrompt = (pluralized = false) => {
+  return `Please describe the image${
+    pluralized ? 's' : ''
+  } in detail, covering relevant aspects such as:
+
+  For photographs, illustrations, or artwork:
+  - The main subject(s) and their appearance, positioning, and actions
+  - The setting, background, and any notable objects or elements
+  - Colors, lighting, and overall mood or atmosphere
+  - Any interesting details, textures, or patterns
+  - The style, technique, or medium used (if discernible)
+  
+  For screenshots or images containing text:
+  - The content and purpose of the text
+  - The layout, formatting, and organization of the information
+  - Any notable visual elements, such as logos, icons, or graphics
+  - The overall context or message conveyed by the screenshot
+  
+  For graphs, charts, or data visualizations:
+  - The type of graph or chart (e.g., bar graph, line chart, pie chart)
+  - The variables being compared or analyzed
+  - Any trends, patterns, or outliers in the data
+  - The axis labels, scales, and units of measurement
+  - The title, legend, and any additional context provided
+  
+  Be as specific and descriptive as possible while maintaining clarity and concision.`;
+};
+
+module.exports = createVisionPrompt;

api/app/clients/prompts/formatGoogleInputs.js

@@ -0,0 +1,42 @@
+/**
+ * Formats an object to match the struct_val, list_val, string_val, float_val, and int_val format.
+ *
+ * @param {Object} obj - The object to be formatted.
+ * @returns {Object} The formatted object.
+ *
+ * Handles different types:
+ * - Arrays are wrapped in list_val and each element is processed.
+ * - Objects are recursively processed.
+ * - Strings are wrapped in string_val.
+ * - Numbers are wrapped in float_val or int_val depending on whether they are floating-point or integers.
+ */
+function formatGoogleInputs(obj) {
+  const formattedObj = {};
+
+  for (const key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      const value = obj[key];
+
+      // Handle arrays
+      if (Array.isArray(value)) {
+        formattedObj[key] = { list_val: value.map((item) => formatGoogleInputs(item)) };
+      }
+      // Handle objects
+      else if (typeof value === 'object' && value !== null) {
+        formattedObj[key] = formatGoogleInputs(value);
+      }
+      // Handle numbers
+      else if (typeof value === 'number') {
+        formattedObj[key] = Number.isInteger(value) ? { int_val: value } : { float_val: value };
+      }
+      // Handle other types (e.g., strings)
+      else {
+        formattedObj[key] = { string_val: [value] };
+      }
+    }
+  }
+
+  return { struct_val: formattedObj };
+}
+
+module.exports = formatGoogleInputs;

api/app/clients/prompts/formatGoogleInputs.spec.js

@@ -0,0 +1,274 @@
+const formatGoogleInputs = require('./formatGoogleInputs');
+
+describe('formatGoogleInputs', () => {
+  it('formats message correctly', () => {
+    const input = {
+      messages: [
+        {
+          content: 'hi',
+          author: 'user',
+        },
+      ],
+      context: 'context',
+      examples: [
+        {
+          input: {
+            author: 'user',
+            content: 'user input',
+          },
+          output: {
+            author: 'bot',
+            content: 'bot output',
+          },
+        },
+      ],
+      parameters: {
+        temperature: 0.2,
+        topP: 0.8,
+        topK: 40,
+        maxOutputTokens: 1024,
+      },
+    };
+
+    const expectedOutput = {
+      struct_val: {
+        messages: {
+          list_val: [
+            {
+              struct_val: {
+                content: {
+                  string_val: ['hi'],
+                },
+                author: {
+                  string_val: ['user'],
+                },
+              },
+            },
+          ],
+        },
+        context: {
+          string_val: ['context'],
+        },
+        examples: {
+          list_val: [
+            {
+              struct_val: {
+                input: {
+                  struct_val: {
+                    author: {
+                      string_val: ['user'],
+                    },
+                    content: {
+                      string_val: ['user input'],
+                    },
+                  },
+                },
+                output: {
+                  struct_val: {
+                    author: {
+                      string_val: ['bot'],
+                    },
+                    content: {
+                      string_val: ['bot output'],
+                    },
+                  },
+                },
+              },
+            },
+          ],
+        },
+        parameters: {
+          struct_val: {
+            temperature: {
+              float_val: 0.2,
+            },
+            topP: {
+              float_val: 0.8,
+            },
+            topK: {
+              int_val: 40,
+            },
+            maxOutputTokens: {
+              int_val: 1024,
+            },
+          },
+        },
+      },
+    };
+
+    const result = formatGoogleInputs(input);
+    expect(JSON.stringify(result)).toEqual(JSON.stringify(expectedOutput));
+  });
+
+  it('formats real payload parts', () => {
+    const input = {
+      instances: [
+        {
+          context: 'context',
+          examples: [
+            {
+              input: {
+                author: 'user',
+                content: 'user input',
+              },
+              output: {
+                author: 'bot',
+                content: 'user output',
+              },
+            },
+          ],
+          messages: [
+            {
+              author: 'user',
+              content: 'hi',
+            },
+          ],
+        },
+      ],
+      parameters: {
+        candidateCount: 1,
+        maxOutputTokens: 1024,
+        temperature: 0.2,
+        topP: 0.8,
+        topK: 40,
+      },
+    };
+    const expectedOutput = {
+      struct_val: {
+        instances: {
+          list_val: [
+            {
+              struct_val: {
+                context: { string_val: ['context'] },
+                examples: {
+                  list_val: [
+                    {
+                      struct_val: {
+                        input: {
+                          struct_val: {
+                            author: { string_val: ['user'] },
+                            content: { string_val: ['user input'] },
+                          },
+                        },
+                        output: {
+                          struct_val: {
+                            author: { string_val: ['bot'] },
+                            content: { string_val: ['user output'] },
+                          },
+                        },
+                      },
+                    },
+                  ],
+                },
+                messages: {
+                  list_val: [
+                    {
+                      struct_val: {
+                        author: { string_val: ['user'] },
+                        content: { string_val: ['hi'] },
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          ],
+        },
+        parameters: {
+          struct_val: {
+            candidateCount: { int_val: 1 },
+            maxOutputTokens: { int_val: 1024 },
+            temperature: { float_val: 0.2 },
+            topP: { float_val: 0.8 },
+            topK: { int_val: 40 },
+          },
+        },
+      },
+    };
+
+    const result = formatGoogleInputs(input);
+    expect(JSON.stringify(result)).toEqual(JSON.stringify(expectedOutput));
+  });
+
+  it('helps create valid payload parts', () => {
+    const instances = {
+      context: 'context',
+      examples: [
+        {
+          input: {
+            author: 'user',
+            content: 'user input',
+          },
+          output: {
+            author: 'bot',
+            content: 'user output',
+          },
+        },
+      ],
+      messages: [
+        {
+          author: 'user',
+          content: 'hi',
+        },
+      ],
+    };
+
+    const expectedInstances = {
+      struct_val: {
+        context: { string_val: ['context'] },
+        examples: {
+          list_val: [
+            {
+              struct_val: {
+                input: {
+                  struct_val: {
+                    author: { string_val: ['user'] },
+                    content: { string_val: ['user input'] },
+                  },
+                },
+                output: {
+                  struct_val: {
+                    author: { string_val: ['bot'] },
+                    content: { string_val: ['user output'] },
+                  },
+                },
+              },
+            },
+          ],
+        },
+        messages: {
+          list_val: [
+            {
+              struct_val: {
+                author: { string_val: ['user'] },
+                content: { string_val: ['hi'] },
+              },
+            },
+          ],
+        },
+      },
+    };
+
+    const parameters = {
+      candidateCount: 1,
+      maxOutputTokens: 1024,
+      temperature: 0.2,
+      topP: 0.8,
+      topK: 40,
+    };
+    const expectedParameters = {
+      struct_val: {
+        candidateCount: { int_val: 1 },
+        maxOutputTokens: { int_val: 1024 },
+        temperature: { float_val: 0.2 },
+        topP: { float_val: 0.8 },
+        topK: { int_val: 40 },
+      },
+    };
+
+    const instancesResult = formatGoogleInputs(instances);
+    const parametersResult = formatGoogleInputs(parameters);
+    expect(JSON.stringify(instancesResult)).toEqual(JSON.stringify(expectedInstances));
+    expect(JSON.stringify(parametersResult)).toEqual(JSON.stringify(expectedParameters));
+  });
+});

api/app/clients/prompts/formatMessages.js

@@ -1,5 +1,28 @@
+const { EModelEndpoint } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
 
+/**
+ * Formats a message to OpenAI Vision API payload format.
+ *
+ * @param {Object} params - The parameters for formatting.
+ * @param {Object} params.message - The message object to format.
+ * @param {string} [params.message.role] - The role of the message sender (must be 'user').
+ * @param {string} [params.message.content] - The text content of the message.
+ * @param {EModelEndpoint} [params.endpoint] - Identifier for specific endpoint handling
+ * @param {Array<string>} [params.image_urls] - The image_urls to attach to the message.
+ * @returns {(Object)} - The formatted message.
+ */
+const formatVisionMessage = ({ message, image_urls, endpoint }) => {
+  if (endpoint === EModelEndpoint.anthropic) {
+    message.content = [...image_urls, { type: 'text', text: message.content }];
+    return message;
+  }
+
+  message.content = [{ type: 'text', text: message.content }, ...image_urls];
+
+  return message;
+};
+
 /**
  * Formats a message to OpenAI payload format based on the provided options.
  *
@@ -10,12 +33,14 @@ const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
  * @param {string} [params.message.sender] - The sender of the message.
  * @param {string} [params.message.text] - The text content of the message.
  * @param {string} [params.message.content] - The content of the message.
+ * @param {Array<string>} [params.message.image_urls] - The image_urls attached to the message for Vision API.
  * @param {string} [params.userName] - The name of the user.
  * @param {string} [params.assistantName] - The name of the assistant.
+ * @param {string} [params.endpoint] - Identifier for specific endpoint handling
  * @param {boolean} [params.langChain=false] - Whether to return a LangChain message object.
  * @returns {(Object|HumanMessage|AIMessage|SystemMessage)} - The formatted message.
  */
-const formatMessage = ({ message, userName, assistantName, langChain = false }) => {
+const formatMessage = ({ message, userName, assistantName, endpoint, langChain = false }) => {
   let { role: _role, _name, sender, text, content: _content, lc_id } = message;
   if (lc_id && lc_id[2] && !langChain) {
     const roleMapping = {
@@ -32,6 +57,15 @@ const formatMessage = ({ message, userName, assistantName, langChain = false })
     content,
   };
 
+  const { image_urls } = message;
+  if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
+    return formatVisionMessage({
+      message: formattedMessage,
+      image_urls: message.image_urls,
+      endpoint,
+    });
+  }
+
   if (_name) {
     formattedMessage.name = _name;
   }

api/app/clients/prompts/formatMessages.spec.js

@@ -1,5 +1,6 @@
-const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
+const { Constants } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
+const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
 
 describe('formatMessage', () => {
   it('formats user message', () => {
@@ -54,7 +55,6 @@ describe('formatMessage', () => {
         _id: '6512cdfb92cbf69fea615331',
         messageId: 'b620bf73-c5c3-4a38-b724-76886aac24c4',
         __v: 0,
-        cancelled: false,
         conversationId: '5c23d24f-941f-4aab-85df-127b596c8aa5',
         createdAt: Date.now(),
         error: false,
@@ -62,7 +62,7 @@ describe('formatMessage', () => {
         isCreatedByUser: true,
         isEdited: false,
         model: null,
-        parentMessageId: '00000000-0000-0000-0000-000000000000',
+        parentMessageId: Constants.NO_PARENT,
         sender: 'User',
         text: 'hi',
         tokenCount: 5,

api/app/clients/prompts/index.js

@@ -4,6 +4,8 @@ const handleInputs = require('./handleInputs');
 const instructions = require('./instructions');
 const titlePrompts = require('./titlePrompts');
 const truncateText = require('./truncateText');
+const createVisionPrompt = require('./createVisionPrompt');
+const createContextHandlers = require('./createContextHandlers');
 
 module.exports = {
   ...formatMessages,
@@ -11,5 +13,7 @@ module.exports = {
   ...handleInputs,
   ...instructions,
   ...titlePrompts,
-  truncateText,
+  ...truncateText,
+  createVisionPrompt,
+  createContextHandlers,
 };

api/app/clients/prompts/titlePrompts.js

@@ -27,7 +27,96 @@ ${convo}`,
   return titlePrompt;
 };
 
+const titleInstruction =
+  'a concise, 5-word-or-less title for the conversation, using its same language, with no punctuation. Apply title case conventions appropriate for the language. Never directly mention the language name or the word "title"';
+const titleFunctionPrompt = `In this environment you have access to a set of tools you can use to generate the conversation title.
+  
+You may call them like this:
+<function_calls>
+<invoke>
+<tool_name>$TOOL_NAME</tool_name>
+<parameters>
+<$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME>
+...
+</parameters>
+</invoke>
+</function_calls>
+
+Here are the tools available:
+<tools>
+<tool_description>
+<tool_name>submit_title</tool_name>
+<description>
+Submit a brief title in the conversation's language, following the parameter description closely.
+</description>
+<parameters>
+<parameter>
+<name>title</name>
+<type>string</type>
+<description>${titleInstruction}</description>
+</parameter>
+</parameters>
+</tool_description>
+</tools>`;
+
+const genTranslationPrompt = (
+  translationPrompt,
+) => `In this environment you have access to a set of tools you can use to translate text.
+  
+You may call them like this:
+<function_calls>
+<invoke>
+<tool_name>$TOOL_NAME</tool_name>
+<parameters>
+<$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME>
+...
+</parameters>
+</invoke>
+</function_calls>
+
+Here are the tools available:
+<tools>
+<tool_description>
+<tool_name>submit_translation</tool_name>
+<description>
+Submit a translation in the target language, following the parameter description and its language closely.
+</description>
+<parameters>
+<parameter>
+<name>translation</name>
+<type>string</type>
+<description>${translationPrompt}
+ONLY include the generated translation without quotations, nor its related key</description>
+</parameter>
+</parameters>
+</tool_description>
+</tools>`;
+
+/**
+ * Parses specified parameter from the provided prompt.
+ * @param {string} prompt - The prompt containing the desired parameter.
+ * @param {string} paramName - The name of the parameter to extract.
+ * @returns {string} The parsed parameter's value or a default value if not found.
+ */
+function parseParamFromPrompt(prompt, paramName) {
+  const paramRegex = new RegExp(`<${paramName}>([\\s\\S]+?)</${paramName}>`);
+  const paramMatch = prompt.match(paramRegex);
+
+  if (paramMatch && paramMatch[1]) {
+    return paramMatch[1].trim();
+  }
+
+  if (prompt && prompt.length) {
+    return `NO TOOL INVOCATION: ${prompt}`;
+  }
+  return `No ${paramName} provided`;
+}
+
 module.exports = {
   langPrompt,
+  titleInstruction,
   createTitlePrompt,
+  titleFunctionPrompt,
+  parseParamFromPrompt,
+  genTranslationPrompt,
 };

api/app/clients/prompts/truncateText.js

@@ -1,10 +1,40 @@
 const MAX_CHAR = 255;
 
-function truncateText(text) {
-  if (text.length > MAX_CHAR) {
-    return `${text.slice(0, MAX_CHAR)}... [text truncated for brevity]`;
+/**
+ * Truncates a given text to a specified maximum length, appending ellipsis and a notification
+ * if the original text exceeds the maximum length.
+ *
+ * @param {string} text - The text to be truncated.
+ * @param {number} [maxLength=MAX_CHAR] - The maximum length of the text after truncation. Defaults to MAX_CHAR.
+ * @returns {string} The truncated text if the original text length exceeds maxLength, otherwise returns the original text.
+ */
+function truncateText(text, maxLength = MAX_CHAR) {
+  if (text.length > maxLength) {
+    return `${text.slice(0, maxLength)}... [text truncated for brevity]`;
   }
   return text;
 }
 
-module.exports = truncateText;
+/**
+ * Truncates a given text to a specified maximum length by showing the first half and the last half of the text,
+ * separated by ellipsis. This method ensures the output does not exceed the maximum length, including the addition
+ * of ellipsis and notification if the original text exceeds the maximum length.
+ *
+ * @param {string} text - The text to be truncated.
+ * @param {number} [maxLength=MAX_CHAR] - The maximum length of the output text after truncation. Defaults to MAX_CHAR.
+ * @returns {string} The truncated text showing the first half and the last half, or the original text if it does not exceed maxLength.
+ */
+function smartTruncateText(text, maxLength = MAX_CHAR) {
+  const ellipsis = '...';
+  const notification = ' [text truncated for brevity]';
+  const halfMaxLength = Math.floor((maxLength - ellipsis.length - notification.length) / 2);
+
+  if (text.length > maxLength) {
+    const startLastHalf = text.length - halfMaxLength;
+    return `${text.slice(0, halfMaxLength)}${ellipsis}${text.slice(startLastHalf)}${notification}`;
+  }
+
+  return text;
+}
+
+module.exports = { truncateText, smartTruncateText };

api/app/clients/specs/BaseClient.test.js

@@ -1,19 +1,34 @@
+const { Constants } = require('librechat-data-provider');
 const { initializeFakeClient } = require('./FakeClient');
 
 jest.mock('../../../lib/db/connectDb');
-jest.mock('../../../models', () => {
-  return function () {
-    return {
-      save: jest.fn(),
-      deleteConvos: jest.fn(),
-      getConvo: jest.fn(),
-      getMessages: jest.fn(),
-      saveMessage: jest.fn(),
-      updateMessage: jest.fn(),
-      saveConvo: jest.fn(),
-    };
-  };
-});
+jest.mock('~/models', () => ({
+  User: jest.fn(),
+  Key: jest.fn(),
+  Session: jest.fn(),
+  Balance: jest.fn(),
+  Transaction: jest.fn(),
+  getMessages: jest.fn().mockResolvedValue([]),
+  saveMessage: jest.fn(),
+  updateMessage: jest.fn(),
+  deleteMessagesSince: jest.fn(),
+  deleteMessages: jest.fn(),
+  getConvoTitle: jest.fn(),
+  getConvo: jest.fn(),
+  saveConvo: jest.fn(),
+  deleteConvos: jest.fn(),
+  getPreset: jest.fn(),
+  getPresets: jest.fn(),
+  savePreset: jest.fn(),
+  deletePresets: jest.fn(),
+  findFileById: jest.fn(),
+  createFile: jest.fn(),
+  updateFile: jest.fn(),
+  deleteFile: jest.fn(),
+  deleteFiles: jest.fn(),
+  getFiles: jest.fn(),
+  updateFileUsage: jest.fn(),
+}));
 
 jest.mock('langchain/chat_models/openai', () => {
   return {
@@ -293,7 +308,7 @@ describe('BaseClient', () => {
     const unorderedMessages = [
       { id: '3', parentMessageId: '2', text: 'Message 3' },
       { id: '2', parentMessageId: '1', text: 'Message 2' },
-      { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+      { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
     ];
 
     it('should return ordered messages based on parentMessageId', () => {
@@ -302,7 +317,7 @@ describe('BaseClient', () => {
         parentMessageId: '3',
       });
       expect(result).toEqual([
-        { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+        { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
         { id: '2', parentMessageId: '1', text: 'Message 2' },
         { id: '3', parentMessageId: '2', text: 'Message 3' },
       ]);
@@ -529,9 +544,9 @@ describe('BaseClient', () => {
       );
     });
 
-    test('setOptions is called with the correct arguments', async () => {
+    test('setOptions is called with the correct arguments only when replaceOptions is set to true', async () => {
       TestClient.setOptions = jest.fn();
-      const opts = { conversationId: '123', parentMessageId: '456' };
+      const opts = { conversationId: '123', parentMessageId: '456', replaceOptions: true };
       await TestClient.sendMessage('Hello, world!', opts);
       expect(TestClient.setOptions).toHaveBeenCalledWith(opts);
       TestClient.setOptions.mockClear();

api/app/clients/specs/FakeClient.js

@@ -40,9 +40,9 @@ class FakeClient extends BaseClient {
       };
     }
 
-    this.maxContextTokens = getModelMaxTokens(this.modelOptions.model) ?? 4097;
+    this.maxContextTokens =
+      this.options.maxContextTokens ?? getModelMaxTokens(this.modelOptions.model) ?? 4097;
   }
-  getCompletion() {}
   buildMessages() {}
   getTokenCount(str) {
     return str.length;
@@ -86,6 +86,19 @@ const initializeFakeClient = (apiKey, options, fakeMessages) => {
     return 'Mock response text';
   });
 
+  // eslint-disable-next-line no-unused-vars
+  TestClient.getCompletion = jest.fn().mockImplementation(async (..._args) => {
+    return {
+      choices: [
+        {
+          message: {
+            content: 'Mock response text',
+          },
+        },
+      ],
+    };
+  });
+
   TestClient.buildMessages = jest.fn(async (messages, parentMessageId) => {
     const orderedMessages = TestClient.constructor.getMessagesForConversation({
       messages,

api/app/clients/specs/OpenAIClient.test.js

@@ -1,8 +1,138 @@
 require('dotenv').config();
+const OpenAI = require('openai');
+const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
+const { genAzureChatCompletion } = require('~/utils/azureUtils');
 const OpenAIClient = require('../OpenAIClient');
-
 jest.mock('meilisearch');
 
+jest.mock('~/lib/db/connectDb');
+jest.mock('~/models', () => ({
+  User: jest.fn(),
+  Key: jest.fn(),
+  Session: jest.fn(),
+  Balance: jest.fn(),
+  Transaction: jest.fn(),
+  getMessages: jest.fn().mockResolvedValue([]),
+  saveMessage: jest.fn(),
+  updateMessage: jest.fn(),
+  deleteMessagesSince: jest.fn(),
+  deleteMessages: jest.fn(),
+  getConvoTitle: jest.fn(),
+  getConvo: jest.fn(),
+  saveConvo: jest.fn(),
+  deleteConvos: jest.fn(),
+  getPreset: jest.fn(),
+  getPresets: jest.fn(),
+  savePreset: jest.fn(),
+  deletePresets: jest.fn(),
+  findFileById: jest.fn(),
+  createFile: jest.fn(),
+  updateFile: jest.fn(),
+  deleteFile: jest.fn(),
+  deleteFiles: jest.fn(),
+  getFiles: jest.fn(),
+  updateFileUsage: jest.fn(),
+}));
+
+jest.mock('langchain/chat_models/openai', () => {
+  return {
+    ChatOpenAI: jest.fn().mockImplementation(() => {
+      return {};
+    }),
+  };
+});
+
+jest.mock('openai');
+
+jest.spyOn(OpenAI, 'constructor').mockImplementation(function (...options) {
+  // We can add additional logic here if needed
+  return new OpenAI(...options);
+});
+
+const finalChatCompletion = jest.fn().mockResolvedValue({
+  choices: [
+    {
+      message: { role: 'assistant', content: 'Mock message content' },
+      finish_reason: 'Mock finish reason',
+    },
+  ],
+});
+
+const stream = jest.fn().mockImplementation(() => {
+  let isDone = false;
+  let isError = false;
+  let errorCallback = null;
+
+  const onEventHandlers = {
+    abort: () => {
+      // Mock abort behavior
+    },
+    error: (callback) => {
+      errorCallback = callback; // Save the error callback for later use
+    },
+    finalMessage: (callback) => {
+      callback({ role: 'assistant', content: 'Mock Response' });
+      isDone = true; // Set stream to done
+    },
+  };
+
+  const mockStream = {
+    on: jest.fn((event, callback) => {
+      if (onEventHandlers[event]) {
+        onEventHandlers[event](callback);
+      }
+      return mockStream;
+    }),
+    finalChatCompletion,
+    controller: { abort: jest.fn() },
+    triggerError: () => {
+      isError = true;
+      if (errorCallback) {
+        errorCallback(new Error('Mock error'));
+      }
+    },
+    [Symbol.asyncIterator]: () => {
+      return {
+        next: () => {
+          if (isError) {
+            return Promise.reject(new Error('Mock error'));
+          }
+          if (isDone) {
+            return Promise.resolve({ done: true });
+          }
+          const chunk = { choices: [{ delta: { content: 'Mock chunk' } }] };
+          return Promise.resolve({ value: chunk, done: false });
+        },
+      };
+    },
+  };
+  return mockStream;
+});
+
+const create = jest.fn().mockResolvedValue({
+  choices: [
+    {
+      message: { content: 'Mock message content' },
+      finish_reason: 'Mock finish reason',
+    },
+  ],
+});
+
+OpenAI.mockImplementation(() => ({
+  beta: {
+    chat: {
+      completions: {
+        stream,
+      },
+    },
+  },
+  chat: {
+    completions: {
+      create,
+    },
+  },
+}));
+
 describe('OpenAIClient', () => {
   let client, client2;
   const model = 'gpt-4';
@@ -12,15 +142,39 @@ describe('OpenAIClient', () => {
     { role: 'assistant', sender: 'Assistant', text: 'Hi', messageId: '2' },
   ];
 
+  const defaultOptions = {
+    // debug: true,
+    req: {},
+    openaiApiKey: 'new-api-key',
+    modelOptions: {
+      model,
+      temperature: 0.7,
+    },
+  };
+
+  const defaultAzureOptions = {
+    azureOpenAIApiInstanceName: 'your-instance-name',
+    azureOpenAIApiDeploymentName: 'your-deployment-name',
+    azureOpenAIApiVersion: '2020-07-01-preview',
+  };
+
+  let originalWarn;
+
+  beforeAll(() => {
+    originalWarn = console.warn;
+    console.warn = jest.fn();
+  });
+
+  afterAll(() => {
+    console.warn = originalWarn;
+  });
+
   beforeEach(() => {
-    const options = {
-      // debug: true,
-      openaiApiKey: 'new-api-key',
-      modelOptions: {
-        model,
-        temperature: 0.7,
-      },
-    };
+    console.warn.mockClear();
+  });
+
+  beforeEach(() => {
+    const options = { ...defaultOptions };
     client = new OpenAIClient('test-api-key', options);
     client2 = new OpenAIClient('test-api-key', options);
     client.summarizeMessages = jest.fn().mockResolvedValue({
@@ -32,6 +186,7 @@ describe('OpenAIClient', () => {
       .fn()
       .mockResolvedValue({ prompt: messages.map((m) => m.text).join('\n') });
     client.constructor.freeAndResetAllEncoders();
+    client.getMessages = jest.fn().mockResolvedValue([]);
   });
 
   describe('setOptions', () => {
@@ -86,7 +241,97 @@ describe('OpenAIClient', () => {
 
       client.setOptions({ reverseProxyUrl: 'https://example.com/completions' });
       expect(client.completionsUrl).toBe('https://example.com/completions');
-      expect(client.langchainProxy).toBeUndefined();
+      expect(client.langchainProxy).toBe('https://example.com/completions');
+    });
+  });
+
+  describe('setOptions with Simplified Azure Integration', () => {
+    afterEach(() => {
+      delete process.env.AZURE_OPENAI_DEFAULT_MODEL;
+      delete process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME;
+    });
+
+    const azureOpenAIApiInstanceName = 'test-instance';
+    const azureOpenAIApiDeploymentName = 'test-deployment';
+    const azureOpenAIApiVersion = '2020-07-01-preview';
+
+    const createOptions = (model) => ({
+      modelOptions: { model },
+      azure: {
+        azureOpenAIApiInstanceName,
+        azureOpenAIApiDeploymentName,
+        azureOpenAIApiVersion,
+      },
+    });
+
+    it('should set model from AZURE_OPENAI_DEFAULT_MODEL when Azure is enabled', () => {
+      process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt-4-azure';
+      const options = createOptions('test');
+      client.azure = options.azure;
+      client.setOptions(options);
+      expect(client.modelOptions.model).toBe('gpt-4-azure');
+    });
+
+    it('should not change model if Azure is not enabled', () => {
+      process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt-4-azure';
+      const originalModel = 'test';
+      client.azure = false;
+      client.setOptions(createOptions('test'));
+      expect(client.modelOptions.model).toBe(originalModel);
+    });
+
+    it('should not change model if AZURE_OPENAI_DEFAULT_MODEL is not set and model is passed', () => {
+      const originalModel = 'GROK-LLM';
+      const options = createOptions(originalModel);
+      client.azure = options.azure;
+      client.setOptions(options);
+      expect(client.modelOptions.model).toBe(originalModel);
+    });
+
+    it('should change model if AZURE_OPENAI_DEFAULT_MODEL is set and model is passed', () => {
+      process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt-4-azure';
+      const originalModel = 'GROK-LLM';
+      const options = createOptions(originalModel);
+      client.azure = options.azure;
+      client.setOptions(options);
+      expect(client.modelOptions.model).toBe(process.env.AZURE_OPENAI_DEFAULT_MODEL);
+    });
+
+    it('should include model in deployment name if AZURE_USE_MODEL_AS_DEPLOYMENT_NAME is set', () => {
+      process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true';
+      const model = 'gpt-4-azure';
+
+      const AzureClient = new OpenAIClient('test-api-key', createOptions(model));
+
+      const expectedValue = `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${model}/chat/completions?api-version=${azureOpenAIApiVersion}`;
+
+      expect(AzureClient.modelOptions.model).toBe(model);
+      expect(AzureClient.azureEndpoint).toBe(expectedValue);
+    });
+
+    it('should include model in deployment name if AZURE_USE_MODEL_AS_DEPLOYMENT_NAME and default model is set', () => {
+      const defaultModel = 'gpt-4-azure';
+      process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true';
+      process.env.AZURE_OPENAI_DEFAULT_MODEL = defaultModel;
+      const model = 'gpt-4-this-is-a-test-model-name';
+
+      const AzureClient = new OpenAIClient('test-api-key', createOptions(model));
+
+      const expectedValue = `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${model}/chat/completions?api-version=${azureOpenAIApiVersion}`;
+
+      expect(AzureClient.modelOptions.model).toBe(defaultModel);
+      expect(AzureClient.azureEndpoint).toBe(expectedValue);
+    });
+
+    it('should not include model in deployment name if AZURE_USE_MODEL_AS_DEPLOYMENT_NAME is not set', () => {
+      const model = 'gpt-4-azure';
+
+      const AzureClient = new OpenAIClient('test-api-key', createOptions(model));
+
+      const expectedValue = `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${azureOpenAIApiDeploymentName}/chat/completions?api-version=${azureOpenAIApiVersion}`;
+
+      expect(AzureClient.modelOptions.model).toBe(model);
+      expect(AzureClient.azureEndpoint).toBe(expectedValue);
     });
   });
 
@@ -309,5 +554,151 @@ describe('OpenAIClient', () => {
         expect(totalTokens).toBe(testCase.expected);
       });
     });
+
+    const vision_request = [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'describe what is in this image?',
+          },
+          {
+            type: 'image_url',
+            image_url: {
+              url: 'https://venturebeat.com/wp-content/uploads/2019/03/openai-1.png',
+              detail: 'high',
+            },
+          },
+        ],
+      },
+    ];
+
+    const expectedTokens = 14;
+    const visionModel = 'gpt-4-vision-preview';
+
+    it(`should return ${expectedTokens} tokens for model ${visionModel} (Vision Request)`, () => {
+      client.modelOptions.model = visionModel;
+      client.selectTokenizer();
+      // 3 tokens for assistant label
+      let totalTokens = 3;
+      for (let message of vision_request) {
+        totalTokens += client.getTokenCountForMessage(message);
+      }
+      expect(totalTokens).toBe(expectedTokens);
+    });
+  });
+
+  describe('sendMessage/getCompletion/chatCompletion', () => {
+    afterEach(() => {
+      delete process.env.AZURE_OPENAI_DEFAULT_MODEL;
+      delete process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME;
+      delete process.env.OPENROUTER_API_KEY;
+    });
+
+    it('should call getCompletion and fetchEventSource when using a text/instruct model', async () => {
+      const model = 'text-davinci-003';
+      const onProgress = jest.fn().mockImplementation(() => ({}));
+
+      const testClient = new OpenAIClient('test-api-key', {
+        ...defaultOptions,
+        modelOptions: { model },
+      });
+
+      const getCompletion = jest.spyOn(testClient, 'getCompletion');
+      await testClient.sendMessage('Hi mom!', { onProgress });
+
+      expect(getCompletion).toHaveBeenCalled();
+      expect(getCompletion.mock.calls.length).toBe(1);
+
+      const currentDateString = new Date().toLocaleDateString('en-us', {
+        year: 'numeric',
+        month: 'long',
+        day: 'numeric',
+      });
+
+      expect(getCompletion.mock.calls[0][0]).toBe(
+        `||>Instructions:\nYou are ChatGPT, a large language model trained by OpenAI. Respond conversationally.\nCurrent date: ${currentDateString}\n\n||>User:\nHi mom!\n||>Assistant:\n`,
+      );
+
+      expect(fetchEventSource).toHaveBeenCalled();
+      expect(fetchEventSource.mock.calls.length).toBe(1);
+
+      // Check if the first argument (url) is correct
+      const firstCallArgs = fetchEventSource.mock.calls[0];
+
+      const expectedURL = 'https://api.openai.com/v1/completions';
+      expect(firstCallArgs[0]).toBe(expectedURL);
+
+      const requestBody = JSON.parse(firstCallArgs[1].body);
+      expect(requestBody).toHaveProperty('model');
+      expect(requestBody.model).toBe(model);
+    });
+
+    it('[Azure OpenAI] should call chatCompletion and OpenAI.stream with correct args', async () => {
+      // Set a default model
+      process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt4-turbo';
+
+      const onProgress = jest.fn().mockImplementation(() => ({}));
+      client.azure = defaultAzureOptions;
+      const chatCompletion = jest.spyOn(client, 'chatCompletion');
+      await client.sendMessage('Hi mom!', {
+        replaceOptions: true,
+        ...defaultOptions,
+        modelOptions: { model: 'gpt4-turbo', stream: true },
+        onProgress,
+        azure: defaultAzureOptions,
+      });
+
+      expect(chatCompletion).toHaveBeenCalled();
+      expect(chatCompletion.mock.calls.length).toBe(1);
+
+      const chatCompletionArgs = chatCompletion.mock.calls[0][0];
+      const { payload } = chatCompletionArgs;
+
+      expect(payload[0].role).toBe('user');
+      expect(payload[0].content).toBe('Hi mom!');
+
+      // Azure OpenAI does not use the model property, and will error if it's passed
+      // This check ensures the model property is not present
+      const streamArgs = stream.mock.calls[0][0];
+      expect(streamArgs).not.toHaveProperty('model');
+
+      // Check if the baseURL is correct
+      const constructorArgs = OpenAI.mock.calls[0][0];
+      const expectedURL = genAzureChatCompletion(defaultAzureOptions).split('/chat')[0];
+      expect(constructorArgs.baseURL).toBe(expectedURL);
+    });
+  });
+
+  describe('checkVisionRequest functionality', () => {
+    let client;
+    const attachments = [{ type: 'image/png' }];
+
+    beforeEach(() => {
+      client = new OpenAIClient('test-api-key', {
+        endpoint: 'ollama',
+        modelOptions: {
+          model: 'initial-model',
+        },
+        modelsConfig: {
+          ollama: ['initial-model', 'llava', 'other-model'],
+        },
+      });
+
+      client.defaultVisionModel = 'non-valid-default-model';
+    });
+
+    afterEach(() => {
+      jest.restoreAllMocks();
+    });
+
+    it('should set "llava" as the model if it is the first valid model when default validation fails', () => {
+      client.checkVisionRequest(attachments);
+
+      expect(client.modelOptions.model).toBe('llava');
+      expect(client.isVisionModel).toBeTruthy();
+      expect(client.modelOptions.stop).toBeUndefined();
+    });
   });
 });

api/app/clients/specs/PluginsClient.test.js

@@ -1,9 +1,10 @@
+const crypto = require('crypto');
+const { Constants } = require('librechat-data-provider');
 const { HumanChatMessage, AIChatMessage } = require('langchain/schema');
 const PluginsClient = require('../PluginsClient');
-const crypto = require('crypto');
 
-jest.mock('../../../lib/db/connectDb');
-jest.mock('../../../models/Conversation', () => {
+jest.mock('~/lib/db/connectDb');
+jest.mock('~/models/Conversation', () => {
   return function () {
     return {
       save: jest.fn(),
@@ -12,6 +13,12 @@ jest.mock('../../../models/Conversation', () => {
   };
 });
 
+const defaultAzureOptions = {
+  azureOpenAIApiInstanceName: 'your-instance-name',
+  azureOpenAIApiDeploymentName: 'your-deployment-name',
+  azureOpenAIApiVersion: '2020-07-01-preview',
+};
+
 describe('PluginsClient', () => {
   let TestAgent;
   let options = {
@@ -60,7 +67,7 @@ describe('PluginsClient', () => {
         TestAgent.setOptions(opts);
       }
       const conversationId = opts.conversationId || crypto.randomUUID();
-      const parentMessageId = opts.parentMessageId || '00000000-0000-0000-0000-000000000000';
+      const parentMessageId = opts.parentMessageId || Constants.NO_PARENT;
       const userMessageId = opts.overrideParentMessageId || crypto.randomUUID();
       this.pastMessages = await TestAgent.loadHistory(
         conversationId,
@@ -144,4 +151,73 @@ describe('PluginsClient', () => {
       expect(chatMessages[0].text).toEqual(userMessage);
     });
   });
+
+  describe('getFunctionModelName', () => {
+    let client;
+
+    beforeEach(() => {
+      client = new PluginsClient('dummy_api_key');
+    });
+
+    test('should return the input when it includes a dash followed by four digits', () => {
+      expect(client.getFunctionModelName('-1234')).toBe('-1234');
+      expect(client.getFunctionModelName('gpt-4-5678-preview')).toBe('gpt-4-5678-preview');
+    });
+
+    test('should return the input for all function-capable models (`0613` models and above)', () => {
+      expect(client.getFunctionModelName('gpt-4-0613')).toBe('gpt-4-0613');
+      expect(client.getFunctionModelName('gpt-4-32k-0613')).toBe('gpt-4-32k-0613');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-0613')).toBe('gpt-3.5-turbo-0613');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-16k-0613')).toBe('gpt-3.5-turbo-16k-0613');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-1106')).toBe('gpt-3.5-turbo-1106');
+      expect(client.getFunctionModelName('gpt-4-1106-preview')).toBe('gpt-4-1106-preview');
+      expect(client.getFunctionModelName('gpt-4-1106')).toBe('gpt-4-1106');
+    });
+
+    test('should return the corresponding model if input is non-function capable (`0314` models)', () => {
+      expect(client.getFunctionModelName('gpt-4-0314')).toBe('gpt-4');
+      expect(client.getFunctionModelName('gpt-4-32k-0314')).toBe('gpt-4');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-0314')).toBe('gpt-3.5-turbo');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-16k-0314')).toBe('gpt-3.5-turbo');
+    });
+
+    test('should return "gpt-3.5-turbo" when the input includes "gpt-3.5-turbo"', () => {
+      expect(client.getFunctionModelName('test gpt-3.5-turbo model')).toBe('gpt-3.5-turbo');
+    });
+
+    test('should return "gpt-4" when the input includes "gpt-4"', () => {
+      expect(client.getFunctionModelName('testing gpt-4')).toBe('gpt-4');
+    });
+
+    test('should return "gpt-3.5-turbo" for input that does not meet any specific condition', () => {
+      expect(client.getFunctionModelName('random string')).toBe('gpt-3.5-turbo');
+      expect(client.getFunctionModelName('')).toBe('gpt-3.5-turbo');
+    });
+  });
+  describe('Azure OpenAI tests specific to Plugins', () => {
+    // TODO: add more tests for Azure OpenAI integration with Plugins
+    // let client;
+    // beforeEach(() => {
+    //   client = new PluginsClient('dummy_api_key');
+    // });
+
+    test('should not call getFunctionModelName when azure options are set', () => {
+      const spy = jest.spyOn(PluginsClient.prototype, 'getFunctionModelName');
+      const model = 'gpt-4-turbo';
+
+      // note, without the azure change in PR #1766, `getFunctionModelName` is called twice
+      const testClient = new PluginsClient('dummy_api_key', {
+        agentOptions: {
+          model,
+          agent: 'functions',
+        },
+        azure: defaultAzureOptions,
+      });
+
+      expect(spy).not.toHaveBeenCalled();
+      expect(testClient.agentOptions.model).toBe(model);
+
+      spy.mockRestore();
+    });
+  });
 });

api/app/clients/tools/AIPluginTool.js

@@ -1,238 +0,0 @@
-const { Tool } = require('langchain/tools');
-const yaml = require('js-yaml');
-
-/*
-export interface AIPluginToolParams {
-  name: string;
-  description: string;
-  apiSpec: string;
-  openaiSpec: string;
-  model: BaseLanguageModel;
-}
-
-export interface PathParameter {
-  name: string;
-  description: string;
-}
-
-export interface Info {
-  title: string;
-  description: string;
-  version: string;
-}
-export interface PathMethod {
-  summary: string;
-  operationId: string;
-  parameters?: PathParameter[];
-}
-
-interface ApiSpec {
-  openapi: string;
-  info: Info;
-  paths: { [key: string]: { [key: string]: PathMethod } };
-}
-*/
-
-function isJson(str) {
-  try {
-    JSON.parse(str);
-  } catch (e) {
-    return false;
-  }
-  return true;
-}
-
-function convertJsonToYamlIfApplicable(spec) {
-  if (isJson(spec)) {
-    const jsonData = JSON.parse(spec);
-    return yaml.dump(jsonData);
-  }
-  return spec;
-}
-
-function extractShortVersion(openapiSpec) {
-  openapiSpec = convertJsonToYamlIfApplicable(openapiSpec);
-  try {
-    const fullApiSpec = yaml.load(openapiSpec);
-    const shortApiSpec = {
-      openapi: fullApiSpec.openapi,
-      info: fullApiSpec.info,
-      paths: {},
-    };
-
-    for (let path in fullApiSpec.paths) {
-      shortApiSpec.paths[path] = {};
-      for (let method in fullApiSpec.paths[path]) {
-        shortApiSpec.paths[path][method] = {
-          summary: fullApiSpec.paths[path][method].summary,
-          operationId: fullApiSpec.paths[path][method].operationId,
-          parameters: fullApiSpec.paths[path][method].parameters?.map((parameter) => ({
-            name: parameter.name,
-            description: parameter.description,
-          })),
-        };
-      }
-    }
-
-    return yaml.dump(shortApiSpec);
-  } catch (e) {
-    console.log(e);
-    return '';
-  }
-}
-function printOperationDetails(operationId, openapiSpec) {
-  openapiSpec = convertJsonToYamlIfApplicable(openapiSpec);
-  let returnText = '';
-  try {
-    let doc = yaml.load(openapiSpec);
-    let servers = doc.servers;
-    let paths = doc.paths;
-    let components = doc.components;
-
-    for (let path in paths) {
-      for (let method in paths[path]) {
-        let operation = paths[path][method];
-        if (operation.operationId === operationId) {
-          returnText += `The API request to do for operationId "${operationId}" is:\n`;
-          returnText += `Method: ${method.toUpperCase()}\n`;
-
-          let url = servers[0].url + path;
-          returnText += `Path: ${url}\n`;
-
-          returnText += 'Parameters:\n';
-          if (operation.parameters) {
-            for (let param of operation.parameters) {
-              let required = param.required ? '' : ' (optional),';
-              returnText += `- ${param.name} (${param.in},${required} ${param.schema.type}): ${param.description}\n`;
-            }
-          } else {
-            returnText += ' None\n';
-          }
-          returnText += '\n';
-
-          let responseSchema = operation.responses['200'].content['application/json'].schema;
-
-          // Check if schema is a reference
-          if (responseSchema.$ref) {
-            // Extract schema name from reference
-            let schemaName = responseSchema.$ref.split('/').pop();
-            // Look up schema in components
-            responseSchema = components.schemas[schemaName];
-          }
-
-          returnText += 'Response schema:\n';
-          returnText += '- Type: ' + responseSchema.type + '\n';
-          returnText += '- Additional properties:\n';
-          returnText += '  - Type: ' + responseSchema.additionalProperties?.type + '\n';
-          if (responseSchema.additionalProperties?.properties) {
-            returnText += '  - Properties:\n';
-            for (let prop in responseSchema.additionalProperties.properties) {
-              returnText += `    - ${prop} (${responseSchema.additionalProperties.properties[prop].type}): Description not provided in OpenAPI spec\n`;
-            }
-          }
-        }
-      }
-    }
-    if (returnText === '') {
-      returnText += `No operation with operationId "${operationId}" found.`;
-    }
-    return returnText;
-  } catch (e) {
-    console.log(e);
-    return '';
-  }
-}
-
-class AIPluginTool extends Tool {
-  /*
-  private _name: string;
-  private _description: string;
-  apiSpec: string;
-  openaiSpec: string;
-  model: BaseLanguageModel;
-  */
-
-  get name() {
-    return this._name;
-  }
-
-  get description() {
-    return this._description;
-  }
-
-  constructor(params) {
-    super();
-    this._name = params.name;
-    this._description = params.description;
-    this.apiSpec = params.apiSpec;
-    this.openaiSpec = params.openaiSpec;
-    this.model = params.model;
-  }
-
-  async _call(input) {
-    let date = new Date();
-    let fullDate = `Date: ${date.getDate()}/${
-      date.getMonth() + 1
-    }/${date.getFullYear()}, Time: ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
-    const prompt = `${fullDate}\nQuestion: ${input} \n${this.apiSpec}.`;
-    console.log(prompt);
-    const gptResponse = await this.model.predict(prompt);
-    let operationId = gptResponse.match(/operationId: (.*)/)?.[1];
-    if (!operationId) {
-      return 'No operationId found in the response';
-    }
-    if (operationId == 'No API path found to answer the question') {
-      return 'No API path found to answer the question';
-    }
-
-    let openApiData = printOperationDetails(operationId, this.openaiSpec);
-
-    return openApiData;
-  }
-
-  static async fromPluginUrl(url, model) {
-    const aiPluginRes = await fetch(url, {});
-    if (!aiPluginRes.ok) {
-      throw new Error(`Failed to fetch plugin from ${url} with status ${aiPluginRes.status}`);
-    }
-    const aiPluginJson = await aiPluginRes.json();
-    const apiUrlRes = await fetch(aiPluginJson.api.url, {});
-    if (!apiUrlRes.ok) {
-      throw new Error(
-        `Failed to fetch API spec from ${aiPluginJson.api.url} with status ${apiUrlRes.status}`,
-      );
-    }
-    const apiUrlJson = await apiUrlRes.text();
-    const shortApiSpec = extractShortVersion(apiUrlJson);
-    return new AIPluginTool({
-      name: aiPluginJson.name_for_model.toLowerCase(),
-      description: `A \`tool\` to learn the API documentation for ${aiPluginJson.name_for_model.toLowerCase()}, after which you can use 'http_request' to make the actual API call. Short description of how to use the API's results: ${
-        aiPluginJson.description_for_model
-      })`,
-      apiSpec: `
-As an AI, your task is to identify the operationId of the relevant API path based on the condensed OpenAPI specifications provided.
-
-Please note:
-
-1. Do not imagine URLs. Only use the information provided in the condensed OpenAPI specifications.
-
-2. Do not guess the operationId. Identify it strictly based on the API paths and their descriptions.
-
-Your output should only include:
-- operationId: The operationId of the relevant API path
-
-If you cannot find a suitable API path based on the OpenAPI specifications, please answer only "operationId: No API path found to answer the question".
-
-Now, based on the question above and the condensed OpenAPI specifications given below, identify the operationId:
-
-\`\`\`
-${shortApiSpec}
-\`\`\`
-`,
-      openaiSpec: apiUrlJson,
-      model: model,
-    });
-  }
-}
-
-module.exports = AIPluginTool;

api/app/clients/tools/AzureAiSearch.js

@@ -0,0 +1,98 @@
+const { z } = require('zod');
+const { StructuredTool } = require('langchain/tools');
+const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
+const { logger } = require('~/config');
+
+class AzureAISearch extends StructuredTool {
+  // Constants for default values
+  static DEFAULT_API_VERSION = '2023-11-01';
+  static DEFAULT_QUERY_TYPE = 'simple';
+  static DEFAULT_TOP = 5;
+
+  // Helper function for initializing properties
+  _initializeField(field, envVar, defaultValue) {
+    return field || process.env[envVar] || defaultValue;
+  }
+
+  constructor(fields = {}) {
+    super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
+
+    // Initialize properties using helper function
+    this.serviceEndpoint = this._initializeField(
+      fields.AZURE_AI_SEARCH_SERVICE_ENDPOINT,
+      'AZURE_AI_SEARCH_SERVICE_ENDPOINT',
+    );
+    this.indexName = this._initializeField(
+      fields.AZURE_AI_SEARCH_INDEX_NAME,
+      'AZURE_AI_SEARCH_INDEX_NAME',
+    );
+    this.apiKey = this._initializeField(fields.AZURE_AI_SEARCH_API_KEY, 'AZURE_AI_SEARCH_API_KEY');
+    this.apiVersion = this._initializeField(
+      fields.AZURE_AI_SEARCH_API_VERSION,
+      'AZURE_AI_SEARCH_API_VERSION',
+      AzureAISearch.DEFAULT_API_VERSION,
+    );
+    this.queryType = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE',
+      AzureAISearch.DEFAULT_QUERY_TYPE,
+    );
+    this.top = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_TOP,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_TOP',
+      AzureAISearch.DEFAULT_TOP,
+    );
+    this.select = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_SELECT,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_SELECT',
+    );
+
+    // Check for required fields
+    if (!this.serviceEndpoint || !this.indexName || !this.apiKey) {
+      throw new Error(
+        'Missing AZURE_AI_SEARCH_SERVICE_ENDPOINT, AZURE_AI_SEARCH_INDEX_NAME, or AZURE_AI_SEARCH_API_KEY environment variable.',
+      );
+    }
+
+    // Create SearchClient
+    this.client = new SearchClient(
+      this.serviceEndpoint,
+      this.indexName,
+      new AzureKeyCredential(this.apiKey),
+      { apiVersion: this.apiVersion },
+    );
+
+    // Define schema
+    this.schema = z.object({
+      query: z.string().describe('Search word or phrase to Azure AI Search'),
+    });
+  }
+
+  // Improved error handling and logging
+  async _call(data) {
+    const { query } = data;
+    try {
+      const searchOption = {
+        queryType: this.queryType,
+        top: this.top,
+      };
+      if (this.select) {
+        searchOption.select = this.select.split(',');
+      }
+      const searchResults = await this.client.search(query, searchOption);
+      const resultDocuments = [];
+      for await (const result of searchResults.results) {
+        resultDocuments.push(result.document);
+      }
+      return JSON.stringify(resultDocuments);
+    } catch (error) {
+      logger.error('Azure AI Search request failed', error);
+      return 'There was an error with Azure AI Search.';
+    }
+  }
+}
+
+module.exports = AzureAISearch;

api/app/clients/tools/AzureCognitiveSearch.js

@@ -1,111 +0,0 @@
-const { Tool } = require('langchain/tools');
-const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
-
-class AzureCognitiveSearch extends Tool {
-  constructor(fields = {}) {
-    super();
-    this.serviceEndpoint =
-      fields.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || this.getServiceEndpoint();
-    this.indexName = fields.AZURE_COGNITIVE_SEARCH_INDEX_NAME || this.getIndexName();
-    this.apiKey = fields.AZURE_COGNITIVE_SEARCH_API_KEY || this.getApiKey();
-
-    this.apiVersion = fields.AZURE_COGNITIVE_SEARCH_API_VERSION || this.getApiVersion();
-
-    this.queryType = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || this.getQueryType();
-    this.top = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP || this.getTop();
-    this.select = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT || this.getSelect();
-
-    this.client = new SearchClient(
-      this.serviceEndpoint,
-      this.indexName,
-      new AzureKeyCredential(this.apiKey),
-      {
-        apiVersion: this.apiVersion,
-      },
-    );
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'azure-cognitive-search';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'azure-cognitive-search\' tool to retrieve search results relevant to your input';
-
-  getServiceEndpoint() {
-    const serviceEndpoint = process.env.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || '';
-    if (!serviceEndpoint) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT environment variable.');
-    }
-    return serviceEndpoint;
-  }
-
-  getIndexName() {
-    const indexName = process.env.AZURE_COGNITIVE_SEARCH_INDEX_NAME || '';
-    if (!indexName) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_INDEX_NAME environment variable.');
-    }
-    return indexName;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.AZURE_COGNITIVE_SEARCH_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getApiVersion() {
-    return process.env.AZURE_COGNITIVE_SEARCH_API_VERSION || '2020-06-30';
-  }
-
-  getQueryType() {
-    return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || 'simple';
-  }
-
-  getTop() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP) {
-      return Number(process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP);
-    } else {
-      return 5;
-    }
-  }
-
-  getSelect() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT) {
-      return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT.split(',');
-    } else {
-      return null;
-    }
-  }
-
-  async _call(query) {
-    try {
-      const searchOption = {
-        queryType: this.queryType,
-        top: this.top,
-      };
-      if (this.select) {
-        searchOption.select = this.select;
-      }
-      const searchResults = await this.client.search(query, searchOption);
-      const resultDocuments = [];
-      for await (const result of searchResults.results) {
-        resultDocuments.push(result.document);
-      }
-      return JSON.stringify(resultDocuments);
-    } catch (error) {
-      console.error(`Azure Cognitive Search request failed: ${error}`);
-      return 'There was an error with Azure Cognitive Search.';
-    }
-  }
-}
-
-module.exports = AzureCognitiveSearch;

api/app/clients/tools/CodeInterpreter.js

@@ -1,52 +0,0 @@
-const { Tool } = require('langchain/tools');
-const WebSocket = require('ws');
-const { promisify } = require('util');
-const fs = require('fs');
-
-class CodeInterpreter extends Tool {
-  constructor() {
-    super();
-    this.name = 'code-interpreter';
-    this.description = `If there is plotting or any image related tasks, save the result as .png file. 
-    No need show the image or plot. USE print(variable_name) if you need output.You can run python codes with this plugin.You have to use print function in python code to get any result from this plugin. 
-    This does not support user input. Even if the code has input() function, change it to an appropriate value.
-    You can show the user the code with input() functions. But the code passed to the plug-in should not contain input().
-    You should provide properly formatted code to this plugin. If the code is executed successfully, the stdout will be returned to you. You have to print that to the user, and if the user had 
-    asked for an explanation, you have to provide one. If the output is "Error From here" or any other error message, 
-    tell the user "Python Engine Failed" and continue with whatever you are supposed to do.`;
-
-    // Create a promisified version of fs.unlink
-    this.unlinkAsync = promisify(fs.unlink);
-  }
-
-  async _call(input) {
-    const websocket = new WebSocket('ws://localhost:3380'); // Update with your WebSocket server URL
-
-    // Wait until the WebSocket connection is open
-    await new Promise((resolve) => {
-      websocket.onopen = resolve;
-    });
-
-    // Send the Python code to the server
-    websocket.send(input);
-
-    // Wait for the result from the server
-    const result = await new Promise((resolve) => {
-      websocket.onmessage = (event) => {
-        resolve(event.data);
-      };
-
-      // Handle WebSocket connection closed
-      websocket.onclose = () => {
-        resolve('Python Engine Failed');
-      };
-    });
-
-    // Close the WebSocket connection
-    websocket.close();
-
-    return result;
-  }
-}
-
-module.exports = CodeInterpreter;

api/app/clients/tools/DALL-E.js

@@ -1,41 +1,43 @@
-// From https://platform.openai.com/docs/api-reference/images/create
-// To use this tool, you must pass in a configured OpenAIApi object.
-const fs = require('fs');
-const OpenAI = require('openai');
-// const { genAzureEndpoint } = require('../../../utils/genAzureEndpoints');
-const { Tool } = require('langchain/tools');
-const saveImageFromUrl = require('./saveImageFromUrl');
 const path = require('path');
+const OpenAI = require('openai');
+const { v4: uuidv4 } = require('uuid');
+const { Tool } = require('langchain/tools');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { FileContext } = require('librechat-data-provider');
+const { getImageBasename } = require('~/server/services/Files/images');
+const extractBaseURL = require('~/utils/extractBaseURL');
+const { logger } = require('~/config');
 
 class OpenAICreateImage extends Tool {
   constructor(fields = {}) {
     super();
 
-    let apiKey = fields.DALLE_API_KEY || this.getApiKey();
-    // let azureKey = fields.AZURE_API_KEY || process.env.AZURE_API_KEY;
-    let config = { apiKey };
+    this.userId = fields.userId;
+    this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
+    let apiKey = fields.DALLE2_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
+
+    const config = { apiKey };
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
+    }
+
+    if (process.env.DALLE2_AZURE_API_VERSION && process.env.DALLE2_BASEURL) {
+      config.baseURL = process.env.DALLE2_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE2_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE2_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE2_API_KEY;
+    }
+
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
+    }
 
-    // if (azureKey) {
-    //   apiKey = azureKey;
-    //   const azureConfig = {
-    //     apiKey,
-    //     azureOpenAIApiInstanceName: process.env.AZURE_OPENAI_API_INSTANCE_NAME || fields.azureOpenAIApiInstanceName,
-    //     azureOpenAIApiDeploymentName: process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME || fields.azureOpenAIApiDeploymentName,
-    //     azureOpenAIApiVersion: process.env.AZURE_OPENAI_API_VERSION || fields.azureOpenAIApiVersion
-    //   };
-    //   config = {
-    //     apiKey,
-    //     basePath: genAzureEndpoint({
-    //       ...azureConfig,
-    //     }),
-    //     baseOptions: {
-    //       headers: { 'api-key': apiKey },
-    //       params: {
-    //         'api-version': azureConfig.azureOpenAIApiVersion // this might change. I got the current value from the sample code at https://oai.azure.com/portal/chat
-    //       }
-    //     }
-    //   };
-    // }
     this.openai = new OpenAI(config);
     this.name = 'dall-e';
     this.description = `You can generate images with 'dall-e'. This tool is exclusively for visual content.
@@ -45,10 +47,24 @@ Guidelines:
 - It's best to follow this format for image creation. Come up with the optional inputs yourself if none are given:
 "Subject: [subject], Style: [style], Color: [color], Details: [details], Emotion: [emotion]"
 - Generate images only once per human query unless explicitly requested by the user`;
+    this.description_for_model =
+      process.env.DALLE2_SYSTEM_PROMPT ??
+      `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
+// 1. Prompts must be in English. Translate to English if needed.
+// 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
+// 3. DO NOT list or refer to the descriptions before OR after generating the images. They should ONLY ever be written out ONCE, in the \`"prompts"\` field of the request. You do not need to ask for permission to generate, just do it!
+// 4. Always mention the image type (photo, oil painting, watercolor painting, illustration, cartoon, drawing, vector, render, etc.) at the beginning of the caption. Unless the captions suggests otherwise, make one of the images a photo.
+// 5. Diversify depictions of ALL images with people to always include always DESCENT and GENDER for EACH person using direct terms. Adjust only human descriptions.
+// - EXPLICITLY specify these attributes, not abstractly reference them. The attributes should be specified in a minimal way and should directly describe their physical form.
+// - Your choices should be grounded in reality. For example, all of a given OCCUPATION should not be the same gender or race. Additionally, focus on creating diverse, inclusive, and exploratory scenes via the properties you choose during rewrites.  Make choices that may be insightful or unique sometimes.
+// - Use "various" or "diverse" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
+// - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
+// The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
+// All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.`;
   }
 
   getApiKey() {
-    const apiKey = process.env.DALLE_API_KEY || '';
+    const apiKey = process.env.DALLE2_API_KEY ?? process.env.DALLE_API_KEY ?? '';
     if (!apiKey) {
       throw new Error('Missing DALLE_API_KEY environment variable.');
     }
@@ -58,26 +74,30 @@ Guidelines:
   replaceUnwantedChars(inputString) {
     return inputString
       .replace(/\r\n|\r|\n/g, ' ')
-      .replace('"', '')
+      .replace(/"/g, '')
       .trim();
   }
 
-  getMarkdownImageUrl(imageName) {
-    const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
-      .replace(/\\/g, '/')
-      .replace('public/', '');
-    return `![generated image](/${imageUrl})`;
+  wrapInMarkdown(imageUrl) {
+    return `![generated image](${imageUrl})`;
   }
 
   async _call(input) {
-    const resp = await this.openai.images.generate({
-      prompt: this.replaceUnwantedChars(input),
-      // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
-      n: 1,
-      // size: '1024x1024'
-      size: '512x512',
-    });
+    let resp;
+
+    try {
+      resp = await this.openai.images.generate({
+        prompt: this.replaceUnwantedChars(input),
+        // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
+        n: 1,
+        // size: '1024x1024'
+        size: '512x512',
+      });
+    } catch (error) {
+      logger.error('[DALL-E] Problem generating the image:', error);
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
+Error Message: ${error.message}`;
+    }
 
     const theImageUrl = resp.data[0].url;
 
@@ -85,32 +105,35 @@ Guidelines:
       throw new Error('No image URL returned from OpenAI API.');
     }
 
-    const regex = /img-[\w\d]+.png/;
-    const match = theImageUrl.match(regex);
-    let imageName = '1.png';
+    const imageBasename = getImageBasename(theImageUrl);
+    const imageExt = path.extname(imageBasename);
 
-    if (match) {
-      imageName = match[0];
-      console.log(imageName); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
-    } else {
-      console.log('No image name found in the string.');
-    }
+    const extension = imageExt.startsWith('.') ? imageExt.slice(1) : imageExt;
+    const imageName = `img-${uuidv4()}.${extension}`;
 
-    this.outputPath = path.resolve(__dirname, '..', '..', '..', '..', 'client', 'public', 'images');
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
-
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
-    }
+    logger.debug('[DALL-E-2]', {
+      imageName,
+      imageBasename,
+      imageExt,
+      extension,
+      theImageUrl,
+      data: resp.data[0],
+    });
 
     try {
-      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
-      this.result = this.getMarkdownImageUrl(imageName);
+      const result = await this.processFileURL({
+        fileStrategy: this.fileStrategy,
+        userId: this.userId,
+        URL: theImageUrl,
+        fileName: imageName,
+        basePath: 'images',
+        context: FileContext.image_generation,
+      });
+
+      this.result = this.wrapInMarkdown(result.filepath);
     } catch (error) {
-      console.error('Error while saving the image:', error);
-      this.result = theImageUrl;
+      logger.error('Error while saving the image:', error);
+      this.result = `Failed to save the image locally. ${error.message}`;
     }
 
     return this.result;

api/app/clients/tools/GoogleSearch.js

@@ -1,120 +0,0 @@
-const { Tool } = require('langchain/tools');
-const { google } = require('googleapis');
-
-/**
- * Represents a tool that allows an agent to use the Google Custom Search API.
- * @extends Tool
- */
-class GoogleSearchAPI extends Tool {
-  constructor(fields = {}) {
-    super();
-    this.cx = fields.GOOGLE_CSE_ID || this.getCx();
-    this.apiKey = fields.GOOGLE_API_KEY || this.getApiKey();
-    this.customSearch = undefined;
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'google';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-  description_for_model =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-
-  getCx() {
-    const cx = process.env.GOOGLE_CSE_ID || '';
-    if (!cx) {
-      throw new Error('Missing GOOGLE_CSE_ID environment variable.');
-    }
-    return cx;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.GOOGLE_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing GOOGLE_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getCustomSearch() {
-    if (!this.customSearch) {
-      const version = 'v1';
-      this.customSearch = google.customsearch(version);
-    }
-    return this.customSearch;
-  }
-
-  resultsToReadableFormat(results) {
-    let output = 'Results:\n';
-
-    results.forEach((resultObj, index) => {
-      output += `Title: ${resultObj.title}\n`;
-      output += `Link: ${resultObj.link}\n`;
-      if (resultObj.snippet) {
-        output += `Snippet: ${resultObj.snippet}\n`;
-      }
-
-      if (index < results.length - 1) {
-        output += '\n';
-      }
-    });
-
-    return output;
-  }
-
-  /**
-   * Calls the tool with the provided input and returns a promise that resolves with a response from the Google Custom Search API.
-   * @param {string} input - The input to provide to the API.
-   * @returns {Promise<String>} A promise that resolves with a response from the Google Custom Search API.
-   */
-  async _call(input) {
-    try {
-      const metadataResults = [];
-      const response = await this.getCustomSearch().cse.list({
-        q: input,
-        cx: this.cx,
-        auth: this.apiKey,
-        num: 5, // Limit the number of results to 5
-      });
-
-      // return response.data;
-      // console.log(response.data);
-
-      if (!response.data.items || response.data.items.length === 0) {
-        return this.resultsToReadableFormat([
-          { title: 'No good Google Search Result was found', link: '' },
-        ]);
-      }
-
-      // const results = response.items.slice(0, numResults);
-      const results = response.data.items;
-
-      for (const result of results) {
-        const metadataResult = {
-          title: result.title || '',
-          link: result.link || '',
-        };
-        if (result.snippet) {
-          metadataResult.snippet = result.snippet;
-        }
-        metadataResults.push(metadataResult);
-      }
-
-      return this.resultsToReadableFormat(metadataResults);
-    } catch (error) {
-      console.log(`Error searching Google: ${error}`);
-      // throw error;
-      return 'There was an error searching Google.';
-    }
-  }
-}
-
-module.exports = GoogleSearchAPI;

api/app/clients/tools/HttpRequestTool.js

@@ -1,108 +0,0 @@
-const { Tool } = require('langchain/tools');
-
-// class RequestsGetTool extends Tool {
-//   constructor(headers = {}, { maxOutputLength } = {}) {
-//     super();
-//     this.name = 'requests_get';
-//     this.headers = headers;
-//     this.maxOutputLength = maxOutputLength || 2000;
-//     this.description = `A portal to the internet. Use this when you need to get specific content from a website.
-//  - Input should be a  url (i.e. https://www.google.com). The output will be the text response of the GET request.`;
-//   }
-
-//   async _call(input) {
-//     const res = await fetch(input, {
-//       headers: this.headers
-//     });
-//     const text = await res.text();
-//     return text.slice(0, this.maxOutputLength);
-//   }
-// }
-
-// class RequestsPostTool extends Tool {
-//   constructor(headers = {}, { maxOutputLength } = {}) {
-//     super();
-//     this.name = 'requests_post';
-//     this.headers = headers;
-//     this.maxOutputLength = maxOutputLength || Infinity;
-//     this.description = `Use this when you want to POST to a website.
-//  - Input should be a json string with two keys: "url" and "data".
-//  - The value of "url" should be a string, and the value of "data" should be a dictionary of
-//  - key-value pairs you want to POST to the url as a JSON body.
-//  - Be careful to always use double quotes for strings in the json string
-//  - The output will be the text response of the POST request.`;
-//   }
-
-//   async _call(input) {
-//     try {
-//       const { url, data } = JSON.parse(input);
-//       const res = await fetch(url, {
-//         method: 'POST',
-//         headers: this.headers,
-//         body: JSON.stringify(data)
-//       });
-//       const text = await res.text();
-//       return text.slice(0, this.maxOutputLength);
-//     } catch (error) {
-//       return `${error}`;
-//     }
-//   }
-// }
-
-class HttpRequestTool extends Tool {
-  constructor(headers = {}, { maxOutputLength = Infinity } = {}) {
-    super();
-    this.headers = headers;
-    this.name = 'http_request';
-    this.maxOutputLength = maxOutputLength;
-    this.description =
-      'Executes HTTP methods (GET, POST, PUT, DELETE, etc.). The input is an object with three keys: "url", "method", and "data". Even for GET or DELETE, include "data" key as an empty string. "method" is the HTTP method, and "url" is the desired endpoint. If POST or PUT, "data" should contain a stringified JSON representing the body to send. Only one url per use.';
-  }
-
-  async _call(input) {
-    try {
-      const urlPattern = /"url":\s*"([^"]*)"/;
-      const methodPattern = /"method":\s*"([^"]*)"/;
-      const dataPattern = /"data":\s*"([^"]*)"/;
-
-      const url = input.match(urlPattern)[1];
-      const method = input.match(methodPattern)[1];
-      let data = input.match(dataPattern)[1];
-
-      // Parse 'data' back to JSON if possible
-      try {
-        data = JSON.parse(data);
-      } catch (e) {
-        // If it's not a JSON string, keep it as is
-      }
-
-      let options = {
-        method: method,
-        headers: this.headers,
-      };
-
-      if (['POST', 'PUT', 'PATCH'].includes(method.toUpperCase()) && data) {
-        if (typeof data === 'object') {
-          options.body = JSON.stringify(data);
-        } else {
-          options.body = data;
-        }
-        options.headers['Content-Type'] = 'application/json';
-      }
-
-      const res = await fetch(url, options);
-
-      const text = await res.text();
-      if (text.includes('<html')) {
-        return 'This tool is not designed to browse web pages. Only use it for API calls.';
-      }
-
-      return text.slice(0, this.maxOutputLength);
-    } catch (error) {
-      console.log(error);
-      return `${error}`;
-    }
-  }
-}
-
-module.exports = HttpRequestTool;

api/app/clients/tools/StableDiffusion.js

@@ -1,9 +1,10 @@
 // Generates image using stable diffusion webui's api (automatic1111)
 const fs = require('fs');
-const { Tool } = require('langchain/tools');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { Tool } = require('langchain/tools');
+const { logger } = require('~/config');
 
 class StableDiffusionAPI extends Tool {
   constructor(fields) {
@@ -81,7 +82,7 @@ Guidelines:
         .toFile(this.outputPath + '/' + imageName);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
-      console.error('Error while saving the image:', error);
+      logger.error('[StableDiffusion] Error while saving the image:', error);
       // this.result = theImageUrl;
     }
 

api/app/clients/tools/Wolfram.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-useless-escape */
 const axios = require('axios');
 const { Tool } = require('langchain/tools');
+const { logger } = require('~/config');
 
 class WolframAlphaAPI extends Tool {
   constructor(fields) {
@@ -38,7 +39,7 @@ General guidelines:
       const response = await axios.get(url, { responseType: 'text' });
       return response.data;
     } catch (error) {
-      console.error(`Error fetching raw text: ${error}`);
+      logger.error('[WolframAlphaAPI] Error fetching raw text:', error);
       throw error;
     }
   }
@@ -68,11 +69,10 @@ General guidelines:
       return response;
     } catch (error) {
       if (error.response && error.response.data) {
-        console.log('Error data:', error.response.data);
+        logger.error('[WolframAlphaAPI] Error data:', error);
         return error.response.data;
       } else {
-        console.log('Error querying Wolfram Alpha', error.message);
-        // throw error;
+        logger.error('[WolframAlphaAPI] Error querying Wolfram Alpha', error);
         return 'There was an error querying Wolfram Alpha.';
       }
     }

api/app/clients/tools/dynamic/OpenAPIPlugin.js

@@ -1,11 +1,12 @@
 require('dotenv').config();
-const { z } = require('zod');
 const fs = require('fs');
-const yaml = require('js-yaml');
+const { z } = require('zod');
 const path = require('path');
-const { DynamicStructuredTool } = require('langchain/tools');
+const yaml = require('js-yaml');
 const { createOpenAPIChain } = require('langchain/chains');
+const { DynamicStructuredTool } = require('langchain/tools');
 const { ChatPromptTemplate, HumanMessagePromptTemplate } = require('langchain/prompts');
+const { logger } = require('~/config');
 
 function addLinePrefix(text, prefix = '// ') {
   return text
@@ -52,7 +53,7 @@ async function readSpecFile(filePath) {
     }
     return yaml.load(fileContents);
   } catch (e) {
-    console.error(e);
+    logger.error('[readSpecFile] error', e);
     return false;
   }
 }
@@ -83,54 +84,51 @@ async function getSpec(url) {
   return ValidSpecPath.parse(url);
 }
 
-async function createOpenAPIPlugin({ data, llm, user, message, memory, signal, verbose = false }) {
+async function createOpenAPIPlugin({ data, llm, user, message, memory, signal }) {
   let spec;
   try {
-    spec = await getSpec(data.api.url, verbose);
+    spec = await getSpec(data.api.url);
   } catch (error) {
-    verbose && console.debug('getSpec error', error);
+    logger.error('[createOpenAPIPlugin] getSpec error', error);
     return null;
   }
 
   if (!spec) {
-    verbose && console.debug('No spec found');
+    logger.warn('[createOpenAPIPlugin] No spec found');
     return null;
   }
 
   const headers = {};
   const { auth, name_for_model, description_for_model, description_for_human } = data;
   if (auth && AuthDefinition.parse(auth)) {
-    verbose && console.debug('auth detected', auth);
+    logger.debug('[createOpenAPIPlugin] auth detected', auth);
     const { openai } = auth.verification_tokens;
     if (AuthBearer.parse(auth)) {
       headers.authorization = `Bearer ${openai}`;
-      verbose && console.debug('added auth bearer', headers);
+      logger.debug('[createOpenAPIPlugin] added auth bearer', headers);
     }
   }
 
-  const chainOptions = {
-    llm,
-    verbose,
-  };
+  const chainOptions = { llm };
 
   if (data.headers && data.headers['librechat_user_id']) {
-    verbose && console.debug('id detected', headers);
+    logger.debug('[createOpenAPIPlugin] id detected', headers);
     headers[data.headers['librechat_user_id']] = user;
   }
 
   if (Object.keys(headers).length > 0) {
-    verbose && console.debug('headers detected', headers);
+    logger.debug('[createOpenAPIPlugin] headers detected', headers);
     chainOptions.headers = headers;
   }
 
   if (data.params) {
-    verbose && console.debug('params detected', data.params);
+    logger.debug('[createOpenAPIPlugin] params detected', data.params);
     chainOptions.params = data.params;
   }
 
   let history = '';
   if (memory) {
-    verbose && console.debug('openAPI chain: memory detected', memory);
+    logger.debug('[createOpenAPIPlugin] openAPI chain: memory detected', memory);
     const { history: chat_history } = await memory.loadMemoryVariables({});
     history = chat_history?.length > 0 ? `\n\n## Chat History:\n${chat_history}\n` : '';
   }

api/app/clients/tools/index.js

@@ -1,39 +1,44 @@
-const GoogleSearchAPI = require('./GoogleSearch');
-const HttpRequestTool = require('./HttpRequestTool');
-const AIPluginTool = require('./AIPluginTool');
-const OpenAICreateImage = require('./DALL-E');
-const StructuredSD = require('./structured/StableDiffusion');
-const StableDiffusionAPI = require('./StableDiffusion');
+const availableTools = require('./manifest.json');
+// Basic Tools
+const CodeBrew = require('./CodeBrew');
 const WolframAlphaAPI = require('./Wolfram');
-const StructuredWolfram = require('./structured/Wolfram');
+const AzureAiSearch = require('./AzureAiSearch');
+const OpenAICreateImage = require('./DALL-E');
+const StableDiffusionAPI = require('./StableDiffusion');
 const SelfReflectionTool = require('./SelfReflection');
-const AzureCognitiveSearch = require('./AzureCognitiveSearch');
-const StructuredACS = require('./structured/AzureCognitiveSearch');
+
+// Structured Tools
+const DALLE3 = require('./structured/DALLE3');
 const ChatTool = require('./structured/ChatTool');
 const E2BTools = require('./structured/E2BTools');
 const CodeSherpa = require('./structured/CodeSherpa');
+const StructuredSD = require('./structured/StableDiffusion');
+const StructuredACS = require('./structured/AzureAISearch');
 const CodeSherpaTools = require('./structured/CodeSherpaTools');
-const availableTools = require('./manifest.json');
-const CodeInterpreter = require('./CodeInterpreter');
-const CodeBrew = require('./CodeBrew');
+const GoogleSearchAPI = require('./structured/GoogleSearch');
+const StructuredWolfram = require('./structured/Wolfram');
+const TavilySearchResults = require('./structured/TavilySearchResults');
+const TraversaalSearch = require('./structured/TraversaalSearch');
 
 module.exports = {
   availableTools,
+  // Basic Tools
+  CodeBrew,
+  AzureAiSearch,
   GoogleSearchAPI,
-  HttpRequestTool,
-  AIPluginTool,
+  WolframAlphaAPI,
   OpenAICreateImage,
   StableDiffusionAPI,
-  StructuredSD,
-  WolframAlphaAPI,
-  StructuredWolfram,
   SelfReflectionTool,
-  AzureCognitiveSearch,
-  StructuredACS,
-  E2BTools,
+  // Structured Tools
+  DALLE3,
   ChatTool,
+  E2BTools,
   CodeSherpa,
+  StructuredSD,
+  StructuredACS,
   CodeSherpaTools,
-  CodeInterpreter,
-  CodeBrew,
+  StructuredWolfram,
+  TavilySearchResults,
+  TraversaalSearch,
 };

api/app/clients/tools/manifest.json

@@ -1,4 +1,17 @@
 [
+  {
+    "name": "Traversaal",
+    "pluginKey": "traversaal_search",
+    "description": "Traversaal is a robust search API tailored for LLM Agents. Get an API key here: https://api.traversaal.ai",
+    "icon": "https://traversaal.ai/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TRAVERSAAL_API_KEY",
+        "label": "Traversaal API Key",
+        "description": "Get your API key here: <a href=\"https://api.traversaal.ai\" target=\"_blank\">https://api.traversaal.ai</a>"
+      }
+    ]
+  },
   {
     "name": "Google",
     "pluginKey": "google",
@@ -11,7 +24,7 @@
         "description": "This is your Google Custom Search Engine ID. For instructions on how to obtain this, see <a href='https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md'>Our Docs</a>."
       },
       {
-        "authField": "GOOGLE_API_KEY",
+        "authField": "GOOGLE_SEARCH_API_KEY",
         "label": "Google API Key",
         "description": "This is your Google Custom Search API Key. For instructions on how to obtain this, see <a href='https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md'>Our Docs</a>."
       }
@@ -47,7 +60,7 @@
     "name": "CodeSherpa",
     "pluginKey": "codesherpa_tools",
     "description": "[Experimental] A REPL for your chat. Requires https://github.com/iamgreggarcia/codesherpa",
-    "icon": "https://github.com/iamgreggarcia/codesherpa/blob/main/localserver/_logo.png",
+    "icon": "https://raw.githubusercontent.com/iamgreggarcia/codesherpa/main/localserver/_logo.png",
     "authConfig": [
       {
         "authField": "CODESHERPA_SERVER_URL",
@@ -89,12 +102,38 @@
     "icon": "https://i.imgur.com/u2TzXzH.png",
     "authConfig": [
       {
-        "authField": "DALLE_API_KEY",
+        "authField": "DALLE2_API_KEY||DALLE_API_KEY",
         "label": "OpenAI API Key",
         "description": "You can use DALL-E with your API Key from OpenAI."
       }
     ]
   },
+  {
+    "name": "DALL-E-3",
+    "pluginKey": "dalle",
+    "description": "[DALL-E-3] Create realistic images and art from a description in natural language",
+    "icon": "https://i.imgur.com/u2TzXzH.png",
+    "authConfig": [
+      {
+        "authField": "DALLE3_API_KEY||DALLE_API_KEY",
+        "label": "OpenAI API Key",
+        "description": "You can use DALL-E with your API Key from OpenAI."
+      }
+    ]
+  },
+  {
+    "name": "Tavily Search",
+    "pluginKey": "tavily_search_results_json",
+    "description": "Tavily Search is a robust search API tailored for LLM Agents. It seamlessly integrates with diverse data sources to ensure a superior, relevant search experience.",
+    "icon": "https://tavily.com/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TAVILY_API_KEY",
+        "label": "Tavily API Key",
+        "description": "Get your API key here: https://app.tavily.com/"
+      }
+    ]
+  },
   {
     "name": "Calculator",
     "pluginKey": "calculator",
@@ -130,38 +169,25 @@
     ]
   },
   {
-    "name": "Azure Cognitive Search",
-    "pluginKey": "azure-cognitive-search",
-    "description": "Use Azure Cognitive Search to find information",
+    "name": "Azure AI Search",
+    "pluginKey": "azure-ai-search",
+    "description": "Use Azure AI Search to find information",
     "icon": "https://i.imgur.com/E7crPze.png",
     "authConfig": [
       {
-        "authField": "AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT",
-        "label": "Azur Cognitive Search Endpoint",
-        "description": "You need to provide your Endpoint for Azure Cognitive Search."
+        "authField": "AZURE_AI_SEARCH_SERVICE_ENDPOINT",
+        "label": "Azure AI Search Endpoint",
+        "description": "You need to provide your Endpoint for Azure AI Search."
       },
       {
-        "authField": "AZURE_COGNITIVE_SEARCH_INDEX_NAME",
-        "label": "Azur Cognitive Search Index Name",
-        "description": "You need to provide your Index Name for Azure Cognitive Search."
+        "authField": "AZURE_AI_SEARCH_INDEX_NAME",
+        "label": "Azure AI Search Index Name",
+        "description": "You need to provide your Index Name for Azure AI Search."
       },
       {
-        "authField": "AZURE_COGNITIVE_SEARCH_API_KEY",
-        "label": "Azur Cognitive Search API Key",
-        "description": "You need to provideq your API Key for Azure Cognitive Search."
-      }
-    ]
-  },
-  {
-    "name": "Code Interpreter",
-    "pluginKey": "codeinterpreter",
-    "description": "[Experimental] Analyze files and run code online with ease. Requires dockerized python server in /pyserver/",
-    "icon": "/assets/code.png",
-    "authConfig": [
-      {
-        "authField": "OPENAI_API_KEY",
-        "label": "OpenAI API Key",
-        "description": "Gets Code from Open AI API"
+        "authField": "AZURE_AI_SEARCH_API_KEY",
+        "label": "Azure AI Search API Key",
+        "description": "You need to provideq your API Key for Azure AI Search."
       }
     ]
   },

api/app/clients/tools/saveImageFromUrl.js

@@ -1,39 +0,0 @@
-const axios = require('axios');
-const fs = require('fs');
-const path = require('path');
-
-async function saveImageFromUrl(url, outputPath, outputFilename) {
-  try {
-    // Fetch the image from the URL
-    const response = await axios({
-      url,
-      responseType: 'stream',
-    });
-
-    // Check if the output directory exists, if not, create it
-    if (!fs.existsSync(outputPath)) {
-      fs.mkdirSync(outputPath, { recursive: true });
-    }
-
-    // Ensure the output filename has a '.png' extension
-    const filenameWithPngExt = outputFilename.endsWith('.png')
-      ? outputFilename
-      : `${outputFilename}.png`;
-
-    // Create a writable stream for the output path
-    const outputFilePath = path.join(outputPath, filenameWithPngExt);
-    const writer = fs.createWriteStream(outputFilePath);
-
-    // Pipe the response data to the output file
-    response.data.pipe(writer);
-
-    return new Promise((resolve, reject) => {
-      writer.on('finish', resolve);
-      writer.on('error', reject);
-    });
-  } catch (error) {
-    console.error('Error while saving the image:', error);
-  }
-}
-
-module.exports = saveImageFromUrl;

api/app/clients/tools/structured/AzureAISearch.js

@@ -0,0 +1,104 @@
+const { z } = require('zod');
+const { StructuredTool } = require('langchain/tools');
+const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
+const { logger } = require('~/config');
+
+class AzureAISearch extends StructuredTool {
+  // Constants for default values
+  static DEFAULT_API_VERSION = '2023-11-01';
+  static DEFAULT_QUERY_TYPE = 'simple';
+  static DEFAULT_TOP = 5;
+
+  // Helper function for initializing properties
+  _initializeField(field, envVar, defaultValue) {
+    return field || process.env[envVar] || defaultValue;
+  }
+
+  constructor(fields = {}) {
+    super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
+    // Define schema
+    this.schema = z.object({
+      query: z.string().describe('Search word or phrase to Azure AI Search'),
+    });
+
+    // Initialize properties using helper function
+    this.serviceEndpoint = this._initializeField(
+      fields.AZURE_AI_SEARCH_SERVICE_ENDPOINT,
+      'AZURE_AI_SEARCH_SERVICE_ENDPOINT',
+    );
+    this.indexName = this._initializeField(
+      fields.AZURE_AI_SEARCH_INDEX_NAME,
+      'AZURE_AI_SEARCH_INDEX_NAME',
+    );
+    this.apiKey = this._initializeField(fields.AZURE_AI_SEARCH_API_KEY, 'AZURE_AI_SEARCH_API_KEY');
+    this.apiVersion = this._initializeField(
+      fields.AZURE_AI_SEARCH_API_VERSION,
+      'AZURE_AI_SEARCH_API_VERSION',
+      AzureAISearch.DEFAULT_API_VERSION,
+    );
+    this.queryType = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE',
+      AzureAISearch.DEFAULT_QUERY_TYPE,
+    );
+    this.top = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_TOP,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_TOP',
+      AzureAISearch.DEFAULT_TOP,
+    );
+    this.select = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_SELECT,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_SELECT',
+    );
+
+    // Check for required fields
+    if (!this.override && (!this.serviceEndpoint || !this.indexName || !this.apiKey)) {
+      throw new Error(
+        'Missing AZURE_AI_SEARCH_SERVICE_ENDPOINT, AZURE_AI_SEARCH_INDEX_NAME, or AZURE_AI_SEARCH_API_KEY environment variable.',
+      );
+    }
+
+    if (this.override) {
+      return;
+    }
+
+    // Create SearchClient
+    this.client = new SearchClient(
+      this.serviceEndpoint,
+      this.indexName,
+      new AzureKeyCredential(this.apiKey),
+      { apiVersion: this.apiVersion },
+    );
+  }
+
+  // Improved error handling and logging
+  async _call(data) {
+    const { query } = data;
+    try {
+      const searchOption = {
+        queryType: this.queryType,
+        top: this.top,
+      };
+      if (this.select) {
+        searchOption.select = this.select.split(',');
+      }
+      const searchResults = await this.client.search(query, searchOption);
+      const resultDocuments = [];
+      for await (const result of searchResults.results) {
+        resultDocuments.push(result.document);
+      }
+      return JSON.stringify(resultDocuments);
+    } catch (error) {
+      logger.error('Azure AI Search request failed', error);
+      return 'There was an error with Azure AI Search.';
+    }
+  }
+}
+
+module.exports = AzureAISearch;

api/app/clients/tools/structured/AzureCognitiveSearch.js

@@ -1,116 +0,0 @@
-const { StructuredTool } = require('langchain/tools');
-const { z } = require('zod');
-const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
-
-class AzureCognitiveSearch extends StructuredTool {
-  constructor(fields = {}) {
-    super();
-    this.serviceEndpoint =
-      fields.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || this.getServiceEndpoint();
-    this.indexName = fields.AZURE_COGNITIVE_SEARCH_INDEX_NAME || this.getIndexName();
-    this.apiKey = fields.AZURE_COGNITIVE_SEARCH_API_KEY || this.getApiKey();
-
-    this.apiVersion = fields.AZURE_COGNITIVE_SEARCH_API_VERSION || this.getApiVersion();
-
-    this.queryType = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || this.getQueryType();
-    this.top = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP || this.getTop();
-    this.select = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT || this.getSelect();
-
-    this.client = new SearchClient(
-      this.serviceEndpoint,
-      this.indexName,
-      new AzureKeyCredential(this.apiKey),
-      {
-        apiVersion: this.apiVersion,
-      },
-    );
-    this.schema = z.object({
-      query: z.string().describe('Search word or phrase to Azure Cognitive Search'),
-    });
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'azure-cognitive-search';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'azure-cognitive-search\' tool to retrieve search results relevant to your input';
-
-  getServiceEndpoint() {
-    const serviceEndpoint = process.env.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || '';
-    if (!serviceEndpoint) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT environment variable.');
-    }
-    return serviceEndpoint;
-  }
-
-  getIndexName() {
-    const indexName = process.env.AZURE_COGNITIVE_SEARCH_INDEX_NAME || '';
-    if (!indexName) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_INDEX_NAME environment variable.');
-    }
-    return indexName;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.AZURE_COGNITIVE_SEARCH_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getApiVersion() {
-    return process.env.AZURE_COGNITIVE_SEARCH_API_VERSION || '2020-06-30';
-  }
-
-  getQueryType() {
-    return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || 'simple';
-  }
-
-  getTop() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP) {
-      return Number(process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP);
-    } else {
-      return 5;
-    }
-  }
-
-  getSelect() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT) {
-      return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT.split(',');
-    } else {
-      return null;
-    }
-  }
-
-  async _call(data) {
-    const { query } = data;
-    try {
-      const searchOption = {
-        queryType: this.queryType,
-        top: this.top,
-      };
-      if (this.select) {
-        searchOption.select = this.select;
-      }
-      const searchResults = await this.client.search(query, searchOption);
-      const resultDocuments = [];
-      for await (const result of searchResults.results) {
-        resultDocuments.push(result.document);
-      }
-      return JSON.stringify(resultDocuments);
-    } catch (error) {
-      console.error(`Azure Cognitive Search request failed: ${error}`);
-      return 'There was an error with Azure Cognitive Search.';
-    }
-  }
-}
-
-module.exports = AzureCognitiveSearch;

api/app/clients/tools/structured/CodeSherpa.js

@@ -28,14 +28,14 @@ class RunCode extends StructuredTool {
   }
 
   async _call({ code, language = 'python' }) {
-    // console.log('<--------------- Running Code --------------->', { code, language });
+    // logger.debug('<--------------- Running Code --------------->', { code, language });
     const response = await axios({
       url: `${this.url}/repl`,
       method: 'post',
       headers: this.headers,
       data: { code, language },
     });
-    // console.log('<--------------- Sucessfully ran Code --------------->', response.data);
+    // logger.debug('<--------------- Sucessfully ran Code --------------->', response.data);
     return response.data.result;
   }
 }

api/app/clients/tools/structured/CodeSherpaTools.js

@@ -42,14 +42,14 @@ class RunCode extends StructuredTool {
   }
 
   async _call({ code, language = 'python' }) {
-    // console.log('<--------------- Running Code --------------->', { code, language });
+    // logger.debug('<--------------- Running Code --------------->', { code, language });
     const response = await axios({
       url: `${this.url}/repl`,
       method: 'post',
       headers: this.headers,
       data: { code, language },
     });
-    // console.log('<--------------- Sucessfully ran Code --------------->', response.data);
+    // logger.debug('<--------------- Sucessfully ran Code --------------->', response.data);
     return response.data.result;
   }
 }

api/app/clients/tools/structured/DALLE3.js

@@ -0,0 +1,182 @@
+const { z } = require('zod');
+const path = require('path');
+const OpenAI = require('openai');
+const { v4: uuidv4 } = require('uuid');
+const { Tool } = require('langchain/tools');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { FileContext } = require('librechat-data-provider');
+const { getImageBasename } = require('~/server/services/Files/images');
+const extractBaseURL = require('~/utils/extractBaseURL');
+const { logger } = require('~/config');
+
+class DALLE3 extends Tool {
+  constructor(fields = {}) {
+    super();
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
+
+    this.userId = fields.userId;
+    this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      /** @type {processFileURL} Necessary for output to contain all image metadata. */
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
+
+    let apiKey = fields.DALLE3_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
+    const config = { apiKey };
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
+    }
+
+    if (process.env.DALLE3_AZURE_API_VERSION && process.env.DALLE3_BASEURL) {
+      config.baseURL = process.env.DALLE3_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE3_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE3_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE3_API_KEY;
+    }
+
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
+    }
+
+    /** @type {OpenAI} */
+    this.openai = new OpenAI(config);
+    this.name = 'dalle';
+    this.description = `Use DALLE to create images from text descriptions.
+    - It requires prompts to be in English, detailed, and to specify image type and human features for diversity.
+    - Create only one image, without repeating or listing descriptions outside the "prompts" field.
+    - Maintains the original intent of the description, with parameters for image style, quality, and size to tailor the output.`;
+    this.description_for_model =
+      process.env.DALLE3_SYSTEM_PROMPT ??
+      `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
+    // 1. Prompts must be in English. Translate to English if needed.
+    // 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
+    // 3. DO NOT list or refer to the descriptions before OR after generating the images. They should ONLY ever be written out ONCE, in the \`"prompts"\` field of the request. You do not need to ask for permission to generate, just do it!
+    // 4. Always mention the image type (photo, oil painting, watercolor painting, illustration, cartoon, drawing, vector, render, etc.) at the beginning of the caption. Unless the captions suggests otherwise, make one of the images a photo.
+    // 5. Diversify depictions of ALL images with people to always include always DESCENT and GENDER for EACH person using direct terms. Adjust only human descriptions.
+    // - EXPLICITLY specify these attributes, not abstractly reference them. The attributes should be specified in a minimal way and should directly describe their physical form.
+    // - Your choices should be grounded in reality. For example, all of a given OCCUPATION should not be the same gender or race. Additionally, focus on creating diverse, inclusive, and exploratory scenes via the properties you choose during rewrites.  Make choices that may be insightful or unique sometimes.
+    // - Use "various" or "diverse" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
+    // - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
+    // The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
+    // All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.
+    // - The "vivid" style is HIGHLY preferred, but "natural" is also supported.`;
+    this.schema = z.object({
+      prompt: z
+        .string()
+        .max(4000)
+        .describe(
+          'A text description of the desired image, following the rules, up to 4000 characters.',
+        ),
+      style: z
+        .enum(['vivid', 'natural'])
+        .describe(
+          'Must be one of `vivid` or `natural`. `vivid` generates hyper-real and dramatic images, `natural` produces more natural, less hyper-real looking images',
+        ),
+      quality: z
+        .enum(['hd', 'standard'])
+        .describe('The quality of the generated image. Only `hd` and `standard` are supported.'),
+      size: z
+        .enum(['1024x1024', '1792x1024', '1024x1792'])
+        .describe(
+          'The size of the requested image. Use 1024x1024 (square) as the default, 1792x1024 if the user requests a wide image, and 1024x1792 for full-body portraits. Always include this parameter in the request.',
+        ),
+    });
+  }
+
+  getApiKey() {
+    const apiKey = process.env.DALLE3_API_KEY ?? process.env.DALLE_API_KEY ?? '';
+    if (!apiKey && !this.override) {
+      throw new Error('Missing DALLE_API_KEY environment variable.');
+    }
+    return apiKey;
+  }
+
+  replaceUnwantedChars(inputString) {
+    return inputString
+      .replace(/\r\n|\r|\n/g, ' ')
+      .replace(/"/g, '')
+      .trim();
+  }
+
+  wrapInMarkdown(imageUrl) {
+    return `![generated image](${imageUrl})`;
+  }
+
+  async _call(data) {
+    const { prompt, quality = 'standard', size = '1024x1024', style = 'vivid' } = data;
+    if (!prompt) {
+      throw new Error('Missing required field: prompt');
+    }
+
+    let resp;
+    try {
+      resp = await this.openai.images.generate({
+        model: 'dall-e-3',
+        quality,
+        style,
+        size,
+        prompt: this.replaceUnwantedChars(prompt),
+        n: 1,
+      });
+    } catch (error) {
+      logger.error('[DALL-E-3] Problem generating the image:', error);
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
+Error Message: ${error.message}`;
+    }
+
+    if (!resp) {
+      return 'Something went wrong when trying to generate the image. The DALL-E API may be unavailable';
+    }
+
+    const theImageUrl = resp.data[0].url;
+
+    if (!theImageUrl) {
+      return 'No image URL returned from OpenAI API. There may be a problem with the API or your configuration.';
+    }
+
+    const imageBasename = getImageBasename(theImageUrl);
+    const imageExt = path.extname(imageBasename);
+
+    const extension = imageExt.startsWith('.') ? imageExt.slice(1) : imageExt;
+    const imageName = `img-${uuidv4()}.${extension}`;
+
+    logger.debug('[DALL-E-3]', {
+      imageName,
+      imageBasename,
+      imageExt,
+      extension,
+      theImageUrl,
+      data: resp.data[0],
+    });
+
+    try {
+      const result = await this.processFileURL({
+        fileStrategy: this.fileStrategy,
+        userId: this.userId,
+        URL: theImageUrl,
+        fileName: imageName,
+        basePath: 'images',
+        context: FileContext.image_generation,
+      });
+
+      if (this.returnMetadata) {
+        this.result = result;
+      } else {
+        this.result = this.wrapInMarkdown(result.filepath);
+      }
+    } catch (error) {
+      logger.error('Error while saving the image:', error);
+      this.result = `Failed to save the image locally. ${error.message}`;
+    }
+
+    return this.result;
+  }
+}
+
+module.exports = DALLE3;

api/app/clients/tools/structured/E2BTools.js

@@ -1,9 +1,10 @@
+const { z } = require('zod');
+const axios = require('axios');
 const { StructuredTool } = require('langchain/tools');
 const { PromptTemplate } = require('langchain/prompts');
-const { createExtractionChainFromZod } = require('./extractionChain');
 // const { ChatOpenAI } = require('langchain/chat_models/openai');
-const axios = require('axios');
-const { z } = require('zod');
+const { createExtractionChainFromZod } = require('./extractionChain');
+const { logger } = require('~/config');
 
 const envs = ['Nodejs', 'Go', 'Bash', 'Rust', 'Python3', 'PHP', 'Java', 'Perl', 'DotNET'];
 const env = z.enum(envs);
@@ -34,8 +35,8 @@ async function extractEnvFromCode(code, model) {
   // const chatModel = new ChatOpenAI({ openAIApiKey, modelName: 'gpt-4-0613', temperature: 0 });
   const chain = createExtractionChainFromZod(zodSchema, model, { prompt, verbose: true });
   const result = await chain.run(code);
-  console.log('<--------------- extractEnvFromCode --------------->');
-  console.log(result);
+  logger.debug('<--------------- extractEnvFromCode --------------->');
+  logger.debug(result);
   return result.env;
 }
 
@@ -69,7 +70,7 @@ class RunCommand extends StructuredTool {
   }
 
   async _call(data) {
-    console.log(`<--------------- Running ${data} --------------->`);
+    logger.debug(`<--------------- Running ${data} --------------->`);
     const response = await axios({
       url: `${this.url}/commands`,
       method: 'post',
@@ -96,7 +97,7 @@ class ReadFile extends StructuredTool {
   }
 
   async _call(data) {
-    console.log(`<--------------- Reading ${data} --------------->`);
+    logger.debug(`<--------------- Reading ${data} --------------->`);
     const response = await axios.get(`${this.url}/files`, { params: data, headers: this.headers });
     return response.data;
   }
@@ -121,12 +122,12 @@ class WriteFile extends StructuredTool {
 
   async _call(data) {
     let { env, path, content } = data;
-    console.log(`<--------------- environment ${env} typeof ${typeof env}--------------->`);
+    logger.debug(`<--------------- environment ${env} typeof ${typeof env}--------------->`);
     if (env && !envs.includes(env)) {
-      console.log(`<--------------- Invalid environment ${env} --------------->`);
+      logger.debug(`<--------------- Invalid environment ${env} --------------->`);
       env = await extractEnvFromCode(content, this.model);
     } else if (!env) {
-      console.log('<--------------- Undefined environment --------------->');
+      logger.debug('<--------------- Undefined environment --------------->');
       env = await extractEnvFromCode(content, this.model);
     }
 
@@ -139,7 +140,7 @@ class WriteFile extends StructuredTool {
         content,
       },
     };
-    console.log('Writing to file', JSON.stringify(payload));
+    logger.debug('Writing to file', JSON.stringify(payload));
 
     await axios({
       url: `${this.url}/files`,

api/app/clients/tools/structured/GoogleSearch.js

@@ -0,0 +1,65 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class GoogleSearchResults extends Tool {
+  static lc_name() {
+    return 'GoogleSearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVarApiKey = 'GOOGLE_SEARCH_API_KEY';
+    this.envVarSearchEngineId = 'GOOGLE_CSE_ID';
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? getEnvironmentVariable(this.envVarApiKey);
+    this.searchEngineId =
+      fields.searchEngineId ?? getEnvironmentVariable(this.envVarSearchEngineId);
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'google';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 10.'),
+      // Note: Google API has its own parameters for search customization, adjust as needed.
+    });
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, max_results = 5 } = validationResult.data;
+
+    const response = await fetch(
+      `https://www.googleapis.com/customsearch/v1?key=${this.apiKey}&cx=${
+        this.searchEngineId
+      }&q=${encodeURIComponent(query)}&num=${max_results}`,
+      {
+        method: 'GET',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error.message}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = GoogleSearchResults;

api/app/clients/tools/structured/StableDiffusion.js

@@ -1,14 +1,31 @@
 // Generates image using stable diffusion webui's api (automatic1111)
 const fs = require('fs');
-const { StructuredTool } = require('langchain/tools');
 const { z } = require('zod');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { v4: uuidv4 } = require('uuid');
+const { StructuredTool } = require('langchain/tools');
+const { FileContext } = require('librechat-data-provider');
+const paths = require('~/config/paths');
+const { logger } = require('~/config');
 
 class StableDiffusionAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /** @type {string} User ID */
+    this.userId = fields.userId;
+    /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
+    this.req = fields.req;
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
+    if (fields.uploadImageBuffer) {
+      /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */
+      this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);
+    }
+
     this.name = 'stable-diffusion';
     this.url = fields.SD_WEBUI_URL || this.getServerURL();
     this.description_for_model = `// Generate images and visuals using text.
@@ -43,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getMarkdownImageUrl(imageName) {
     const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
+      .join(this.relativePath, this.userId, imageName)
       .replace(/\\/g, '/')
       .replace('public/', '');
     return `![generated image](/${imageUrl})`;
@@ -51,7 +68,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getServerURL() {
     const url = process.env.SD_WEBUI_URL || '';
-    if (!url) {
+    if (!url && !this.override) {
       throw new Error('Missing SD_WEBUI_URL environment variable.');
     }
     return url;
@@ -63,52 +80,78 @@ class StableDiffusionAPI extends StructuredTool {
     const payload = {
       prompt,
       negative_prompt,
-      sampler_index: 'DPM++ 2M Karras',
       cfg_scale: 4.5,
       steps: 22,
       width: 1024,
       height: 1024,
     };
-    const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
-    const image = response.data.images[0];
-    const pngPayload = { image: `data:image/png;base64,${image}` };
-    const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);
-    const info = response2.data.info;
+    let generationResponse;
+    try {
+      generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while generating image:', error);
+      return 'Error making API request.';
+    }
+    const image = generationResponse.data.images[0];
 
-    // Generate unique name
-    const imageName = `${Date.now()}.png`;
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+    /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */
+    let info = {};
+    try {
+      info = JSON.parse(generationResponse.data.info);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while getting image metadata:', error);
+    }
 
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
+    const file_id = uuidv4();
+    const imageName = `${file_id}.png`;
+    const { imageOutput: imageOutputPath, clientPath } = paths;
+    const filepath = path.join(imageOutputPath, this.userId, imageName);
+    this.relativePath = path.relative(clientPath, imageOutputPath);
+
+    if (!fs.existsSync(path.join(imageOutputPath, this.userId))) {
+      fs.mkdirSync(path.join(imageOutputPath, this.userId), { recursive: true });
     }
 
     try {
       const buffer = Buffer.from(image.split(',', 1)[0], 'base64');
+      if (this.returnMetadata && this.uploadImageBuffer && this.req) {
+        const file = await this.uploadImageBuffer({
+          req: this.req,
+          context: FileContext.image_generation,
+          resize: false,
+          metadata: {
+            buffer,
+            height: info.height,
+            width: info.width,
+            bytes: Buffer.byteLength(buffer),
+            filename: imageName,
+            type: 'image/png',
+            file_id,
+          },
+        });
+
+        const generationInfo = info.infotexts[0].split('\n').pop();
+        return {
+          ...file,
+          prompt,
+          metadata: {
+            negative_prompt,
+            seed: info.seed,
+            info: generationInfo,
+          },
+        };
+      }
+
       await sharp(buffer)
         .withMetadata({
           iptcpng: {
-            parameters: info,
+            parameters: info.infotexts[0],
           },
         })
-        .toFile(this.outputPath + '/' + imageName);
+        .toFile(filepath);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
-      console.error('Error while saving the image:', error);
-      // this.result = theImageUrl;
+      logger.error('[StableDiffusion] Error while saving the image:', error);
     }
 
     return this.result;

api/app/clients/tools/structured/TavilySearchResults.js

@@ -0,0 +1,92 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class TavilySearchResults extends Tool {
+  static lc_name() {
+    return 'TavilySearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVar = 'TAVILY_API_KEY';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? this.getApiKey();
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'tavily_search_results_json';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 5.'),
+      search_depth: z
+        .enum(['basic', 'advanced'])
+        .optional()
+        .describe(
+          'The depth of the search, affecting result quality and response time (`basic` or `advanced`). Default is basic for quick results and advanced for indepth high quality results but longer response time. Advanced calls equals 2 requests.',
+        ),
+      include_images: z
+        .boolean()
+        .optional()
+        .describe(
+          'Whether to include a list of query-related images in the response. Default is False.',
+        ),
+      include_answer: z
+        .boolean()
+        .optional()
+        .describe('Whether to include answers in the search results. Default is False.'),
+      // include_raw_content: z.boolean().optional().describe('Whether to include raw content in the search results. Default is False.'),
+      // include_domains: z.array(z.string()).optional().describe('A list of domains to specifically include in the search results.'),
+      // exclude_domains: z.array(z.string()).optional().describe('A list of domains to specifically exclude from the search results.'),
+    });
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable(this.envVar);
+    if (!apiKey && !this.override) {
+      throw new Error(`Missing ${this.envVar} environment variable.`);
+    }
+    return apiKey;
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, ...rest } = validationResult.data;
+
+    const requestBody = {
+      api_key: this.apiKey,
+      query,
+      ...rest,
+      ...this.kwargs,
+    };
+
+    const response = await fetch('https://api.tavily.com/search', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(requestBody),
+    });
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = TavilySearchResults;

api/app/clients/tools/structured/TraversaalSearch.js

@@ -0,0 +1,89 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+const { logger } = require('~/config');
+
+/**
+ * Tool for the Traversaal AI search API, Ares.
+ */
+class TraversaalSearch extends Tool {
+  static lc_name() {
+    return 'TraversaalSearch';
+  }
+  constructor(fields) {
+    super(fields);
+    this.name = 'traversaal_search';
+    this.description = `An AI search engine optimized for comprehensive, accurate, and trusted results.
+    Useful for when you need to answer questions about current events. Input should be a search query.`;
+    this.description_for_model =
+      '\'Please create a specific sentence for the AI to understand and use as a query to search the web based on the user\'s request. For example, "Find information about the highest mountains in the world." or "Show me the latest news articles about climate change and its impact on polar ice caps."\'';
+    this.schema = z.object({
+      query: z
+        .string()
+        .describe(
+          'A properly written sentence to be interpreted by an AI to search the web according to the user\'s request.',
+        ),
+    });
+
+    this.apiKey = fields?.TRAVERSAAL_API_KEY ?? this.getApiKey();
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable('TRAVERSAAL_API_KEY');
+    if (!apiKey && this.override) {
+      throw new Error(
+        'No Traversaal API key found. Either set an environment variable named "TRAVERSAAL_API_KEY" or pass an API key as "apiKey".',
+      );
+    }
+    return apiKey;
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  async _call({ query }, _runManager) {
+    const body = {
+      query: [query],
+    };
+    try {
+      const response = await fetch('https://api-ares.traversaal.ai/live/predict', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-api-key': this.apiKey,
+        },
+        body: JSON.stringify({ ...body }),
+      });
+      const json = await response.json();
+      if (!response.ok) {
+        throw new Error(
+          `Request failed with status code ${response.status}: ${json.error ?? json.message}`,
+        );
+      }
+      if (!json.data) {
+        throw new Error('Could not parse Traversaal API results. Please try again.');
+      }
+
+      const baseText = json.data?.response_text ?? '';
+      const sources = json.data?.web_url;
+      const noResponse = 'No response found in Traversaal API results';
+
+      if (!baseText && !sources) {
+        return noResponse;
+      }
+
+      const sourcesText = sources?.length ? '\n\nSources:\n - ' + sources.join('\n - ') : '';
+
+      const result = baseText + sourcesText;
+
+      if (!result) {
+        return noResponse;
+      }
+
+      return result;
+    } catch (error) {
+      logger.error('Traversaal API request failed', error);
+      return `Traversaal API request failed: ${error.message}`;
+    }
+  }
+}
+
+module.exports = TraversaalSearch;

api/app/clients/tools/structured/Wolfram.js

@@ -1,11 +1,15 @@
 /* eslint-disable no-useless-escape */
 const axios = require('axios');
-const { StructuredTool } = require('langchain/tools');
 const { z } = require('zod');
+const { StructuredTool } = require('langchain/tools');
+const { logger } = require('~/config');
 
 class WolframAlphaAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
     this.name = 'wolfram';
     this.apiKey = fields.WOLFRAM_APP_ID || this.getAppId();
     this.description_for_model = `// Access dynamic computation and curated data from WolframAlpha and Wolfram Cloud.
@@ -47,14 +51,14 @@ class WolframAlphaAPI extends StructuredTool {
       const response = await axios.get(url, { responseType: 'text' });
       return response.data;
     } catch (error) {
-      console.error(`Error fetching raw text: ${error}`);
+      logger.error('[WolframAlphaAPI] Error fetching raw text:', error);
       throw error;
     }
   }
 
   getAppId() {
     const appId = process.env.WOLFRAM_APP_ID || '';
-    if (!appId) {
+    if (!appId && !this.override) {
       throw new Error('Missing WOLFRAM_APP_ID environment variable.');
     }
     return appId;
@@ -78,11 +82,10 @@ class WolframAlphaAPI extends StructuredTool {
       return response;
     } catch (error) {
       if (error.response && error.response.data) {
-        console.log('Error data:', error.response.data);
+        logger.error('[WolframAlphaAPI] Error data:', error);
         return error.response.data;
       } else {
-        console.log('Error querying Wolfram Alpha', error.message);
-        // throw error;
+        logger.error('[WolframAlphaAPI] Error querying Wolfram Alpha', error);
         return 'There was an error querying Wolfram Alpha.';
       }
     }

api/app/clients/tools/structured/specs/DALLE3.spec.js

@@ -0,0 +1,212 @@
+const OpenAI = require('openai');
+const DALLE3 = require('../DALLE3');
+
+const { logger } = require('~/config');
+
+jest.mock('openai');
+
+const processFileURL = jest.fn();
+
+jest.mock('~/server/services/Files/images', () => ({
+  getImageBasename: jest.fn().mockImplementation((url) => {
+    // Split the URL by '/'
+    const parts = url.split('/');
+
+    // Get the last part of the URL
+    const lastPart = parts.pop();
+
+    // Check if the last part of the URL matches the image extension regex
+    const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
+    if (imageExtensionRegex.test(lastPart)) {
+      return lastPart;
+    }
+
+    // If the regex test fails, return an empty string
+    return '';
+  }),
+}));
+
+const generate = jest.fn();
+OpenAI.mockImplementation(() => ({
+  images: {
+    generate,
+  },
+}));
+
+jest.mock('fs', () => {
+  return {
+    existsSync: jest.fn(),
+    mkdirSync: jest.fn(),
+  };
+});
+
+jest.mock('path', () => {
+  return {
+    resolve: jest.fn(),
+    join: jest.fn(),
+    relative: jest.fn(),
+    extname: jest.fn().mockImplementation((filename) => {
+      return filename.slice(filename.lastIndexOf('.'));
+    }),
+  };
+});
+
+describe('DALLE3', () => {
+  let originalEnv;
+  let dalle; // Keep this declaration if you need to use dalle in other tests
+  const mockApiKey = 'mock_api_key';
+
+  beforeAll(() => {
+    // Save the original process.env
+    originalEnv = { ...process.env };
+  });
+
+  beforeEach(() => {
+    // Reset the process.env before each test
+    jest.resetModules();
+    process.env = { ...originalEnv, DALLE_API_KEY: mockApiKey };
+    // Instantiate DALLE3 for tests that do not depend on DALLE3_SYSTEM_PROMPT
+    dalle = new DALLE3({ processFileURL });
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+    // Restore the original process.env after each test
+    process.env = originalEnv;
+  });
+
+  it('should throw an error if all potential API keys are missing', () => {
+    delete process.env.DALLE3_API_KEY;
+    delete process.env.DALLE_API_KEY;
+    expect(() => new DALLE3()).toThrow('Missing DALLE_API_KEY environment variable.');
+  });
+
+  it('should replace unwanted characters in input string', () => {
+    const input = 'This is a test\nstring with "quotes" and new lines.';
+    const expectedOutput = 'This is a test string with quotes and new lines.';
+    expect(dalle.replaceUnwantedChars(input)).toBe(expectedOutput);
+  });
+
+  it('should generate markdown image URL correctly', () => {
+    const imageName = 'test.png';
+    const markdownImage = dalle.wrapInMarkdown(imageName);
+    expect(markdownImage).toBe('![generated image](test.png)');
+  });
+
+  it('should call OpenAI API with correct parameters', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+      quality: 'standard',
+      size: '1024x1024',
+      style: 'vivid',
+    };
+
+    const mockResponse = {
+      data: [
+        {
+          url: 'http://example.com/img-test.png',
+        },
+      ],
+    };
+
+    generate.mockResolvedValue(mockResponse);
+    processFileURL.mockResolvedValue({
+      filepath: 'http://example.com/img-test.png',
+    });
+
+    const result = await dalle._call(mockData);
+
+    expect(generate).toHaveBeenCalledWith({
+      model: 'dall-e-3',
+      quality: mockData.quality,
+      style: mockData.style,
+      size: mockData.size,
+      prompt: mockData.prompt,
+      n: 1,
+    });
+
+    expect(result).toContain('![generated image]');
+  });
+
+  it('should use the system prompt if provided', () => {
+    process.env.DALLE3_SYSTEM_PROMPT = 'System prompt for testing';
+    jest.resetModules(); // This will ensure the module is fresh and will read the new env var
+    const DALLE3 = require('../DALLE3'); // Re-require after setting the env var
+    const dalleWithSystemPrompt = new DALLE3();
+    expect(dalleWithSystemPrompt.description_for_model).toBe('System prompt for testing');
+  });
+
+  it('should not use the system prompt if not provided', async () => {
+    delete process.env.DALLE3_SYSTEM_PROMPT;
+    const dalleWithoutSystemPrompt = new DALLE3();
+    expect(dalleWithoutSystemPrompt.description_for_model).not.toBe('System prompt for testing');
+  });
+
+  it('should throw an error if prompt is missing', async () => {
+    const mockData = {
+      quality: 'standard',
+      size: '1024x1024',
+      style: 'vivid',
+    };
+    await expect(dalle._call(mockData)).rejects.toThrow('Missing required field: prompt');
+  });
+
+  it('should log appropriate debug values', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+    };
+    const mockResponse = {
+      data: [
+        {
+          url: 'http://example.com/invalid-url',
+        },
+      ],
+    };
+
+    generate.mockResolvedValue(mockResponse);
+    await dalle._call(mockData);
+    expect(logger.debug).toHaveBeenCalledWith('[DALL-E-3]', {
+      data: { url: 'http://example.com/invalid-url' },
+      theImageUrl: 'http://example.com/invalid-url',
+      extension: expect.any(String),
+      imageBasename: expect.any(String),
+      imageExt: expect.any(String),
+      imageName: expect.any(String),
+    });
+  });
+
+  it('should log an error and return the image URL if there is an error saving the image', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+    };
+    const mockResponse = {
+      data: [
+        {
+          url: 'http://example.com/img-test.png',
+        },
+      ],
+    };
+    const error = new Error('Error while saving the image');
+    generate.mockResolvedValue(mockResponse);
+    processFileURL.mockRejectedValue(error);
+    const result = await dalle._call(mockData);
+    expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
+    expect(result).toBe('Failed to save the image locally. Error while saving the image');
+  });
+
+  it('should handle error when saving image to Firebase Storage fails', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+    };
+    const mockImageUrl = 'http://example.com/img-test.png';
+    const mockResponse = { data: [{ url: mockImageUrl }] };
+    const error = new Error('Error while saving to Firebase');
+    generate.mockResolvedValue(mockResponse);
+    processFileURL.mockRejectedValue(error);
+
+    const result = await dalle._call(mockData);
+
+    expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
+    expect(result).toContain('Failed to save the image');
+  });
+});

api/app/clients/tools/util/handleOpenAIErrors.js

@@ -0,0 +1,31 @@
+const OpenAI = require('openai');
+const { logger } = require('~/config');
+
+/**
+ * Handles errors that may occur when making requests to OpenAI's API.
+ * It checks the instance of the error and prints a specific warning message
+ * to the console depending on the type of error encountered.
+ * It then calls an optional error callback function with the error object.
+ *
+ * @param {Error} err - The error object thrown by OpenAI API.
+ * @param {Function} errorCallback - A callback function that is called with the error object.
+ * @param {string} [context='stream'] - A string providing context where the error occurred, defaults to 'stream'.
+ */
+async function handleOpenAIErrors(err, errorCallback, context = 'stream') {
+  if (err instanceof OpenAI.APIError && err?.message?.includes('abort')) {
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] Aborted Message`);
+  }
+  if (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason')) {
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] Missing finish_reason`);
+  } else if (err instanceof OpenAI.APIError) {
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] API error`);
+  } else {
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] Unhandled error type`);
+  }
+
+  if (errorCallback) {
+    errorCallback(err);
+  }
+}
+
+module.exports = handleOpenAIErrors;

api/app/clients/tools/util/handleTools.js

@@ -1,30 +1,32 @@
-const { getUserPluginAuthValue } = require('../../../../server/services/PluginService');
-const { OpenAIEmbeddings } = require('langchain/embeddings/openai');
 const { ZapierToolKit } = require('langchain/agents');
-const { SerpAPI, ZapierNLAWrapper } = require('langchain/tools');
-const { ChatOpenAI } = require('langchain/chat_models/openai');
 const { Calculator } = require('langchain/tools/calculator');
 const { WebBrowser } = require('langchain/tools/webbrowser');
+const { SerpAPI, ZapierNLAWrapper } = require('langchain/tools');
+const { OpenAIEmbeddings } = require('langchain/embeddings/openai');
+const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const {
   availableTools,
-  CodeInterpreter,
-  AIPluginTool,
+  // Basic Tools
+  CodeBrew,
+  AzureAISearch,
   GoogleSearchAPI,
   WolframAlphaAPI,
-  StructuredWolfram,
-  HttpRequestTool,
   OpenAICreateImage,
   StableDiffusionAPI,
-  StructuredSD,
-  AzureCognitiveSearch,
-  StructuredACS,
+  // Structured Tools
+  DALLE3,
   E2BTools,
   CodeSherpa,
+  StructuredSD,
+  StructuredACS,
   CodeSherpaTools,
-  CodeBrew,
+  TraversaalSearch,
+  StructuredWolfram,
+  TavilySearchResults,
 } = require('../');
-const { loadSpecs } = require('./loadSpecs');
 const { loadToolSuite } = require('./loadToolSuite');
+const { loadSpecs } = require('./loadSpecs');
+const { logger } = require('~/config');
 
 const getOpenAIKey = async (options, user) => {
   let openAIApiKey = options.openAIApiKey ?? process.env.OPENAI_API_KEY;
@@ -32,6 +34,14 @@ const getOpenAIKey = async (options, user) => {
   return openAIApiKey || (await getUserPluginAuthValue(user, 'OPENAI_API_KEY'));
 };
 
+/**
+ * Validates the availability and authentication of tools for a user based on environment variables or user-specific plugin authentication values.
+ * Tools without required authentication or with valid authentication are considered valid.
+ *
+ * @param {Object} user The user object for whom to validate tool access.
+ * @param {Array<string>} tools An array of tool identifiers to validate. Defaults to an empty array.
+ * @returns {Promise<Array<string>>} A promise that resolves to an array of valid tool identifiers.
+ */
 const validateTools = async (user, tools = []) => {
   try {
     const validToolsSet = new Set(tools);
@@ -39,16 +49,34 @@ const validateTools = async (user, tools = []) => {
       validToolsSet.has(tool.pluginKey),
     );
 
+    /**
+     * Validates the credentials for a given auth field or set of alternate auth fields for a tool.
+     * If valid admin or user authentication is found, the function returns early. Otherwise, it removes the tool from the set of valid tools.
+     *
+     * @param {string} authField The authentication field or fields (separated by "||" for alternates) to validate.
+     * @param {string} toolName The identifier of the tool being validated.
+     */
     const validateCredentials = async (authField, toolName) => {
-      const adminAuth = process.env[authField];
-      if (adminAuth && adminAuth.length > 0) {
-        return;
+      const fields = authField.split('||');
+      for (const field of fields) {
+        const adminAuth = process.env[field];
+        if (adminAuth && adminAuth.length > 0) {
+          return;
+        }
+
+        let userAuth = null;
+        try {
+          userAuth = await getUserPluginAuthValue(user, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !userAuth) {
+            throw err;
+          }
+        }
+        if (userAuth && userAuth.length > 0) {
+          return;
+        }
       }
 
-      const userAuth = await getUserPluginAuthValue(user, authField);
-      if (userAuth && userAuth.length > 0) {
-        return;
-      }
       validToolsSet.delete(toolName);
     };
 
@@ -64,24 +92,59 @@ const validateTools = async (user, tools = []) => {
 
     return Array.from(validToolsSet.values());
   } catch (err) {
-    console.log('There was a problem validating tools', err);
-    throw new Error(err);
+    logger.error('[validateTools] There was a problem validating tools', err);
+    throw new Error('There was a problem validating tools');
   }
 };
 
-const loadToolWithAuth = async (user, authFields, ToolConstructor, options = {}) => {
+/**
+ * Initializes a tool with authentication values for the given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {string} userId The user ID for which the tool is being loaded.
+ * @param {Array<string>} authFields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+ * @param {typeof import('langchain/tools').Tool} ToolConstructor The constructor function for the tool to be initialized.
+ * @param {Object} options Optional parameters to be passed to the tool constructor alongside authentication values.
+ * @returns {Function} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication.
+ */
+const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => {
   return async function () {
     let authValues = {};
 
-    for (const authField of authFields) {
-      let authValue = process.env[authField];
-      if (!authValue) {
-        authValue = await getUserPluginAuthValue(user, authField);
+    /**
+     * Finds the first non-empty value for the given authentication field, supporting alternate fields.
+     * @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+     * @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found.
+     */
+    const findAuthValue = async (fields) => {
+      for (const field of fields) {
+        let value = process.env[field];
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+        try {
+          value = await getUserPluginAuthValue(userId, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !value) {
+            throw err;
+          }
+        }
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+      }
+      return null;
+    };
+
+    for (let authField of authFields) {
+      const fields = authField.split('||');
+      const result = await findAuthValue(fields);
+      if (result) {
+        authValues[result.authField] = result.authValue;
       }
-      authValues[authField] = authValue;
     }
 
-    return new ToolConstructor({ ...options, ...authValues });
+    return new ToolConstructor({ ...options, ...authValues, userId });
   };
 };
 
@@ -92,16 +155,18 @@ const loadTools = async ({
   returnMap = false,
   tools = [],
   options = {},
+  skipSpecs = false,
 }) => {
   const toolConstructors = {
+    tavily_search_results_json: TavilySearchResults,
     calculator: Calculator,
-    codeinterpreter: CodeInterpreter,
     google: GoogleSearchAPI,
     wolfram: functions ? StructuredWolfram : WolframAlphaAPI,
     'dall-e': OpenAICreateImage,
     'stable-diffusion': functions ? StructuredSD : StableDiffusionAPI,
-    'azure-cognitive-search': functions ? StructuredACS : AzureCognitiveSearch,
+    'azure-ai-search': functions ? StructuredACS : AzureAISearch,
     CodeBrew: CodeBrew,
+    traversaal_search: TraversaalSearch,
   };
 
   const openAIApiKey = await getOpenAIKey(options, user);
@@ -162,25 +227,28 @@ const loadTools = async ({
       const zapier = new ZapierNLAWrapper({ apiKey });
       return ZapierToolKit.fromZapierNLAWrapper(zapier);
     },
-    plugins: async () => {
-      return [
-        new HttpRequestTool(),
-        await AIPluginTool.fromPluginUrl(
-          'https://www.klarna.com/.well-known/ai-plugin.json',
-          new ChatOpenAI({ openAIApiKey: options.openAIApiKey, temperature: 0 }),
-        ),
-      ];
-    },
   };
 
   const requestedTools = {};
 
   if (functions) {
+    toolConstructors.dalle = DALLE3;
     toolConstructors.codesherpa = CodeSherpa;
   }
 
+  const imageGenOptions = {
+    req: options.req,
+    fileStrategy: options.fileStrategy,
+    processFileURL: options.processFileURL,
+    returnMetadata: options.returnMetadata,
+    uploadImageBuffer: options.uploadImageBuffer,
+  };
+
   const toolOptions = {
     serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' },
+    dalle: imageGenOptions,
+    'dall-e': imageGenOptions,
+    'stable-diffusion': imageGenOptions,
   };
 
   const toolAuthFields = {};
@@ -203,7 +271,7 @@ const loadTools = async ({
 
     if (toolConstructors[tool]) {
       const options = toolOptions[tool] || {};
-      const toolInstance = await loadToolWithAuth(
+      const toolInstance = loadToolWithAuth(
         user,
         toolAuthFields[tool],
         toolConstructors[tool],
@@ -219,7 +287,7 @@ const loadTools = async ({
   }
 
   let specs = null;
-  if (functions && remainingTools.length > 0) {
+  if (functions && remainingTools.length > 0 && skipSpecs !== true) {
     specs = await loadSpecs({
       llm: model,
       user,
@@ -246,6 +314,9 @@ const loadTools = async ({
   let result = [];
   for (const tool of tools) {
     const validTool = requestedTools[tool];
+    if (!validTool) {
+      continue;
+    }
     const plugin = await validTool();
 
     if (Array.isArray(plugin)) {
@@ -259,6 +330,7 @@ const loadTools = async ({
 };
 
 module.exports = {
+  loadToolWithAuth,
   validateTools,
   loadTools,
 };

api/app/clients/tools/util/handleTools.test.js

@@ -4,26 +4,33 @@ const mockUser = {
   findByIdAndDelete: jest.fn(),
 };
 
-var mockPluginService = {
+const mockPluginService = {
   updateUserPluginAuth: jest.fn(),
   deleteUserPluginAuth: jest.fn(),
   getUserPluginAuthValue: jest.fn(),
 };
 
-jest.mock('../../../../models/User', () => {
+jest.mock('~/models/User', () => {
   return function () {
     return mockUser;
   };
 });
 
-jest.mock('../../../../server/services/PluginService', () => mockPluginService);
+jest.mock('~/server/services/PluginService', () => mockPluginService);
 
-const User = require('../../../../models/User');
-const { validateTools, loadTools } = require('./');
-const PluginService = require('../../../../server/services/PluginService');
-const { BaseChatModel } = require('langchain/chat_models/openai');
 const { Calculator } = require('langchain/tools/calculator');
-const { availableTools, OpenAICreateImage, GoogleSearchAPI, StructuredSD } = require('../');
+const { BaseChatModel } = require('langchain/chat_models/openai');
+
+const User = require('~/models/User');
+const PluginService = require('~/server/services/PluginService');
+const { validateTools, loadTools, loadToolWithAuth } = require('./handleTools');
+const {
+  availableTools,
+  OpenAICreateImage,
+  GoogleSearchAPI,
+  StructuredSD,
+  WolframAlphaAPI,
+} = require('../');
 
 describe('Tool Handlers', () => {
   let fakeUser;
@@ -44,7 +51,10 @@ describe('Tool Handlers', () => {
     });
     mockPluginService.updateUserPluginAuth.mockImplementation(
       (userId, authField, _pluginKey, credential) => {
-        userAuthValues[`${userId}-${authField}`] = credential;
+        const fields = authField.split('||');
+        fields.forEach((field) => {
+          userAuthValues[`${userId}-${field}`] = credential;
+        });
       },
     );
 
@@ -53,6 +63,7 @@ describe('Tool Handlers', () => {
       username: 'fakeuser',
       email: 'fakeuser@example.com',
       emailVerified: false,
+      // file deepcode ignore NoHardcodedPasswords/test: fake value
       password: 'fakepassword123',
       avatar: '',
       provider: 'local',
@@ -133,6 +144,18 @@ describe('Tool Handlers', () => {
       loadTool2 = toolFunctions[sampleTools[1]];
       loadTool3 = toolFunctions[sampleTools[2]];
     });
+
+    let originalEnv;
+
+    beforeEach(() => {
+      originalEnv = process.env;
+      process.env = { ...originalEnv };
+    });
+
+    afterEach(() => {
+      process.env = originalEnv;
+    });
+
     it('returns the expected load functions for requested tools', async () => {
       expect(loadTool1).toBeDefined();
       expect(loadTool2).toBeDefined();
@@ -149,6 +172,86 @@ describe('Tool Handlers', () => {
       expect(authTool).toBeInstanceOf(ToolClass);
       expect(tool).toBeInstanceOf(ToolClass2);
     });
+
+    it('should initialize an authenticated tool with primary auth field', async () => {
+      process.env.DALLE2_API_KEY = 'mocked_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool with alternate auth field when primary is missing', async () => {
+      delete process.env.DALLE2_API_KEY; // Ensure the primary key is not set
+      process.env.DALLE_API_KEY = 'mocked_alternate_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'DALLE2_API_KEY',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when env vars are missing', async () => {
+      mockPluginService.updateUserPluginAuth('userId', 'DALLE_API_KEY', 'dalle', 'mocked_api_key');
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(2);
+    });
+
+    it('should initialize an authenticated tool with singular auth field', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool when env var is set', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when singular env var is missing', async () => {
+      delete process.env.WOLFRAM_APP_ID; // Ensure the environment variable is not set
+      mockPluginService.getUserPluginAuthValue.mockResolvedValue('mocked_user_auth_value');
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
     it('should throw an error for an unauthenticated tool', async () => {
       try {
         await loadTool2();

api/app/clients/tools/util/index.js

@@ -1,6 +1,8 @@
 const { validateTools, loadTools } = require('./handleTools');
+const handleOpenAIErrors = require('./handleOpenAIErrors');
 
 module.exports = {
+  handleOpenAIErrors,
   validateTools,
   loadTools,
 };

api/app/clients/tools/util/loadSpecs.js

@@ -1,7 +1,8 @@
 const fs = require('fs');
 const path = require('path');
 const { z } = require('zod');
-const { createOpenAPIPlugin } = require('../dynamic/OpenAPIPlugin');
+const { logger } = require('~/config');
+const { createOpenAPIPlugin } = require('~/app/clients/tools/dynamic/OpenAPIPlugin');
 
 // The minimum Manifest definition
 const ManifestDefinition = z.object({
@@ -26,28 +27,17 @@ const ManifestDefinition = z.object({
   legal_info_url: z.string().optional(),
 });
 
-function validateJson(json, verbose = true) {
+function validateJson(json) {
   try {
     return ManifestDefinition.parse(json);
   } catch (error) {
-    if (verbose) {
-      console.debug('validateJson error', error);
-    }
+    logger.debug('[validateJson] manifest parsing error', error);
     return false;
   }
 }
 
 // omit the LLM to return the well known jsons as objects
-async function loadSpecs({
-  llm,
-  user,
-  message,
-  tools = [],
-  map = false,
-  memory,
-  signal,
-  verbose = false,
-}) {
+async function loadSpecs({ llm, user, message, tools = [], map = false, memory, signal }) {
   const directoryPath = path.join(__dirname, '..', '.well-known');
   let files = [];
 
@@ -60,7 +50,7 @@ async function loadSpecs({
       await fs.promises.access(filePath, fs.constants.F_OK);
       files.push(tools[i] + '.json');
     } catch (err) {
-      console.error(`File ${tools[i] + '.json'} does not exist`);
+      logger.error(`[loadSpecs] File ${tools[i] + '.json'} does not exist`, err);
     }
   }
 
@@ -73,9 +63,7 @@ async function loadSpecs({
   const validJsons = [];
   const constructorMap = {};
 
-  if (verbose) {
-    console.debug('files', files);
-  }
+  logger.debug('[validateJson] files', files);
 
   for (const file of files) {
     if (path.extname(file) === '.json') {
@@ -84,7 +72,7 @@ async function loadSpecs({
       const json = JSON.parse(fileContent);
 
       if (!validateJson(json)) {
-        verbose && console.debug('Invalid json', json);
+        logger.debug('[validateJson] Invalid json', json);
         continue;
       }
 
@@ -97,13 +85,12 @@ async function loadSpecs({
             memory,
             signal,
             user,
-            verbose,
           });
         continue;
       }
 
       if (llm) {
-        validJsons.push(createOpenAPIPlugin({ data: json, llm, verbose }));
+        validJsons.push(createOpenAPIPlugin({ data: json, llm }));
         continue;
       }
 
@@ -117,10 +104,8 @@ async function loadSpecs({
 
   const plugins = (await Promise.all(validJsons)).filter((plugin) => plugin);
 
-  // if (verbose) {
-  //   console.debug('plugins', plugins);
-  //   console.debug(plugins[0].name);
-  // }
+  //   logger.debug('[validateJson] plugins', plugins);
+  //   logger.debug(plugins[0].name);
 
   return plugins;
 }

api/app/clients/tools/util/loadToolSuite.js

@@ -1,17 +1,49 @@
-const { getUserPluginAuthValue } = require('../../../../server/services/PluginService');
+const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const { availableTools } = require('../');
+const { logger } = require('~/config');
 
-const loadToolSuite = async ({ pluginKey, tools, user, options }) => {
+/**
+ * Loads a suite of tools with authentication values for a given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {Object} params Parameters for loading the tool suite.
+ * @param {string} params.pluginKey Key identifying the plugin whose tools are to be loaded.
+ * @param {Array<Function>} params.tools Array of tool constructor functions.
+ * @param {Object} params.user User object for whom the tools are being loaded.
+ * @param {Object} [params.options={}] Optional parameters to be passed to each tool constructor.
+ * @returns {Promise<Array>} A promise that resolves to an array of instantiated tools.
+ */
+const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
   const authConfig = availableTools.find((tool) => tool.pluginKey === pluginKey).authConfig;
   const suite = [];
   const authValues = {};
 
-  for (const auth of authConfig) {
-    let authValue = process.env[auth.authField];
-    if (!authValue) {
-      authValue = await getUserPluginAuthValue(user, auth.authField);
+  const findAuthValue = async (authField) => {
+    const fields = authField.split('||');
+    for (const field of fields) {
+      let value = process.env[field];
+      if (value) {
+        return value;
+      }
+      try {
+        value = await getUserPluginAuthValue(user, field);
+        if (value) {
+          return value;
+        }
+      } catch (err) {
+        logger.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
+      }
+    }
+    return null;
+  };
+
+  for (const auth of authConfig) {
+    const authValue = await findAuthValue(auth.authField);
+    if (authValue !== null) {
+      authValues[auth.authField] = authValue;
+    } else {
+      logger.warn(`[loadToolSuite] No auth value found for ${auth.authField}`);
     }
-    authValues[auth.authField] = authValue;
   }
 
   for (const tool of tools) {

api/app/titleConvoBing.js

@@ -1,5 +1,6 @@
-const { isEnabled } = require('../server/utils');
 const throttle = require('lodash/throttle');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
 
 const titleConvo = async ({ text, response }) => {
   let title = 'New Chat';
@@ -30,11 +31,10 @@ const titleConvo = async ({ text, response }) => {
     const res = await titleGenerator.sendMessage(titlePrompt, options);
     title = res.response.replace(/Title: /, '').replace(/[".]/g, '');
   } catch (e) {
-    console.error(e);
-    console.log('There was an issue generating title, see error above');
+    logger.error('There was an issue generating title with BingAI', e);
   }
 
-  console.log('CONVERSATION TITLE', title);
+  logger.debug('[/ask/bingAI] CONVERSATION TITLE: ' + title);
   return title;
 };
 

api/cache/banViolation.js

@@ -1,6 +1,9 @@
-const Session = require('../models/Session');
+const { ViolationTypes } = require('librechat-data-provider');
+const { isEnabled, math, removePorts } = require('~/server/utils');
 const getLogStores = require('./getLogStores');
-const { isEnabled, math, removePorts } = require('../server/utils');
+const Session = require('~/models/Session');
+const { logger } = require('~/config');
+
 const { BAN_VIOLATIONS, BAN_INTERVAL } = process.env ?? {};
 const interval = math(BAN_INTERVAL, 20);
 
@@ -46,18 +49,25 @@ const banViolation = async (req, res, errorMessage) => {
   await Session.deleteAllUserSessions(user_id);
   res.clearCookie('refreshToken');
 
-  const banLogs = getLogStores('ban');
-  const duration = banLogs.opts.ttl;
+  const banLogs = getLogStores(ViolationTypes.BAN);
+  const duration = errorMessage.duration || banLogs.opts.ttl;
 
   if (duration <= 0) {
     return;
   }
 
   req.ip = removePorts(req);
-  console.log(`[BAN] Banning user ${user_id} @ ${req.ip} for ${duration / 1000 / 60} minutes`);
+  logger.info(
+    `[BAN] Banning user ${user_id} ${req.ip ? `@ ${req.ip} ` : ''}for ${
+      duration / 1000 / 60
+    } minutes`,
+  );
+
   const expiresAt = Date.now() + duration;
   await banLogs.set(user_id, { type, violation_count, duration, expiresAt });
-  await banLogs.set(req.ip, { type, user_id, violation_count, duration, expiresAt });
+  if (req.ip) {
+    await banLogs.set(req.ip, { type, user_id, violation_count, duration, expiresAt });
+  }
 
   errorMessage.ban = true;
   errorMessage.ban_duration = duration;