✨ v0.7.0 (#2273)

* chore: remove version comment from pre-commit shell script

* chore: Dockerfile update

.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM node:18-bullseye
+
+RUN useradd -m -s /bin/bash vscode
+RUN mkdir -p /workspaces && chown -R vscode:vscode /workspaces
+WORKDIR /workspaces

.devcontainer/devcontainer.json

@@ -1,58 +1,18 @@
-// {
-// 	"name": "LibreChat_dev",
-// 	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
-// 	"dockerComposeFile": "docker-compose.yml",
-// 	// The 'service' property is the name of the service for the container that VS Code should
-// 	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
-// 	"service": "librechat",
-// 	// The 'workspaceFolder' property is the path VS Code should open by default when
-// 	// connected. Corresponds to a volume mount in .devcontainer/docker-compose.yml
-// 	"workspaceFolder": "/workspace"
-//     //,
-// 	// // Set *default* container specific settings.json values on container create.
-// 	// "settings": {},
-// 	// // Add the IDs of extensions you want installed when the container is created.
-// 	// "extensions": [],
-// 	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
-// 	// "shutdownAction": "none",
-// 	// Uncomment the next line to use 'postCreateCommand' to run commands after the container is created.
-// 	// "postCreateCommand": "uname -a",
-// 	// Comment out to connect as root instead. To add a non-root user, see: https://aka.ms/vscode-remote/containers/non-root.
-// 	// "remoteUser": "vscode"
-// }
 {
-    // "name": "LibreChat_dev",
-    "dockerComposeFile": "docker-compose.yml",
-    "service": "app",
-    // "image": "node:19-alpine",
-    // "workspaceFolder": "/workspaces",
-    "workspaceFolder": "/workspace",
-    // Set *default* container specific settings.json values on container create.
-    // "overrideCommand": true,
-    "customizations": {
-        "vscode": {
-            "extensions": [],
-            "settings": {
-                "terminal.integrated.profiles.linux": {
-                    "bash": null
-                }
-            }
+  "dockerComposeFile": "docker-compose.yml",
+  "service": "app",
+  "workspaceFolder": "/workspaces",
+  "customizations": {
+    "vscode": {
+      "extensions": [],
+      "settings": {
+        "terminal.integrated.profiles.linux": {
+          "bash": null
         }
-    },
-    "postCreateCommand": "",
-    // "workspaceMount": "src=${localWorkspaceFolder},dst=/code,type=bind,consistency=cached"
-
-    // "runArgs": [
-    //     "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined",
-    //     "-v", "/tmp/.X11-unix:/tmp/.X11-unix",
-    //     "-v", "${env:XAUTHORITY}:/root/.Xauthority:rw",
-    //     "-v", "/home/${env:USER}/.cdh:/root/.cdh",
-    //     "-e", "DISPLAY=${env:DISPLAY}",
-    //     "--name=tgw_assistant_backend_dev",
-    //     "--network=host"
-    // ],
-    // "settings": {
-    //     "terminal.integrated.shell.linux": "/bin/bash"
-    // },
-    "features": {"ghcr.io/devcontainers/features/git:1": {}}
+      }
+    }
+  },
+  "postCreateCommand": "",
+  "features": { "ghcr.io/devcontainers/features/git:1": {} },
+  "remoteUser": "vscode"
 }

.devcontainer/docker-compose.yml

@@ -1,17 +1,11 @@
-version: '3.4'
+version: "3.8"
 
 services:
   app:
-    # container_name: LibreChat_dev
-    image: node:19-bullseye
-    # Using a Dockerfile is optional, but included for completeness.
-    # build: 
-    #   context: .
-    #   dockerfile: Dockerfile
-    #   # [Optional] You can use build args to set options. e.g. 'VARIANT' below affects the image in the Dockerfile
-    #   args: 
-    #     VARIANT: buster
-    # network_mode: "host"
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
+    # restart: always
     links:
       - mongodb
       - meilisearch
@@ -21,17 +15,16 @@ services:
     - "host.docker.internal:host-gateway"
 
     volumes:
-    #   # This is where VS Code should expect to find your project's source code and the value of "workspaceFolder" in .devcontainer/devcontainer.json
-      - ..:/workspace:cached
-    #   # - /app/client/node_modules
-    #   # - ./api:/app/api
-    #   # - ./.env:/app/.env
-    #   # - ./.env.development:/app/.env.development
-    #   # - ./.env.production:/app/.env.production
-    #   # - /app/api/node_modules
-
-    #   # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details.
-    #   # - /var/run/docker.sock:/var/run/docker.sock 
+      # This is where VS Code should expect to find your project's source code and the value of "workspaceFolder" in .devcontainer/devcontainer.json
+      - ..:/workspaces:cached
+      # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details.
+      # - /var/run/docker.sock:/var/run/docker.sock 
+    environment:
+      - HOST=0.0.0.0
+      - MONGO_URI=mongodb://mongodb:27017/LibreChat
+      # - CHATGPT_REVERSE_PROXY=http://host.docker.internal:8080/api/conversation # if you are hosting your own chatgpt reverse proxy with docker
+      # - OPENAI_REVERSE_PROXY=http://host.docker.internal:8070/v1/chat/completions # if you are hosting your own chatgpt reverse proxy with docker
+      - MEILI_HOST=http://meilisearch:7700
 
     # Runs app on the same network as the service container, allows "forwardPorts" in devcontainer.json function.
     # network_mode: service:another-service
@@ -39,45 +32,34 @@ services:
     # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. 
     # (Adding the "ports" property to this file will not forward from a Codespace.)
 
-    # Uncomment the next line to use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
-    # user: vscode
-
-    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
-    # cap_add:
-    #   - SYS_PTRACE
-    # security_opt:
-    #   - seccomp:unconfined
+    # Use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
+    user: vscode
 
     # Overrides default command so things don't shut down after the process ends.
     command: /bin/sh -c "while sleep 1000; do :; done"  
 
   mongodb:
     container_name: chat-mongodb
-    # network_mode: "host"
     expose:
       - 27017
     # ports:
     #   - 27018:27017
     image: mongo
-  #   restart: always
+    # restart: always
     volumes:
       - ./data-node:/data/db
     command: mongod --noauth
   meilisearch:
     container_name: chat-meilisearch
-    image: getmeili/meilisearch:v1.0
-    # network_mode: "host"
+    image: getmeili/meilisearch:v1.5
+    # restart: always
     expose:
       - 7700
-    # ports:
-    #   - 7700:7700
-    # env_file:
-    #   - .env
+    # Uncomment this to access meilisearch from outside docker
+    # ports: 
+    #   - 7700:7700 # if exposing these ports, make sure your master key is not the default value
     environment:
-      - SEARCH=false
-      - MEILI_HOST=http://0.0.0.0:7700
-      - MEILI_HTTP_ADDR=0.0.0.0:7700
+      - MEILI_NO_ANALYTICS=true
       - MEILI_MASTER_KEY=5c71cf56d672d009e36070b5bc5e47b743535ae55c818ae3b735bb6ebfb4ba63
     volumes:
-      - ./meili_data:/meili_data
-  
+      - ./meili_data_v1.5:/meili_data

.dockerignore

@@ -1,5 +1,17 @@
+**/.circleci
+**/.editorconfig
+**/.dockerignore
+**/.git
+**/.DS_Store
+**/.vscode
 **/node_modules
-client/dist/images
+
+# Specific patterns to ignore
 data-node
-.env
-**/.env
+meili_data*
+librechat*
+Dockerfile*
+docs
+
+# Ignore all hidden files
+.*

.env.example

@@ -1,48 +1,248 @@
-##########################
-# Server configuration: 
-##########################
+#=====================================================================#
+#                       LibreChat Configuration                       #
+#=====================================================================#
+# Please refer to the reference documentation for assistance          #
+# with configuring your LibreChat environment. The guide is           #
+# available both online and within your local LibreChat               #
+# directory:                                                          #
+# Online: https://docs.librechat.ai/install/configuration/dotenv.html #
+# Locally: ./docs/install/configuration/dotenv.md                     #
+#=====================================================================#
 
-APP_TITLE=LibreChat
+#==================================================#
+#               Server Configuration               #
+#==================================================#
 
-# Uncomment to add a custom footer. 
-# Uncomment and make empty "" to remove the footer.
-# CUSTOM_FOOTER="My custom footer"
-
-# The server will listen to localhost:3080 by default. You can change the target IP as you want. 
-# If you want to make this server available externally, for example to share the server with others 
-# or expose this from a Docker container, set host to 0.0.0.0 or your external IP interface. 
-# Tips: Setting host to 0.0.0.0 means listening on all interfaces. It's not a real IP.  
-# Use localhost:port rather than 0.0.0.0:port to access the server. 
-# Set Node env to development if running in dev mode.
 HOST=localhost
 PORT=3080
 
-# Note: the following enables user balances, which you can add manually
-# or you will need to build out a balance accruing system for users.
-# For more info, see https://docs.librechat.ai/features/token_usage.html
+MONGO_URI=mongodb://127.0.0.1:27017/LibreChat
 
-# To manually add balances, run the following command:
-# `npm run add-balance`
+DOMAIN_CLIENT=http://localhost:3080
+DOMAIN_SERVER=http://localhost:3080
 
-# You can also specify the email and token credit amount to add, e.g.:
-# `npm run add-balance example@example.com 1000`
+NO_INDEX=true
 
-# This works well to track your own usage for personal use; 1000 credits = $0.001 (1 mill USD)
+#===============#
+# JSON Logging  #
+#===============#
 
-# Set to true to enable token credit balances for the OpenAI/Plugins endpoints
-CHECK_BALANCE=false
+# Use when process console logs in cloud deployment like GCP/AWS
+CONSOLE_JSON=false
 
-# Automated Moderation System
-# The Automated Moderation System uses a scoring mechanism to track user violations. As users commit actions
-# like excessive logins, registrations, or messaging, they accumulate violation scores. Upon reaching
-# a set threshold, the user and their IP are temporarily banned. This system ensures platform security
-# by monitoring and penalizing rapid or suspicious activities.
+#===============#
+# Debug Logging #
+#===============#
 
-BAN_VIOLATIONS=true # Whether or not to enable banning users for violations (they will still be logged)
-BAN_DURATION=1000 * 60 * 60 * 2 # how long the user and associated IP are banned for
-BAN_INTERVAL=20 # a user will be banned everytime their score reaches/crosses over the interval threshold
+DEBUG_LOGGING=true
+DEBUG_CONSOLE=false
 
-# The score for each violation
+#=============#
+# Permissions #
+#=============#
+
+# UID=1000
+# GID=1000
+
+#===============#
+# Configuration #
+#===============#
+# Use an absolute path, a relative path, or a URL
+
+# CONFIG_PATH="/alternative/path/to/librechat.yaml"
+
+#===================================================#
+#                     Endpoints                     #
+#===================================================#
+
+# ENDPOINTS=openAI,assistants,azureOpenAI,bingAI,google,gptPlugins,anthropic
+
+PROXY=
+
+#===================================#
+# Known Endpoints - librechat.yaml  #
+#===================================#
+# https://docs.librechat.ai/install/configuration/ai_endpoints.html
+
+# GROQ_API_KEY=
+# SHUTTLEAI_KEY=
+# OPENROUTER_KEY=
+# MISTRAL_API_KEY=
+# ANYSCALE_API_KEY=
+# FIREWORKS_API_KEY=
+# PERPLEXITY_API_KEY=
+# TOGETHERAI_API_KEY=
+
+#============#
+# Anthropic  #
+#============#
+
+ANTHROPIC_API_KEY=user_provided
+# ANTHROPIC_MODELS=claude-3-opus-20240229,claude-3-sonnet-20240229,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
+# ANTHROPIC_REVERSE_PROXY=
+
+#============#
+# Azure      #
+#============#
+
+
+# Note: these variables are DEPRECATED
+# Use the `librechat.yaml` configuration for `azureOpenAI` instead
+# You may also continue to use them if you opt out of using the `librechat.yaml` configuration
+
+# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated
+# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 # Deprecated
+# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE # Deprecated
+# AZURE_API_KEY= # Deprecated
+# AZURE_OPENAI_API_INSTANCE_NAME= # Deprecated
+# AZURE_OPENAI_API_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_VERSION= # Deprecated
+# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated
+# PLUGINS_USE_AZURE="true" # Deprecated
+
+#============#
+# BingAI     #
+#============#
+
+BINGAI_TOKEN=user_provided
+# BINGAI_HOST=https://cn.bing.com
+
+#============#
+# Google     #
+#============#
+
+GOOGLE_KEY=user_provided
+# GOOGLE_MODELS=gemini-pro,gemini-pro-vision,chat-bison,chat-bison-32k,codechat-bison,codechat-bison-32k,text-bison,text-bison-32k,text-unicorn,code-gecko,code-bison,code-bison-32k
+# GOOGLE_REVERSE_PROXY=
+
+#============#
+# OpenAI     #
+#============#
+
+OPENAI_API_KEY=user_provided
+# OPENAI_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k
+
+DEBUG_OPENAI=false
+
+# TITLE_CONVO=false
+# OPENAI_TITLE_MODEL=gpt-3.5-turbo
+
+# OPENAI_SUMMARIZE=true
+# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo
+
+# OPENAI_FORCE_PROMPT=true
+
+# OPENAI_REVERSE_PROXY=
+
+# OPENAI_ORGANIZATION=
+
+#====================#
+#   Assistants API   #
+#====================#
+
+ASSISTANTS_API_KEY=user_provided
+# ASSISTANTS_BASE_URL=
+# ASSISTANTS_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-16k,gpt-3.5-turbo,gpt-4,gpt-4-0314,gpt-4-32k-0314,gpt-4-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-1106,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview
+
+#============#
+# OpenRouter #
+#============#
+
+# OPENROUTER_API_KEY=
+
+#============#
+# Plugins    #
+#============#
+
+# PLUGIN_MODELS=gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613
+
+DEBUG_PLUGINS=true
+
+CREDS_KEY=f34be427ebb29de8d88c107a71546019685ed8b241d8f2ed00c3df97ad2566f0
+CREDS_IV=e2341419ec3dd3d19b13a1a87fafcbfb
+
+# Azure AI Search
+#-----------------
+AZURE_AI_SEARCH_SERVICE_ENDPOINT=
+AZURE_AI_SEARCH_INDEX_NAME=
+AZURE_AI_SEARCH_API_KEY=
+
+AZURE_AI_SEARCH_API_VERSION=
+AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE=
+AZURE_AI_SEARCH_SEARCH_OPTION_TOP=
+AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
+
+# DALL·E
+#----------------
+# DALLE_API_KEY=
+# DALLE3_API_KEY=
+# DALLE2_API_KEY=
+# DALLE3_SYSTEM_PROMPT=
+# DALLE2_SYSTEM_PROMPT=
+# DALLE_REVERSE_PROXY=
+# DALLE3_BASEURL=
+# DALLE2_BASEURL=
+
+# DALL·E (via Azure OpenAI)
+# Note: requires some of the variables above to be set
+#----------------
+# DALLE3_AZURE_API_VERSION=
+# DALLE2_AZURE_API_VERSION=
+
+# Google
+#-----------------
+GOOGLE_API_KEY=
+GOOGLE_CSE_ID=
+
+# SerpAPI
+#-----------------
+SERPAPI_API_KEY=
+
+# Stable Diffusion
+#-----------------
+SD_WEBUI_URL=http://host.docker.internal:7860
+
+# Tavily
+#-----------------
+TAVILY_API_KEY=
+
+# Traversaal
+#-----------------
+TRAVERSAAL_API_KEY=
+
+# WolframAlpha
+#-----------------
+WOLFRAM_APP_ID=
+
+# Zapier
+#-----------------
+ZAPIER_NLA_API_KEY=
+
+#==================================================#
+#                      Search                      #
+#==================================================#
+
+SEARCH=true
+MEILI_NO_ANALYTICS=true
+MEILI_HOST=http://0.0.0.0:7700
+MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
+
+#===================================================#
+#                    User System                    #
+#===================================================#
+
+#========================#
+# Moderation             #
+#========================#
+
+OPENAI_MODERATION=false
+OPENAI_MODERATION_API_KEY=
+# OPENAI_MODERATION_REVERSE_PROXY=
+
+BAN_VIOLATIONS=true
+BAN_DURATION=1000 * 60 * 60 * 2
+BAN_INTERVAL=20
 
 LOGIN_VIOLATION_SCORE=1
 REGISTRATION_VIOLATION_SCORE=1
@@ -50,398 +250,121 @@ CONCURRENT_VIOLATION_SCORE=1
 MESSAGE_VIOLATION_SCORE=1
 NON_BROWSER_VIOLATION_SCORE=20
 
-# Login and registration rate limiting.
+LOGIN_MAX=7
+LOGIN_WINDOW=5
+REGISTER_MAX=5
+REGISTER_WINDOW=60
 
-LOGIN_MAX=7 # The max amount of logins allowed per IP per LOGIN_WINDOW
-LOGIN_WINDOW=5 # in minutes, determines the window of time for LOGIN_MAX logins
-REGISTER_MAX=5 # The max amount of registrations allowed per IP per REGISTER_WINDOW
-REGISTER_WINDOW=60 # in minutes, determines the window of time for REGISTER_MAX registrations
+LIMIT_CONCURRENT_MESSAGES=true
+CONCURRENT_MESSAGE_MAX=2
 
-# Message rate limiting (per user & IP)
+LIMIT_MESSAGE_IP=true
+MESSAGE_IP_MAX=40
+MESSAGE_IP_WINDOW=1
 
-LIMIT_CONCURRENT_MESSAGES=true # Whether to limit the amount of messages a user can send per request
-CONCURRENT_MESSAGE_MAX=2 # The max amount of messages a user can send per request
+LIMIT_MESSAGE_USER=false
+MESSAGE_USER_MAX=40
+MESSAGE_USER_WINDOW=1
 
-LIMIT_MESSAGE_IP=true # Whether to limit the amount of messages an IP can send per MESSAGE_IP_WINDOW
-MESSAGE_IP_MAX=40 # The max amount of messages an IP can send per MESSAGE_IP_WINDOW
-MESSAGE_IP_WINDOW=1 # in minutes, determines the window of time for MESSAGE_IP_MAX messages
+ILLEGAL_MODEL_REQ_SCORE=5
 
-# Note: You can utilize both limiters, but default is to limit by IP only.
-LIMIT_MESSAGE_USER=false # Whether to limit the amount of messages an IP can send per MESSAGE_USER_WINDOW
-MESSAGE_USER_MAX=40 # The max amount of messages an IP can send per MESSAGE_USER_WINDOW
-MESSAGE_USER_WINDOW=1 # in minutes, determines the window of time for MESSAGE_USER_MAX messages
+#========================#
+# Balance                #
+#========================#
 
-# If you have permission problems, set here the UID and GID of the user running
-# the docker compose command. The applications in the container will run with these uid/gid.
-# UID=1000
-# GID=1000
+CHECK_BALANCE=false
 
-# Change this to proxy any API request. 
-# It's useful if your machine has difficulty calling the original API server. 
-# PROXY=
+#========================#
+# Registration and Login #
+#========================#
 
-# Change this to your MongoDB URI if different. I recommend appending LibreChat. 
-MONGO_URI=mongodb://127.0.0.1:27018/LibreChat
-
-##########################
-# OpenAI Endpoint: 
-##########################
-
-# Access key from OpenAI platform. 
-# Leave it blank to disable this feature.
-# Set to "user_provided" to allow the user to provide their API key from the UI.
-OPENAI_API_KEY=user_provided
-
-DEBUG_OPENAI=false # Set to true to enable debug mode for the OpenAI endpoint
-
-# Identify the available models, separated by commas *without spaces*. 
-# The first will be default. 
-# Leave it blank to use internal settings. 
-# OPENAI_MODELS=gpt-3.5-turbo-1106,gpt-4-1106-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,text-davinci-003,gpt-4,gpt-4-0314,gpt-4-0613
-
-# Titling is enabled by default when initiating a conversation.
-# Uncomment the following variable to disable this feature.
-# TITLE_CONVO=false
-
-# (Optional) The default model used for titling by is gpt-3.5-turbo-0613
-# You can change it by uncommenting the following and setting the desired model
-# Must be compatible with the OpenAI Endpoint.
-# OPENAI_TITLE_MODEL=gpt-3.5-turbo
-
-# (Optional/Experimental) Enable message summarization by uncommenting the following:
-# Note: this may affect response time when a summary is being generated.
-# OPENAI_SUMMARIZE=true
-
-# Not yet implemented: this will be a conversation option enabled by default to save users on tokens
-# We are using the ConversationSummaryBufferMemory method to summarize messages.
-# To learn more about this, see this article:
-# https://www.pinecone.io/learn/series/langchain/langchain-conversational-memory/
-
-# (Optional) The default model used for summarizing is gpt-3.5-turbo
-# You can change it by uncommenting the following and setting the desired model
-# Must be compatible with the OpenAI Endpoint.
-# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo
-
-# Reverse proxy settings for OpenAI: 
-# https://github.com/waylaidwanderer/node-chatgpt-api#using-a-reverse-proxy
-# The URL must match the "url/v1," pattern, the "openai" suffix is also allowed.
-# Examples:
-#   - https://open.ai/v1
-#   - https://open.ai/v1/ACCOUNT/GATEWAY/openai
-#   - https://open.ai/v1/hi/openai
-
-# OPENAI_REVERSE_PROXY=
-
-# (Advanced) Sometimes when using Local LLM APIs, you may need to force the API
-# to be called with a `prompt` payload instead of a `messages` payload; to mimic the
-# a `/v1/completions` request instead of `/v1/chat/completions`
-# This may be the case for LocalAI with some models. To do so, uncomment the following:
-# OPENAI_FORCE_PROMPT=true
-
-# (Advanced) For customization of the DALL-E-3 System prompt, 
-# uncomment the following, and provide your own prompt:
-# See official prompt for reference:
-# https://github.com/spdustin/ChatGPT-AutoExpert/blob/main/_system-prompts/dall-e.md
-# DALLE3_SYSTEM_PROMPT="Your System Prompt here"
-
-# (Advanced) DALL-E Proxy settings
-# This is separate from its OpenAI counterpart for customization purposes
-
-# Reverse proxy settings, changes the baseURL for the DALL-E-3 API Calls
-# The URL must match the "url/v1," pattern, the "openai" suffix is also allowed.
-# Examples:
-#   - https://open.ai/v1
-#   - https://open.ai/v1/ACCOUNT/GATEWAY/openai
-#   - https://open.ai/v1/hi/openai
-
-# DALLE_REVERSE_PROXY=
-
-# Note: if you have PROXY set, it will be used for DALLE calls also, which is universal for the app
-
-##########################
-# OpenRouter (overrides OpenAI and Plugins Endpoints): 
-##########################
-
-# OpenRouter is a legitimate proxy service to a multitude of LLMs, both closed and open source, including:
-# OpenAI models, Anthropic models, Meta's Llama models, pygmalionai/mythalion-13b
-# and many more open source models. Newer integrations are usually discounted, too!
-
-# Note: this overrides the OpenAI and Plugins Endpoints.
-# See ./docs/install/free_ai_apis.md for more info.
-
-# OPENROUTER_API_KEY=
-
-##########################
-# AZURE Endpoint:   
-##########################
-
-# To use Azure with this project, set the following variables. These will be used to build the API URL.
-# Chat completion: 
-# `https://{AZURE_OPENAI_API_INSTANCE_NAME}.openai.azure.com/openai/deployments/{AZURE_OPENAI_API_DEPLOYMENT_NAME}/chat/completions?api-version={AZURE_OPENAI_API_VERSION}`;
-# You should also consider changing the `OPENAI_MODELS` variable above to the models available in your instance/deployment.
-# Note: I've noticed that the Azure API is much faster than the OpenAI API, so the streaming looks almost instantaneous.
-# Note "AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME" and "AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME" are optional but might be used in the future
-
-# AZURE_API_KEY=
-# AZURE_OPENAI_API_INSTANCE_NAME=
-# AZURE_OPENAI_API_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_VERSION=
-# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=
-
-# NOTE: As of 2023-11-10, the Azure API only allows one model per deployment,
-# It's recommended to name your deployments after the model name, e.g. "gpt-35-turbo,"
-# which allows for fast deployment switching and AZURE_USE_MODEL_AS_DEPLOYMENT_NAME enabled.
-# However, you can use non-model deployment names and setting the AZURE_OPENAI_DEFAULT_MODEL to ensure it works as expected.
-
-# Identify the available models, separated by commas *without spaces*.
-# The first will be default. Leave it blank or as is to use internal settings. 
-# NOTE: as deployment names can't have periods, they will be removed when the endpoint is generated.
-AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4
-
-# (Advanced) this enables the use of the model name as the deployment name, e.g. "gpt-3.5-turbo" as the deployment name
-AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
-
-# (Advanced) this overrides the model setting for Azure, in case you want to use your custom deployment names
-# as the values for AZURE_OPENAI_MODELS
-# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo
-
-# To use Azure with the Plugins endpoint, you need the variables above, and uncomment the following variable:
-# NOTE: This may not work as expected and Azure OpenAI may not support OpenAI Functions yet
-# Omit/leave it commented to use the default OpenAI API
-
-# PLUGINS_USE_AZURE="true"
-
-##########################
-# ChatGPT Endpoint: 
-##########################
-
-# ChatGPT Browser Client (free but use at your own risk) 
-# Access token from https://chat.openai.com/api/auth/session 
-# Exposes your access token to `CHATGPT_REVERSE_PROXY`
-# Set to "user_provided" to allow the user to provide its token from the UI.
-# Leave it blank to disable this endpoint 
-CHATGPT_TOKEN=user_provided
-
-# Identify the available models, separated by commas. The first will be default. 
-# Leave it blank to use internal settings. 
-CHATGPT_MODELS=text-davinci-002-render-sha,gpt-4
-# NOTE: you can add gpt-4-plugins, gpt-4-code-interpreter, and gpt-4-browsing to the list above and use the models for these features;
-# however, the view/display portion of these features are not supported, but you can use the underlying models, which have higher token context
-# Also: text-davinci-002-render-paid is deprecated as of May 2023
-
-# Reverse proxy setting for OpenAI
-# https://github.com/waylaidwanderer/node-chatgpt-api#using-a-reverse-proxy
-# By default it will use the node-chatgpt-api recommended proxy, (it's a third party server)
-# CHATGPT_REVERSE_PROXY=<YOUR REVERSE PROXY>
-
-##########################
-# BingAI Endpoint:   
-##########################
-
-# Also used for Sydney and jailbreak
-# To get your Access token for Bing, login to https://www.bing.com
-# Use dev tools or an extension while logged into the site to copy the content of the _U cookie. 
-# If this fails, follow these instructions https://github.com/danny-avila/LibreChat/issues/370#issuecomment-1560382302 to provide the full cookie strings 
-# or check out our discord https://discord.com/channels/1086345563026489514/1143941308684177429
-# Set to "user_provided" to allow the user to provide its token from the UI.
-# Leave it blank to disable this endpoint. 
-BINGAI_TOKEN=user_provided
-
-# BingAI Host: 
-# Necessary for some people in different countries, e.g. China (https://cn.bing.com) 
-# Leave it blank to use default server. 
-# BINGAI_HOST=https://cn.bing.com 
-
-#############################
-# Plugins:
-#############################
-
-# Identify the available models, separated by commas *without spaces*. 
-# The first will be default. 
-# Leave it blank to use internal settings. 
-# PLUGIN_MODELS=gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,gpt-4,gpt-4-0314,gpt-4-0613
-
-DEBUG_PLUGINS=true # Set to false or comment out to disable debug mode for plugins
-
-# For securely storing credentials, you need a fixed key and IV. You can set them here for prod and dev environments
-# If you don't set them, the app will crash on startup.
-# You need a 32-byte key (64 characters in hex) and 16-byte IV (32 characters in hex)
-# Use this replit to generate some quickly: https://replit.com/@daavila/crypto#index.js
-# Here are some examples (THESE ARE NOT SECURE!)
-CREDS_KEY=f34be427ebb29de8d88c107a71546019685ed8b241d8f2ed00c3df97ad2566f0
-CREDS_IV=e2341419ec3dd3d19b13a1a87fafcbfb
-
-# AI-Assisted Google Search  
-# This bot supports searching google for answers to your questions with assistance from GPT! 
-# See detailed instructions here: https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md
-GOOGLE_API_KEY=
-GOOGLE_CSE_ID=
-
-# StableDiffusion WebUI
-# This bot supports StableDiffusion WebUI, using it's API to generated requested images.
-# See detailed instructions here: https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/stable_diffusion.md
-# Use "http://127.0.0.1:7860" with local install and "http://host.docker.internal:7860" for docker
-SD_WEBUI_URL=http://host.docker.internal:7860
-
-# Azure Cognitive Search
-# This plugin supports searching Azure Cognitive Search for answers to your questions.
-# See detailed instructions here: https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/azure_cognitive_search.md
-AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT=
-AZURE_COGNITIVE_SEARCH_INDEX_NAME=
-AZURE_COGNITIVE_SEARCH_API_KEY=
-
-AZURE_COGNITIVE_SEARCH_API_VERSION=
-AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE=
-AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP=
-AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT=
-
-##########################
-# PaLM (Google) Endpoint: 
-##########################
-
-# Follow the instruction here to setup:
-# https://github.com/danny-avila/LibreChat/blob/main/docs/install/apis_and_tokens.md
-
-PALM_KEY=user_provided
-
-# In case you need a reverse proxy for this endpoint:
-# GOOGLE_REVERSE_PROXY= 
-
-##########################
-# Anthropic Endpoint: 
-##########################
-# Access key from https://console.anthropic.com/
-# Leave it blank to disable this feature.
-# Set to "user_provided" to allow the user to provide their API key from the UI.
-# Note that access to claude-1 may potentially become unavailable with the release of claude-2.
-ANTHROPIC_API_KEY=user_provided
-ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2
-
-##########################
-# Proxy: To be Used by all endpoints
-##########################
-
-PROXY=
-
-##########################
-# Search: 
-##########################
-
-# ENABLING SEARCH MESSAGES/CONVOS 
-# Requires the installation of the free self-hosted Meilisearch or a paid Remote Plan (Remote not tested) 
-# The easiest setup for this is through docker-compose, which takes care of it for you. 
-SEARCH=true
-
-# HIGHLY RECOMMENDED: Disable anonymized telemetry analytics for MeiliSearch for absolute privacy.
-MEILI_NO_ANALYTICS=true
-
-# REQUIRED FOR SEARCH: MeiliSearch Host, mainly for the API server to connect to the search server. 
-# Replace '0.0.0.0' with 'meilisearch' if serving MeiliSearch with docker-compose. 
-MEILI_HOST=http://0.0.0.0:7700
-
-# REQUIRED FOR SEARCH: MeiliSearch HTTP Address, mainly for docker-compose to expose the search server. 
-# Replace '0.0.0.0' with 'meilisearch' if serving MeiliSearch with docker-compose. 
-MEILI_HTTP_ADDR=0.0.0.0:7700
-
-# REQUIRED FOR SEARCH: In production env., a secure key is needed. You can generate your own. 
-# This master key must be at least 16 bytes, composed of valid UTF-8 characters. 
-# MeiliSearch will throw an error and refuse to launch if no master key is provided, 
-# or if it is under 16 bytes. MeiliSearch will suggest a secure autogenerated master key.
-# Using docker, it seems recognized as production so use a secure key.
-# This is a ready made secure key for docker-compose, you can replace it with your own.
-MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
-
-##########################
-# User System:
-##########################
-
-# Allow Public Registration
+ALLOW_EMAIL_LOGIN=true
 ALLOW_REGISTRATION=true
-
-# Allow Social Registration
 ALLOW_SOCIAL_LOGIN=false
-
-# Allow Social Registration (WORKS ONLY for Google, Github, Discord)
 ALLOW_SOCIAL_REGISTRATION=false
 
-# JWT Secrets
-# You should use secure values. The examples given are 32-byte keys (64 characters in hex)
-# Use this replit to generate some quickly: https://replit.com/@daavila/crypto#index.js
+SESSION_EXPIRY=1000 * 60 * 15
+REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7
+
 JWT_SECRET=16f8c0ef4a5d391b26034086c628469d3f9f497f08163ab9b40137092f2909ef
 JWT_REFRESH_SECRET=eaa5191f2914e30b9387fd84e254e4ba6fc51b4654968a9b0803b456a54b8418
 
-# Google:
-# Add your Google Client ID and Secret here, you must register an app with Google Cloud to get these values
-# https://cloud.google.com/
-GOOGLE_CLIENT_ID=
-GOOGLE_CLIENT_SECRET=
-GOOGLE_CALLBACK_URL=/oauth/google/callback
+# Discord
+DISCORD_CLIENT_ID=
+DISCORD_CLIENT_SECRET=
+DISCORD_CALLBACK_URL=/oauth/discord/callback
 
-# Facebook:
-# Add your Facebook Client ID and Secret here, you must register an app with Facebook to get these values
-# https://developers.facebook.com/
+# Facebook
 FACEBOOK_CLIENT_ID=
 FACEBOOK_CLIENT_SECRET=
 FACEBOOK_CALLBACK_URL=/oauth/facebook/callback
 
-# OpenID:
-# See OpenID provider to get the below values
-# Create random string for OPENID_SESSION_SECRET
-# For Azure AD
-#   ISSUER:  https://login.microsoftonline.com/(tenant id)/v2.0/
-#   SCOPE:   openid profile email
+# GitHub
+GITHUB_CLIENT_ID=
+GITHUB_CLIENT_SECRET=
+GITHUB_CALLBACK_URL=/oauth/github/callback
+
+# Google
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
+GOOGLE_CALLBACK_URL=/oauth/google/callback
+
+# OpenID
 OPENID_CLIENT_ID=
 OPENID_CLIENT_SECRET=
 OPENID_ISSUER=
 OPENID_SESSION_SECRET=
 OPENID_SCOPE="openid profile email"
 OPENID_CALLBACK_URL=/oauth/openid/callback
-# If LABEL and URL are left empty, then the default OpenID label and logo are used.
+
 OPENID_BUTTON_LABEL=
 OPENID_IMAGE_URL=
 
-# Set the expiration delay for the secure cookie with the JWT token
-# Recommend session expiry to be 15 minutes
-# Delay is in millisecond e.g. 7 days is 1000*60*60*24*7
-SESSION_EXPIRY=1000 * 60 * 15
-REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7
+#========================#
+# Email Password Reset   #
+#========================#
 
-# Github:
-# Get the Client ID and Secret from your Discord Application
-# Add your Discord Client ID and Client Secret here:
+EMAIL_SERVICE=
+EMAIL_HOST=
+EMAIL_PORT=25
+EMAIL_ENCRYPTION=
+EMAIL_ENCRYPTION_HOSTNAME=
+EMAIL_ALLOW_SELFSIGNED=
+EMAIL_USERNAME=
+EMAIL_PASSWORD=
+EMAIL_FROM_NAME=
+EMAIL_FROM=noreply@librechat.ai
 
-GITHUB_CLIENT_ID=your_client_id
-GITHUB_CLIENT_SECRET=your_client_secret
-GITHUB_CALLBACK_URL=/oauth/github/callback # this should be the same for everyone
+#========================#
+# Firebase CDN           #
+#========================#
 
-# Discord:
-# Get the Client ID and Secret from your Discord Application
-# Add your Github Client ID and Client Secret here:
+FIREBASE_API_KEY=
+FIREBASE_AUTH_DOMAIN=
+FIREBASE_PROJECT_ID=
+FIREBASE_STORAGE_BUCKET=
+FIREBASE_MESSAGING_SENDER_ID=
+FIREBASE_APP_ID=
 
-DISCORD_CLIENT_ID=your_client_id
-DISCORD_CLIENT_SECRET=your_client_secret
-DISCORD_CALLBACK_URL=/oauth/discord/callback # this should be the same for everyone
+#===================================================#
+#                        UI                         #
+#===================================================#
 
-###########################
-# Application Domains
-###########################
+APP_TITLE=LibreChat
+# CUSTOM_FOOTER="My custom footer"
+HELP_AND_FAQ_URL=https://librechat.ai
 
-# Note: 
-# Server = Backend 
-# Client = Public (the client is the url you visit)
-# For the Google login to work in dev mode, you will need to change DOMAIN_SERVER to localhost:3090 or place it in .env.development
+# SHOW_BIRTHDAY_ICON=true
 
-DOMAIN_CLIENT=http://localhost:3080
-DOMAIN_SERVER=http://localhost:3080
+#==================================================#
+#                      Others                      #
+#==================================================#
+#   You should leave the following commented out   #
 
-###########################
-# Email
-###########################
+# NODE_ENV=
 
-# Email is used for password reset. Note that all 4 values must be set for email to work.
-# Failing to set the 4 values will result in LibreChat using the unsecured password reset!
-EMAIL_SERVICE=   # eg. gmail
-EMAIL_USERNAME=  # eg. your email address if using gmail
-EMAIL_PASSWORD=  # eg. this is the "app password" if using gmail
-EMAIL_FROM=noreply@librechat.ai # email address for from field, it is required to set a value here even in the cases where it's not porperly working.
+# REDIS_URI=
+# USE_REDIS=
+
+# E2E_USER_EMAIL=
+# E2E_USER_PASSWORD=

.eslintrc.js

@@ -19,6 +19,10 @@ module.exports = {
     'e2e/playwright-report/**/*',
     'packages/data-provider/types/**/*',
     'packages/data-provider/dist/**/*',
+    'packages/data-provider/test_bundle/**/*',
+    'data-node/**/*',
+    'meili_data/**/*',
+    'node_modules/**/*',
   ],
   parser: '@typescript-eslint/parser',
   parserOptions: {
@@ -128,6 +132,12 @@ module.exports = {
         },
       ],
     },
+    {
+      files: ['./packages/data-provider/specs/**/*.ts'],
+      parserOptions: {
+        project: './packages/data-provider/tsconfig.spec.json',
+      },
+    },
   ],
   settings: {
     react: {

.github/CODE_OF_CONDUCT.md

@@ -60,7 +60,7 @@ representative at an online or offline event.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported to the community leaders responsible for enforcement here on GitHub or
-on the official [Discord Server](https://discord.gg/uDyZ5Tzhct). 
+on the official [Discord Server](https://discord.librechat.ai). 
 All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the

.github/CONTRIBUTING.md

@@ -8,7 +8,7 @@ If the feature you would like to contribute has not already received prior appro
 
 Please note that a pull request involving a feature that has not been reviewed and approved by the project maintainers may be rejected. We appreciate your understanding and cooperation.
 
-If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.gg/uDyZ5Tzhct), where you can engage with other contributors and seek guidance from the community.
+If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.librechat.ai), where you can engage with other contributors and seek guidance from the community.
 
 ## Our Standards
 

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -7,14 +7,6 @@ body:
     attributes:
       value: |
         Thanks for taking the time to fill out this bug report!
-  - type: input
-    id: contact
-    attributes:
-      label: Contact Details
-      description: How can we get in touch with you if we need more info?
-      placeholder: ex. email@example.com
-    validations:
-      required: false
   - type: textarea
     id: what-happened
     attributes:
@@ -58,7 +50,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -7,14 +7,6 @@ body:
     attributes:
       value: |
         Thank you for taking the time to fill this out!
-  - type: input
-    id: contact
-    attributes:
-      label: Contact Details
-      description: How can we contact you if we need more information?
-      placeholder: ex. email@example.com
-    validations:
-      required: false
   - type: textarea
     id: what
     attributes:

.github/ISSUE_TEMPLATE/QUESTION.yml

@@ -7,14 +7,6 @@ body:
     attributes:
       value: |
         Thanks for taking the time to fill this!
-  - type: input
-    id: contact
-    attributes:
-      label: Contact Details
-      description: How can we get in touch with you if we need more info?
-      placeholder: ex. email@example.com
-    validations:
-      required: false
   - type: textarea
     id: what-is-your-question
     attributes:

.github/SECURITY.md

@@ -12,7 +12,7 @@ When reporting a security vulnerability, you have the following options to reach
 
 - **Option 2: GitHub Issues**: You can initiate first contact via GitHub Issues. However, please note that initial contact through GitHub Issues should not include any sensitive details.
 
-- **Option 3: Discord Server**: You can join our [Discord community](https://discord.gg/5rbRxn4uME) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
+- **Option 3: Discord Server**: You can join our [Discord community](https://discord.librechat.ai) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
 
 _After the initial contact, we will establish a private communication channel for further discussion._
 
@@ -39,11 +39,11 @@ Please note that as a security-conscious community, we may not always disclose d
 
 This security policy applies to the following GitHub repository:
 
-- Repository: [LibreChat](https://github.com/danny-avila/LibreChat)
+- Repository: [LibreChat](https://github.librechat.ai)
 
 ## Contact
 
-If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.gg/NGaa9RPCft) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
+If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.librechat.ai) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
 
 ## Acknowledgments
 

.github/playwright.yml

@@ -0,0 +1,72 @@
+# name: Playwright Tests
+# on:
+#   pull_request:
+#     branches: 
+#       - main
+#       - dev
+#       - release/*
+#     paths:
+#       - 'api/**'
+#       - 'client/**'
+#       - 'packages/**'
+#       - 'e2e/**'
+# jobs:
+#   tests_e2e:
+#     name: Run Playwright tests
+#     if: github.event.pull_request.head.repo.full_name == 'danny-avila/LibreChat'
+#     timeout-minutes: 60
+#     runs-on: ubuntu-latest
+#     env:
+#       NODE_ENV: CI
+#       CI: true
+#       SEARCH: false
+#       BINGAI_TOKEN: user_provided
+#       CHATGPT_TOKEN: user_provided
+#       MONGO_URI: ${{ secrets.MONGO_URI }}
+#       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+#       E2E_USER_EMAIL: ${{ secrets.E2E_USER_EMAIL }}
+#       E2E_USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
+#       JWT_SECRET: ${{ secrets.JWT_SECRET }}
+#       JWT_REFRESH_SECRET: ${{ secrets.JWT_REFRESH_SECRET }}
+#       CREDS_KEY: ${{ secrets.CREDS_KEY }}
+#       CREDS_IV: ${{ secrets.CREDS_IV }}
+#       DOMAIN_CLIENT: ${{ secrets.DOMAIN_CLIENT }}
+#       DOMAIN_SERVER: ${{ secrets.DOMAIN_SERVER }}
+#       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 # Skip downloading during npm install
+#       PLAYWRIGHT_BROWSERS_PATH: 0 # Places binaries to node_modules/@playwright/test
+#       TITLE_CONVO: false
+#     steps:
+#       - uses: actions/checkout@v4
+#       - uses: actions/setup-node@v4
+#         with:
+#           node-version: 18
+#           cache: 'npm'
+
+#       - name: Install global dependencies
+#         run: npm ci
+
+#       # - name: Remove sharp dependency
+#       #   run: rm -rf node_modules/sharp
+
+#       # - name: Install sharp with linux dependencies
+#       #   run: cd api && SHARP_IGNORE_GLOBAL_LIBVIPS=1 npm install --arch=x64 --platform=linux --libc=glibc sharp
+
+#       - name: Build Client
+#         run: npm run frontend
+
+#       - name: Install Playwright
+#         run: |
+#           npx playwright install-deps
+#           npm install -D @playwright/test@latest
+#           npx playwright install chromium
+
+#       - name: Run Playwright tests
+#         run: npm run e2e:ci
+
+#       - name: Upload playwright report
+#         uses: actions/upload-artifact@v3
+#         if: always()
+#         with:
+#           name: playwright-report
+#           path: e2e/playwright-report/
+#           retention-days: 30

.github/pull_request_template.md

@@ -1,7 +1,7 @@
 # Pull Request Template
 
 
-### ⚠️ Before Submitting a PR, read the [Contributing Docs](./CONTRIBUTING.md) in full!
+### ⚠️ Before Submitting a PR, read the [Contributing Docs](https://github.com/danny-avila/LibreChat/blob/main/.github/CONTRIBUTING.md) in full!
 
 ## Summary
 
@@ -15,7 +15,9 @@ Please delete any irrelevant options.
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] This change requires a documentation update
-- [ ] Documentation update 
+- [ ] Translation update
+- [ ] Documentation update
+
 
 ## Testing
 
@@ -25,6 +27,8 @@ Please describe your test process and include instructions so that we can reprod
 
 ## Checklist
 
+Please delete any irrelevant options.
+
 - [ ] My code adheres to this project's style guidelines
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented in any complex areas of my code
@@ -33,3 +37,4 @@ Please describe your test process and include instructions so that we can reprod
 - [ ] I have written tests demonstrating that my changes are effective or that my feature works
 - [ ] Local unit tests pass with my changes
 - [ ] Any changes dependent on mine have been merged and published in downstream modules.
+- [ ] New documents have been locally validated with mkdocs

.github/workflows/backend-review.yml

@@ -23,9 +23,9 @@ jobs:
       BAN_INTERVAL: ${{ secrets.BAN_INTERVAL }}
       NODE_ENV: CI
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Use Node.js 20.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
@@ -35,10 +35,28 @@ jobs:
 
       - name: Install Data Provider
         run: npm run build:data-provider
+      
+      - name: Create empty auth.json file
+        run: |
+          mkdir -p api/data
+          echo '{}' > api/data/auth.json
+
+      - name: Check for Circular dependency in rollup
+        working-directory: ./packages/data-provider
+        run: |
+          output=$(npm run rollup:api)
+          echo "$output"
+          if echo "$output" | grep -q "Circular dependency"; then
+            echo "Error: Circular dependency detected!"
+            exit 1
+          fi
 
       - name: Run unit tests
         run: cd api && npm run test:ci
 
+      - name: Run librechat-data-provider unit tests
+        run: cd packages/data-provider && npm run test:ci
+
       - name: Run linters
         uses: wearerequired/lint-action@v2
         with:

.github/workflows/container.yml

@@ -1,52 +0,0 @@
-name: Docker Compose Build on Tag
-
-# The workflow is triggered when a tag is pushed
-on:
-  push:
-    tags:
-      - "*"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v1
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Run docker-compose build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-          docker-compose build
-          docker build -f Dockerfile.multi --target api-build -t librechat-api .
-
-      # Get Tag Name
-      - name: Get Tag Name
-        id: tag_name
-        run: echo "TAG_NAME=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
-
-      # Tag it properly before push to github
-      - name: tag image and push
-        run: |
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:latest

.github/workflows/data-provider.yml

@@ -11,8 +11,8 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: 16
       - run: cd packages/data-provider && npm ci
@@ -22,8 +22,8 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: 16
           registry-url: 'https://registry.npmjs.org'

.github/workflows/deploy.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # checkout the repo
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@main
+        uses: actions/checkout@v4
 
       - name: 'Login via Azure CLI'
         uses: azure/login@v1

.github/workflows/dev-images.yml

@@ -8,19 +8,33 @@ on:
     paths:
       - 'api/**'
       - 'client/**'
+      - 'packages/**'
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-dev-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat-dev
 
     steps:
       # Check out the repository
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v1
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       # Log in to GitHub Container Registry
       - name: Log in to GitHub Container Registry
@@ -30,22 +44,29 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Build Docker images
-      - name: Build Docker images
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
         run: |
           cp .env.example .env
-          docker build -f Dockerfile.multi --target api-build -t librechat-dev-api .
-          docker build -f Dockerfile -t librechat-dev .
 
-      # Tag and push the images to GitHub Container Registry
-      - name: Tag and push images
-        run: |
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.sha }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.sha }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/frontend-review.yml

@@ -20,9 +20,9 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Use Node.js 20.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'

.github/workflows/generate_embeddings.yml

@@ -0,0 +1,20 @@
+name: 'generate_embeddings'
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: supabase/embeddings-generator@v0.0.5
+        with:
+          supabase-url: ${{ secrets.SUPABASE_URL }}
+          supabase-service-role-key: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
+          openai-key: ${{ secrets.OPENAI_DOC_EMBEDDINGS_KEY }}
+          docs-root-path: 'docs'

.github/workflows/latest-images-main.yml

@@ -1,40 +0,0 @@
-name: Docker Compose Build on Main Branch
-
-on:
-  workflow_dispatch:  # This line allows manual triggering
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v1
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Run docker-compose build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-          docker-compose build
-          docker build -f Dockerfile.multi --target api-build -t librechat-api .
-
-      # Tag and push the images with the 'latest' tag
-      - name: Tag image and push
-        run: |
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:latest

.github/workflows/main-image-workflow.yml

@@ -0,0 +1,69 @@
+name: Docker Compose Build Latest Main Image Tag (Manual Dispatch)
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Fetch tags and set the latest tag
+        run: |
+          git fetch --tags
+          echo "LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)" >> $GITHUB_ENV
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/mkdocs.yaml

@@ -9,7 +9,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: 3.x
@@ -21,4 +21,7 @@ jobs:
           restore-keys: |
             mkdocs-material-
       - run: pip install mkdocs-material 
+      - run: pip install mkdocs-nav-weight
+      - run: pip install mkdocs-publisher
+      - run: pip install mkdocs-exclude
       - run: mkdocs gh-deploy --force

.github/workflows/playwright.yml

@@ -1,72 +0,0 @@
-name: Playwright Tests
-on:
-  pull_request:
-    branches: 
-      - main
-      - dev
-      - release/*
-    paths:
-      - 'api/**'
-      - 'client/**'
-      - 'packages/**'
-      - 'e2e/**'
-jobs:
-  tests_e2e:
-    name: Run Playwright tests
-    if: github.event.pull_request.head.repo.full_name == 'danny-avila/LibreChat'
-    timeout-minutes: 60
-    runs-on: ubuntu-latest
-    env:
-      NODE_ENV: CI
-      CI: true
-      SEARCH: false
-      BINGAI_TOKEN: user_provided
-      CHATGPT_TOKEN: user_provided
-      MONGO_URI: ${{ secrets.MONGO_URI }}
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      E2E_USER_EMAIL: ${{ secrets.E2E_USER_EMAIL }}
-      E2E_USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
-      JWT_SECRET: ${{ secrets.JWT_SECRET }}
-      JWT_REFRESH_SECRET: ${{ secrets.JWT_REFRESH_SECRET }}
-      CREDS_KEY: ${{ secrets.CREDS_KEY }}
-      CREDS_IV: ${{ secrets.CREDS_IV }}
-      DOMAIN_CLIENT: ${{ secrets.DOMAIN_CLIENT }}
-      DOMAIN_SERVER: ${{ secrets.DOMAIN_SERVER }}
-      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 # Skip downloading during npm install
-      PLAYWRIGHT_BROWSERS_PATH: 0 # Places binaries to node_modules/@playwright/test
-      TITLE_CONVO: false
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18
-          cache: 'npm'
-
-      - name: Install global dependencies
-        run: npm ci
-
-      # - name: Remove sharp dependency
-      #   run: rm -rf node_modules/sharp
-
-      # - name: Install sharp with linux dependencies
-      #   run: cd api && SHARP_IGNORE_GLOBAL_LIBVIPS=1 npm install --arch=x64 --platform=linux --libc=glibc sharp
-
-      - name: Build Client
-        run: npm run frontend
-
-      - name: Install Playwright
-        run: |
-          npx playwright install-deps
-          npm install -D @playwright/test@latest
-          npx playwright install chromium
-
-      - name: Run Playwright tests
-        run: npm run e2e:ci
-
-      - name: Upload playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: e2e/playwright-report/
-          retention-days: 30

.github/workflows/tag-images.yml

@@ -0,0 +1,67 @@
+name: Docker Images Build on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      # Check out the repository
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.gitignore

@@ -2,7 +2,7 @@
 
 # Logs
 data-node
-meili_data
+meili_data*
 data/
 logs
 *.log
@@ -40,7 +40,7 @@ meili_data/
 api/node_modules/
 client/node_modules/
 bower_components/
-types/
+*.d.ts
 
 # Floobits
 .floo
@@ -48,6 +48,10 @@ types/
 .floo
 .flooignore
 
+#config file
+librechat.yaml
+librechat.yml
+
 # Environment
 .npmrc
 .env*
@@ -66,10 +70,16 @@ src/style - official.css
 .DS_Store
 *.code-workspace
 .idea
+*.iml
 *.pem
 config.local.ts
 **/storageState.json
 junit.xml
+**/.venv/
+
+# docker override file
+docker-compose.override.yaml
+docker-compose.override.yml
 
 # meilisearch
 meilisearch
@@ -78,4 +88,12 @@ data.ms/*
 auth.json
 
 /packages/ux-shared/
-/images
+/images
+
+!client/src/components/Nav/SettingsTabs/Data/
+
+# User uploads
+uploads/
+
+# owner
+release/

.husky/pre-commit

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env sh 
 set -e
 . "$(dirname -- "$0")/_/husky.sh"
 [ -n "$CI" ] && exit 0

Dockerfile

@@ -1,17 +1,26 @@
-# Base node image
-FROM node:19-alpine AS node
+# v0.7.0
 
-COPY . /app
+# Base node image
+FROM node:18-alpine3.18 AS node
+
+RUN apk add g++ make py3-pip
+RUN npm install -g node-gyp
+RUN apk --no-cache add curl
+
+RUN mkdir -p /app && chown node:node /app
 WORKDIR /app
 
-# Install call deps - Install curl for health check
-RUN apk --no-cache add curl && \
-    # We want to inherit env from the container, not the file
-    # This will preserve any existing env file if it's already in souce
-    # otherwise it will create a new one
-    touch .env && \
-    # Build deps in seperate 
-    npm ci
+USER node
+
+COPY --chown=node:node . .
+
+# Allow mounting of these files, which have no default
+# values.
+RUN touch .env
+RUN npm config set fetch-retry-maxtimeout 600000
+RUN npm config set fetch-retries 5
+RUN npm config set fetch-retry-mintimeout 15000
+RUN npm install --no-audit
 
 # React client build
 ENV NODE_OPTIONS="--max-old-space-size=2048"

Dockerfile.multi

@@ -1,34 +1,37 @@
+# v0.7.0
+
 # Build API, Client and Data Provider
-FROM node:19-alpine AS base
-
-WORKDIR /app
-COPY config/loader.js ./config/
-RUN npm install dotenv
-
-WORKDIR /app/api
-COPY api/package*.json ./
-COPY api/ ./
-RUN npm install
-
-# React client build
-FROM base AS client-build
-WORKDIR /app/client
-COPY ./client/ ./
+FROM node:20-alpine AS base
 
+# Build data-provider
+FROM base AS data-provider-build
 WORKDIR /app/packages/data-provider
 COPY ./packages/data-provider ./
 RUN npm install
 RUN npm run build
+
+# React client build
+FROM data-provider-build AS client-build
+WORKDIR /app/client
+COPY ./client/ ./
+# Copy data-provider to client's node_modules
 RUN mkdir -p /app/client/node_modules/librechat-data-provider/
 RUN cp -R /app/packages/data-provider/* /app/client/node_modules/librechat-data-provider/
-
-WORKDIR /app/client
 RUN npm install
 ENV NODE_OPTIONS="--max-old-space-size=2048"
 RUN npm run build
 
 # Node API setup
-FROM base AS api-build
+FROM data-provider-build AS api-build
+WORKDIR /app/api
+COPY api/package*.json ./
+COPY api/ ./
+# Copy helper scripts
+COPY config/ ./
+# Copy data-provider to API's node_modules
+RUN mkdir -p /app/api/node_modules/librechat-data-provider/
+RUN cp -R /app/packages/data-provider/* /app/api/node_modules/librechat-data-provider/
+RUN npm install
 COPY --from=client-build /app/client/dist /app/client/dist
 EXPOSE 3080
 ENV HOST=0.0.0.0

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 LibreChat
+Copyright (c) 2024 LibreChat
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,14 +1,14 @@
 <p align="center">
-  <a href="https://docs.librechat.ai">
+  <a href="https://librechat.ai">
     <img src="docs/assets/LibreChat.svg" height="256">
   </a>
-  <a href="https://docs.librechat.ai">
-    <h1 align="center">LibreChat</h1>
-  </a>
+  <h1 align="center">
+    <a href="https://librechat.ai">LibreChat</a>
+  </h1>
 </p>
 
 <p align="center">
-  <a href="https://discord.gg/NGaa9RPCft"> 
+  <a href="https://discord.librechat.ai"> 
     <img
       src="https://img.shields.io/discord/1086345563026489514?label=&logo=discord&style=for-the-badge&logoWidth=20&logoColor=white&labelColor=000000&color=blueviolet">
   </a>
@@ -20,120 +20,82 @@
     <img
       src="https://img.shields.io/badge/DOCS-blue.svg?style=for-the-badge&logo=read-the-docs&logoColor=white&labelColor=000000&logoWidth=20">
   </a>
-  <a aria-label="Sponsors" href="#sponsors">
+  <a aria-label="Sponsors" href="https://github.com/sponsors/danny-avila">
     <img
       src="https://img.shields.io/badge/SPONSORS-brightgreen.svg?style=for-the-badge&logo=github-sponsors&logoColor=white&labelColor=000000&logoWidth=20">
   </a>
 </p>
 
-## All-In-One AI Conversations with LibreChat ##
+<p align="center">
+<a href="https://railway.app/template/b5k2mn?referralCode=HI9hWz">
+  <img src="https://railway.app/button.svg" alt="Deploy on Railway" height="30">
+</a>
+<a href="https://zeabur.com/templates/0X2ZY8">
+  <img src="https://zeabur.com/button.svg" alt="Deploy on Zeabur" height="30"/>
+</a>
+<a href="https://template.cloud.sealos.io/deploy?templateName=librechat">
+  <img src="https://raw.githubusercontent.com/labring-actions/templates/main/Deploy-on-Sealos.svg" alt="Deploy on Sealos" height="30">
+</a>
+</p>
+
+# 📃 Features
+
+- 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and latest updates
+- 💬 Multimodal Chat:
+    - Upload and analyze images with Claude 3, GPT-4, and Gemini Vision 📸
+    - Chat with Files using Custom Endpoints, OpenAI, Azure, Anthropic, & Google. 🗃️
+    - Advanced Agents with Files, Code Interpreter, Tools, and API Actions 🔦
+      - Available through the [OpenAI Assistants API](https://platform.openai.com/docs/assistants/overview) 🌤️
+      - Non-OpenAI Agents in Active Development 🚧
+- 🌎 Multilingual UI:
+  - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
+  - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands, עברית
+- 🤖 AI model selection: OpenAI, Azure OpenAI, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins, Assistants API (including Azure Assistants)
+- 💾 Create, Save, & Share Custom Presets
+- 🔄 Edit, Resubmit, and Continue messages with conversation branching
+- 📤 Export conversations as screenshots, markdown, text, json.
+- 🔍 Search all messages/conversations
+- 🔌 Plugins, including web access, image generation with DALL-E-3 and more
+- 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
+- ⚙️ Configure Proxy, Reverse Proxy, Docker, & many Deployment options
+- 📖 Completely Open-Source & Built in Public
+- 🧑‍🤝‍🧑 Community-driven development, support, and feedback
+
+[For a thorough review of our features, see our docs here](https://docs.librechat.ai/features/plugins/introduction.html) 📚
+
+## 🪶 All-In-One AI Conversations with LibreChat
+
 LibreChat brings together the future of assistant AIs with the revolutionary technology of OpenAI's ChatGPT. Celebrating the original styling, LibreChat gives you the ability to integrate multiple AI models. It also integrates and enhances original client features such as conversation and message search, prompt templates and plugins.
 
 With LibreChat, you no longer need to opt for ChatGPT Plus and can instead use free or pay-per-call APIs. We welcome contributions, cloning, and forking to enhance the capabilities of this advanced chatbot platform.
-  
+
 <!-- https://github.com/danny-avila/LibreChat/assets/110412045/c1eb0c0f-41f6-4335-b982-84b278b53d59 -->
 
 [![Watch the video](https://img.youtube.com/vi/pNIOs1ovsXw/maxresdefault.jpg)](https://youtu.be/pNIOs1ovsXw)
 Click on the thumbnail to open the video☝️
 
-# Features
-- Response streaming identical to ChatGPT through server-sent events
-- UI from original ChatGPT, including Dark mode
-- AI model selection: OpenAI API, BingAI, ChatGPT Browser, PaLM2, Anthropic (Claude), Plugins
-- Create, Save, & Share custom presets - [More info on prompt presets here](https://github.com/danny-avila/LibreChat/releases/tag/v0.3.0)
-- Edit and Resubmit messages with conversation branching
-- Search all messages/conversations - [More info here](https://github.com/danny-avila/LibreChat/releases/tag/v0.1.0)
-- Plugins now available (including web access, image generation and more)
+---
+
+## 📚 Documentation
+
+For more information on how to use our advanced features, install and configure our software, and access our guidelines and tutorials, please check out our documentation at [docs.librechat.ai](https://docs.librechat.ai)
 
 ---
 
-## ⚠️ [Breaking Changes](docs/general_info/breaking_changes.md) ⚠️
+## 📝 Changelog
 
-**Please read this before updating from a previous version**
-
----
-
-## Changelog 
 Keep up with the latest updates by visiting the releases page - [Releases](https://github.com/danny-avila/LibreChat/releases)
 
----
-
-<h1>Table of Contents</h1>
-
-<details open>
-  <summary><strong>Getting Started</strong></summary>
-
-  * Installation
-    * [Docker Compose Install🐳](docs/install/docker_compose_install.md)
-    * [Linux Install🐧](docs/install/linux_install.md)
-    * [Mac Install🍎](docs/install/mac_install.md)
-    * [Windows Install💙](docs/install/windows_install.md)
-  * Configuration
-    * [APIs and Tokens](docs/install/apis_and_tokens.md)
-    * [User Auth System](docs/install/user_auth_system.md)
-    * [Online MongoDB Database](docs/install/mongodb.md)
-    * [Default Language](docs/install/default_language.md)
-</details>
-
-<details>
-  <summary><strong>General Information</strong></summary>
-
-  * [Code of Conduct](.github/CODE_OF_CONDUCT.md)
-  * [Project Origin](docs/general_info/project_origin.md)
-  * [Multilingual Information](docs/general_info/multilingual_information.md)
-  * [Tech Stack](docs/general_info/tech_stack.md)   
-</details>
-
-<details>
-  <summary><strong>Features</strong></summary>
-
-  * **Plugins**
-    * [Introduction](docs/features/plugins/introduction.md)
-    * [Google](docs/features/plugins/google_search.md)
-    * [Stable Diffusion](docs/features/plugins/stable_diffusion.md)
-    * [Wolfram](docs/features/plugins/wolfram.md)
-    * [Make Your Own Plugin](docs/features/plugins/make_your_own.md)
-    * [Using official ChatGPT Plugins](docs/features/plugins/chatgpt_plugins_openapi.md)
-
-  
-  * [Automated Moderation](docs/features/mod_system.md)
-  * [Third-Party Tools](docs/features/third_party.md)
-  * [Proxy](docs/features/proxy.md)
-  * [Bing Jailbreak](docs/features/bing_jailbreak.md)
-  * [Token Usage](docs/features/token_usage.md)
-</details>
-
-<details>
-  <summary><strong>Cloud Deployment</strong></summary>
-
-  * [DigitalOcean](docs/deployment/digitalocean.md)
-  * [Azure](docs/deployment/azure-terraform.md)
-  * [Linode](docs/deployment/linode.md)
-  * [Cloudflare](docs/deployment/cloudflare.md)
-  * [Ngrok](docs/deployment/ngrok.md)
-  * [HuggingFace](docs/deployment/huggingface.md)
-  * [Render](docs/deployment/render.md)
-  * [Meilisearch in Render](docs/deployment/meilisearch_in_render.md)
-  * [Hetzner](docs/deployment/hetzner_ubuntu.md)
-  * [Heroku](docs/deployment/heroku.md)
-</details>
-
-<details>
-  <summary><strong>Contributions</strong></summary>
-  
-  * [Contributor Guidelines](.github/CONTRIBUTING.md)
-  * [Documentation Guidelines](docs/contributions/documentation_guidelines.md)
-  * [Contribute a Translation](docs/contributions/translation_contribution.md)
-  * [Code Standards and Conventions](docs/contributions/coding_conventions.md)
-  * [Testing](docs/contributions/testing.md)
-  * [Security](.github/SECURITY.md)
-  * [Project Roadmap](https://github.com/users/danny-avila/projects/2)
-</details>
-
+**⚠️ [Breaking Changes](docs/general_info/breaking_changes.md)**
+Please consult the breaking changes before updating.
 
 ---
 
-## Star History
+## ⭐ Star History
+
+<p align="center">
+<a href="https://trendshift.io/repositories/4685" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4685" alt="danny-avila%2FLibreChat | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</p>
 
 <a href="https://star-history.com/#danny-avila/LibreChat&Date">
   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
@@ -141,24 +103,16 @@ Keep up with the latest updates by visiting the releases page - [Releases](https
 
 ---
 
-## Sponsors
+## ✨ Contributions
 
-  Sponsored by <a href="https://github.com/mjtechguy"><b>@mjtechguy</b></a>, <a href="https://github.com/SphaeroX"><b>@SphaeroX</b></a>, <a href="https://github.com/DavidDev1334"><b>@DavidDev1334</b></a>, <a href="https://github.com/fuegovic"><b>@fuegovic</b></a>, <a href="https://github.com/Pharrcyde"><b>@Pharrcyde</b></a> 
-  
----
+Contributions, suggestions, bug reports and fixes are welcome!
 
-## Contributors
-Contributions and suggestions bug reports and fixes are welcome!
-Please read the documentation before you do!
+For new features, components, or extensions, please open an issue and discuss before sending a PR.
 
 ---
 
-For new features, components, or extensions, please open an issue and discuss before sending a PR. 
+## 💖 This project exists in its current state thanks to all the people who contribute
 
-- Join the [Discord community](https://discord.gg/uDyZ5Tzhct)
-
-This project exists in its current state thanks to all the people who contribute
----
 <a href="https://github.com/danny-avila/LibreChat/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=danny-avila/LibreChat" />
 </a>

api/app/bingai.js

@@ -1,6 +1,7 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
-const { getUserKey, checkUserKeyExpiry } = require('../server/services/UserService');
+const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
+const { logger } = require('~/config');
 
 const askBing = async ({
   text,
@@ -100,7 +101,7 @@ const askBing = async ({
     }
   }
 
-  console.log('bing options', options);
+  logger.debug('bing options', options);
 
   const res = await bingAIClient.sendMessage(text, options);
 

api/app/chatgpt-browser.js

@@ -1,5 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
+const { Constants } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('../server/services/UserService');
 
 const browserClient = async ({
@@ -48,7 +49,7 @@ const browserClient = async ({
     options = { ...options, parentMessageId, conversationId };
   }
 
-  if (parentMessageId === '00000000-0000-0000-0000-000000000000') {
+  if (parentMessageId === Constants.NO_PARENT) {
     delete options.conversationId;
   }
 

api/app/clients/AnthropicClient.js

@@ -1,20 +1,42 @@
-// const { Agent, ProxyAgent } = require('undici');
-const BaseClient = require('./BaseClient');
-const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const Anthropic = require('@anthropic-ai/sdk');
+const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
+const {
+  getResponseSender,
+  EModelEndpoint,
+  validateVisionModel,
+} = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images/encode');
+const {
+  titleFunctionPrompt,
+  parseTitleFromPrompt,
+  truncateText,
+  formatMessage,
+  createContextHandlers,
+} = require('./prompts');
+const spendTokens = require('~/models/spendTokens');
+const { getModelMaxTokens } = require('~/utils');
+const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
 
 const HUMAN_PROMPT = '\n\nHuman:';
 const AI_PROMPT = '\n\nAssistant:';
 
 const tokenizersCache = {};
 
+/** Helper function to introduce a delay before retrying */
+function delayBeforeRetry(attempts, baseDelay = 1000) {
+  return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts));
+}
+
 class AnthropicClient extends BaseClient {
-  constructor(apiKey, options = {}, cacheOptions = {}) {
-    super(apiKey, options, cacheOptions);
+  constructor(apiKey, options = {}) {
+    super(apiKey, options);
     this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY;
-    this.sender = 'Anthropic';
     this.userLabel = HUMAN_PROMPT;
     this.assistantLabel = AI_PROMPT;
+    this.contextStrategy = options.contextStrategy
+      ? options.contextStrategy.toLowerCase()
+      : 'discard';
     this.setOptions(options);
   }
 
@@ -40,13 +62,20 @@ class AnthropicClient extends BaseClient {
       ...modelOptions,
       // set some good defaults (check for undefined in some cases because they may be 0)
       model: modelOptions.model || 'claude-1',
-      temperature: typeof modelOptions.temperature === 'undefined' ? 0.7 : modelOptions.temperature, // 0 - 1, 0.7 is recommended
+      temperature: typeof modelOptions.temperature === 'undefined' ? 1 : modelOptions.temperature, // 0 - 1, 1 is default
       topP: typeof modelOptions.topP === 'undefined' ? 0.7 : modelOptions.topP, // 0 - 1, default: 0.7
       topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40
       stop: modelOptions.stop, // no stop method for now
     };
 
-    this.maxContextTokens = this.options.maxContextTokens || 99999;
+    this.isClaude3 = this.modelOptions.model.includes('claude-3');
+    this.useMessages = this.isClaude3 || !!this.options.attachments;
+
+    this.defaultVisionModel = this.options.visionModel ?? 'claude-3-sonnet-20240229';
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
+
+    this.maxContextTokens =
+      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ?? 100000;
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500;
     this.maxPromptTokens =
       this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens;
@@ -59,6 +88,14 @@ class AnthropicClient extends BaseClient {
       );
     }
 
+    this.sender =
+      this.options.sender ??
+      getResponseSender({
+        model: this.modelOptions.model,
+        endpoint: EModelEndpoint.anthropic,
+        modelLabel: this.options.modelLabel,
+      });
+
     this.startToken = '||>';
     this.endToken = '';
     this.gptEncoder = this.constructor.getTokenizer('cl100k_base');
@@ -77,17 +114,90 @@ class AnthropicClient extends BaseClient {
     return this;
   }
 
+  /**
+   * Get the initialized Anthropic client.
+   * @returns {Anthropic} The Anthropic client instance.
+   */
   getClient() {
+    /** @type {Anthropic.default.RequestOptions} */
+    const options = {
+      apiKey: this.apiKey,
+    };
+
     if (this.options.reverseProxyUrl) {
-      return new Anthropic({
-        apiKey: this.apiKey,
-        baseURL: this.options.reverseProxyUrl,
-      });
-    } else {
-      return new Anthropic({
-        apiKey: this.apiKey,
-      });
+      options.baseURL = this.options.reverseProxyUrl;
     }
+
+    return new Anthropic(options);
+  }
+
+  getTokenCountForResponse(response) {
+    return this.getTokenCountForMessage({
+      role: 'assistant',
+      content: response.text,
+    });
+  }
+
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
+   * - Sets `this.isVisionModel` to `true` if vision request.
+   * - Deletes `this.modelOptions.stop` if vision request.
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.anthropic];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    const visionModelAvailable = availableModels?.includes(this.defaultVisionModel);
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      visionModelAvailable &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+  }
+
+  /**
+   * Calculate the token cost in tokens for an image based on its dimensions and detail level.
+   *
+   * For reference, see: https://docs.anthropic.com/claude/docs/vision#image-costs
+   *
+   * @param {Object} image - The image object.
+   * @param {number} image.width - The width of the image.
+   * @param {number} image.height - The height of the image.
+   * @returns {number} The calculated token cost measured by tokens.
+   *
+   */
+  calculateImageTokenCost({ width, height }) {
+    return Math.ceil((width * height) / 750);
+  }
+
+  async addImageURLs(message, attachments) {
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      EModelEndpoint.anthropic,
+    );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
+  async recordTokenUsage({ promptTokens, completionTokens, model, context = 'message' }) {
+    await spendTokens(
+      {
+        context,
+        user: this.user,
+        conversationId: this.conversationId,
+        model: model ?? this.modelOptions.model,
+        endpointTokenConfig: this.options.endpointTokenConfig,
+      },
+      { promptTokens, completionTokens },
+    );
   }
 
   async buildMessages(messages, parentMessageId) {
@@ -95,32 +205,148 @@ class AnthropicClient extends BaseClient {
       messages,
       parentMessageId,
     });
-    if (this.options.debug) {
-      console.debug('AnthropicClient: orderedMessages', orderedMessages, parentMessageId);
+
+    logger.debug('[AnthropicClient] orderedMessages', { orderedMessages, parentMessageId });
+
+    if (this.options.attachments) {
+      const attachments = await this.options.attachments;
+      const images = attachments.filter((file) => file.type.includes('image'));
+
+      if (images.length && !this.isVisionModel) {
+        throw new Error('Images are only supported with the Claude 3 family of models');
+      }
+
+      const latestMessage = orderedMessages[orderedMessages.length - 1];
+
+      if (this.message_file_map) {
+        this.message_file_map[latestMessage.messageId] = attachments;
+      } else {
+        this.message_file_map = {
+          [latestMessage.messageId]: attachments,
+        };
+      }
+
+      const files = await this.addImageURLs(latestMessage, attachments);
+
+      this.options.attachments = files;
     }
 
-    const formattedMessages = orderedMessages.map((message) => ({
-      author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
-      content: message?.content ?? message.text,
-    }));
+    if (this.message_file_map) {
+      this.contextHandlers = createContextHandlers(
+        this.options.req,
+        orderedMessages[orderedMessages.length - 1].text,
+      );
+    }
+
+    const formattedMessages = orderedMessages.map((message, i) => {
+      const formattedMessage = this.useMessages
+        ? formatMessage({
+          message,
+          endpoint: EModelEndpoint.anthropic,
+        })
+        : {
+          author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
+          content: message?.content ?? message.text,
+        };
+
+      const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+      /* If tokens were never counted, or, is a Vision request and the message has files, count again */
+      if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {
+        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
+      }
+
+      /* If message has files, calculate image token cost */
+      if (this.message_file_map && this.message_file_map[message.messageId]) {
+        const attachments = this.message_file_map[message.messageId];
+        for (const file of attachments) {
+          if (file.embedded) {
+            this.contextHandlers?.processFile(file);
+            continue;
+          }
+
+          orderedMessages[i].tokenCount += this.calculateImageTokenCost({
+            width: file.width,
+            height: file.height,
+          });
+        }
+      }
+
+      formattedMessage.tokenCount = orderedMessages[i].tokenCount;
+      return formattedMessage;
+    });
+
+    if (this.contextHandlers) {
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + (this.options.promptPrefix ?? '');
+    }
+
+    let { context: messagesInWindow, remainingContextTokens } =
+      await this.getMessagesWithinTokenLimit(formattedMessages);
+
+    const tokenCountMap = orderedMessages
+      .slice(orderedMessages.length - messagesInWindow.length)
+      .reduce((map, message, index) => {
+        const { messageId } = message;
+        if (!messageId) {
+          return map;
+        }
+
+        map[messageId] = orderedMessages[index].tokenCount;
+        return map;
+      }, {});
+
+    logger.debug('[AnthropicClient]', {
+      messagesInWindow: messagesInWindow.length,
+      remainingContextTokens,
+    });
 
     let lastAuthor = '';
     let groupedMessages = [];
 
-    for (let message of formattedMessages) {
+    for (let i = 0; i < messagesInWindow.length; i++) {
+      const message = messagesInWindow[i];
+      const author = message.role ?? message.author;
       // If last author is not same as current author, add to new group
-      if (lastAuthor !== message.author) {
-        groupedMessages.push({
-          author: message.author,
+      if (lastAuthor !== author) {
+        const newMessage = {
           content: [message.content],
-        });
-        lastAuthor = message.author;
+        };
+
+        if (message.role) {
+          newMessage.role = message.role;
+        } else {
+          newMessage.author = message.author;
+        }
+
+        groupedMessages.push(newMessage);
+        lastAuthor = author;
         // If same author, append content to the last group
       } else {
         groupedMessages[groupedMessages.length - 1].content.push(message.content);
       }
     }
 
+    groupedMessages = groupedMessages.map((msg, i) => {
+      const isLast = i === groupedMessages.length - 1;
+      if (msg.content.length === 1) {
+        const content = msg.content[0];
+        return {
+          ...msg,
+          // reason: final assistant content cannot end with trailing whitespace
+          content:
+            isLast && this.useMessages && msg.role === 'assistant' && typeof content === 'string'
+              ? content?.trim()
+              : content,
+        };
+      }
+
+      if (!this.useMessages && msg.tokenCount) {
+        delete msg.tokenCount;
+      }
+
+      return msg;
+    });
+
     let identityPrefix = '';
     if (this.options.userLabel) {
       identityPrefix = `\nHuman's name: ${this.options.userLabel}`;
@@ -146,9 +372,10 @@ class AnthropicClient extends BaseClient {
     // Prompt AI to respond, empty if last message was from AI
     let isEdited = lastAuthor === this.assistantLabel;
     const promptSuffix = isEdited ? '' : `${promptPrefix}${this.assistantLabel}\n`;
-    let currentTokenCount = isEdited
-      ? this.getTokenCount(promptPrefix)
-      : this.getTokenCount(promptSuffix);
+    let currentTokenCount =
+      isEdited || this.useMessages
+        ? this.getTokenCount(promptPrefix)
+        : this.getTokenCount(promptSuffix);
 
     let promptBody = '';
     const maxTokenCount = this.maxPromptTokens;
@@ -216,7 +443,69 @@ class AnthropicClient extends BaseClient {
       return true;
     };
 
-    await buildPromptBody();
+    const messagesPayload = [];
+    const buildMessagesPayload = async () => {
+      let canContinue = true;
+
+      if (promptPrefix) {
+        this.systemMessage = promptPrefix;
+      }
+
+      while (currentTokenCount < maxTokenCount && groupedMessages.length > 0 && canContinue) {
+        const message = groupedMessages.pop();
+
+        let tokenCountForMessage = message.tokenCount ?? this.getTokenCountForMessage(message);
+
+        const newTokenCount = currentTokenCount + tokenCountForMessage;
+        const exceededMaxCount = newTokenCount > maxTokenCount;
+
+        if (exceededMaxCount && messagesPayload.length === 0) {
+          throw new Error(
+            `Prompt is too long. Max token count is ${maxTokenCount}, but prompt is ${newTokenCount} tokens long.`,
+          );
+        } else if (exceededMaxCount) {
+          canContinue = false;
+          break;
+        }
+
+        delete message.tokenCount;
+        messagesPayload.unshift(message);
+        currentTokenCount = newTokenCount;
+
+        // Switch off isEdited after using it once
+        if (isEdited && message.role === 'assistant') {
+          isEdited = false;
+        }
+
+        // Wait for next tick to avoid blocking the event loop
+        await new Promise((resolve) => setImmediate(resolve));
+      }
+    };
+
+    const processTokens = () => {
+      // Add 2 tokens for metadata after all messages have been counted.
+      currentTokenCount += 2;
+
+      // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
+      this.modelOptions.maxOutputTokens = Math.min(
+        this.maxContextTokens - currentTokenCount,
+        this.maxResponseTokens,
+      );
+    };
+
+    if (this.modelOptions.model.startsWith('claude-3')) {
+      await buildMessagesPayload();
+      processTokens();
+      return {
+        prompt: messagesPayload,
+        context: messagesInWindow,
+        promptTokens: currentTokenCount,
+        tokenCountMap,
+      };
+    } else {
+      await buildPromptBody();
+      processTokens();
+    }
 
     if (nextMessage.remove) {
       promptBody = promptBody.replace(nextMessage.messageString, '');
@@ -226,20 +515,24 @@ class AnthropicClient extends BaseClient {
 
     let prompt = `${promptBody}${promptSuffix}`;
 
-    // Add 2 tokens for metadata after all messages have been counted.
-    currentTokenCount += 2;
-
-    // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
-    this.modelOptions.maxOutputTokens = Math.min(
-      this.maxContextTokens - currentTokenCount,
-      this.maxResponseTokens,
-    );
-
-    return { prompt, context };
+    return { prompt, context, promptTokens: currentTokenCount, tokenCountMap };
   }
 
   getCompletion() {
-    console.log('AnthropicClient doesn\'t use getCompletion (all handled in sendCompletion)');
+    logger.debug('AnthropicClient doesn\'t use getCompletion (all handled in sendCompletion)');
+  }
+
+  /**
+   * Creates a message or completion response using the Anthropic client.
+   * @param {Anthropic} client - The Anthropic client instance.
+   * @param {Anthropic.default.MessageCreateParams | Anthropic.default.CompletionCreateParams} options - The options for the message or completion.
+   * @param {boolean} useMessages - Whether to use messages or completions. Defaults to `this.useMessages`.
+   * @returns {Promise<Anthropic.default.Message | Anthropic.default.Completion>} The response from the Anthropic client.
+   */
+  async createResponse(client, options, useMessages) {
+    return useMessages ?? this.useMessages
+      ? await client.messages.create(options)
+      : await client.completions.create(options);
   }
 
   async sendCompletion(payload, { onProgress, abortController }) {
@@ -254,12 +547,7 @@ class AnthropicClient extends BaseClient {
       modelOptions.stream = true;
     }
 
-    const { debug } = this.options;
-    if (debug) {
-      console.debug();
-      console.debug(modelOptions);
-      console.debug();
-    }
+    logger.debug('modelOptions', { modelOptions });
 
     const client = this.getClient();
     const metadata = {
@@ -276,45 +564,88 @@ class AnthropicClient extends BaseClient {
       topP: top_p,
       topK: top_k,
     } = this.modelOptions;
+
     const requestOptions = {
-      prompt: payload,
       model,
       stream: stream || true,
-      max_tokens_to_sample: maxOutputTokens || 1500,
       stop_sequences,
       temperature,
       metadata,
       top_p,
       top_k,
     };
-    if (this.options.debug) {
-      console.log('AnthropicClient: requestOptions');
-      console.dir(requestOptions, { depth: null });
-    }
-    const response = await client.completions.create(requestOptions);
 
-    signal.addEventListener('abort', () => {
-      if (this.options.debug) {
-        console.log('AnthropicClient: message aborted!');
-      }
-      response.controller.abort();
-    });
-
-    for await (const completion of response) {
-      if (this.options.debug) {
-        // Uncomment to debug message stream
-        // console.debug(completion);
-      }
-      text += completion.completion;
-      onProgress(completion.completion);
+    if (this.useMessages) {
+      requestOptions.messages = payload;
+      requestOptions.max_tokens = maxOutputTokens || 1500;
+    } else {
+      requestOptions.prompt = payload;
+      requestOptions.max_tokens_to_sample = maxOutputTokens || 1500;
     }
 
-    signal.removeEventListener('abort', () => {
-      if (this.options.debug) {
-        console.log('AnthropicClient: message aborted!');
+    if (this.systemMessage) {
+      requestOptions.system = this.systemMessage;
+    }
+
+    logger.debug('[AnthropicClient]', { ...requestOptions });
+
+    const handleChunk = (currentChunk) => {
+      if (currentChunk) {
+        text += currentChunk;
+        onProgress(currentChunk);
       }
-      response.controller.abort();
-    });
+    };
+
+    const maxRetries = 3;
+    async function processResponse() {
+      let attempts = 0;
+
+      while (attempts < maxRetries) {
+        let response;
+        try {
+          response = await this.createResponse(client, requestOptions);
+
+          signal.addEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+
+          for await (const completion of response) {
+            // Handle each completion as before
+            if (completion?.delta?.text) {
+              handleChunk(completion.delta.text);
+            } else if (completion.completion) {
+              handleChunk(completion.completion);
+            }
+          }
+
+          // Successful processing, exit loop
+          break;
+        } catch (error) {
+          attempts += 1;
+          logger.warn(
+            `User: ${this.user} | Anthropic Request ${attempts} failed: ${error.message}`,
+          );
+
+          if (attempts < maxRetries) {
+            await delayBeforeRetry(attempts, 350);
+          } else {
+            throw new Error(`Operation failed after ${maxRetries} attempts: ${error.message}`);
+          }
+        } finally {
+          signal.removeEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+        }
+      }
+    }
+
+    await processResponse.bind(this)();
 
     return text.trim();
   }
@@ -323,14 +654,13 @@ class AnthropicClient extends BaseClient {
     return {
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
+      resendFiles: this.options.resendFiles,
       ...this.modelOptions,
     };
   }
 
   getBuildMessagesOptions() {
-    if (this.options.debug) {
-      console.log('AnthropicClient doesn\'t use getBuildMessagesOptions');
-    }
+    logger.debug('AnthropicClient doesn\'t use getBuildMessagesOptions');
   }
 
   static getTokenizer(encoding, isModelName = false, extendSpecialTokens = {}) {
@@ -350,6 +680,78 @@ class AnthropicClient extends BaseClient {
   getTokenCount(text) {
     return this.gptEncoder.encode(text, 'all').length;
   }
+
+  /**
+   * Generates a concise title for a conversation based on the user's input text and response.
+   * Involves sending a chat completion request with specific instructions for title generation.
+   *
+   * This function capitlizes on [Anthropic's function calling training](https://docs.anthropic.com/claude/docs/functions-external-tools).
+   *
+   * @param {Object} params - The parameters for the conversation title generation.
+   * @param {string} params.text - The user's input.
+   * @param {string} [params.responseText=''] - The AI's immediate response to the user.
+   *
+   * @returns {Promise<string | 'New Chat'>} A promise that resolves to the generated conversation title.
+   *                            In case of failure, it will return the default title, "New Chat".
+   */
+  async titleConvo({ text, responseText = '' }) {
+    let title = 'New Chat';
+    const convo = `<initial_message>
+  ${truncateText(text)}
+  </initial_message>
+  <response>
+  ${JSON.stringify(truncateText(responseText))}
+  </response>`;
+
+    const { ANTHROPIC_TITLE_MODEL } = process.env ?? {};
+    const model = this.options.titleModel ?? ANTHROPIC_TITLE_MODEL ?? 'claude-3-haiku-20240307';
+    const system = titleFunctionPrompt;
+
+    const titleChatCompletion = async () => {
+      const content = `<conversation_context>
+  ${convo}
+  </conversation_context>
+  
+  Please generate a title for this conversation.`;
+
+      const titleMessage = { role: 'user', content };
+      const requestOptions = {
+        model,
+        temperature: 0.3,
+        max_tokens: 1024,
+        system,
+        stop_sequences: ['\n\nHuman:', '\n\nAssistant', '</function_calls>'],
+        messages: [titleMessage],
+      };
+
+      try {
+        const response = await this.createResponse(this.getClient(), requestOptions, true);
+        let promptTokens = response?.usage?.input_tokens;
+        let completionTokens = response?.usage?.output_tokens;
+        if (!promptTokens) {
+          promptTokens = this.getTokenCountForMessage(titleMessage);
+          promptTokens += this.getTokenCountForMessage({ role: 'system', content: system });
+        }
+        if (!completionTokens) {
+          completionTokens = this.getTokenCountForMessage(response.content[0]);
+        }
+        await this.recordTokenUsage({
+          model,
+          promptTokens,
+          completionTokens,
+          context: 'title',
+        });
+        const text = response.content[0].text;
+        title = parseTitleFromPrompt(text);
+      } catch (e) {
+        logger.error('[AnthropicClient] There was an issue generating the title', e);
+      }
+    };
+
+    await titleChatCompletion();
+    logger.debug('[AnthropicClient] Convo Title: ' + title);
+    return title;
+  }
 }
 
 module.exports = AnthropicClient;

api/app/clients/BaseClient.js

@@ -1,8 +1,11 @@
 const crypto = require('crypto');
+const { supportsBalanceCheck, Constants } = require('librechat-data-provider');
+const { getConvo, getMessages, saveMessage, updateMessage, saveConvo } = require('~/models');
+const { addSpaceIfNeeded, isEnabled } = require('~/server/utils');
+const checkBalance = require('~/models/checkBalance');
+const { getFiles } = require('~/models/File');
 const TextStream = require('./TextStream');
-const { getConvo, getMessages, saveMessage, updateMessage, saveConvo } = require('../../models');
-const { addSpaceIfNeeded, isEnabled } = require('../../server/utils');
-const checkBalance = require('../../models/checkBalance');
+const { logger } = require('~/config');
 
 class BaseClient {
   constructor(apiKey, options = {}) {
@@ -41,15 +44,14 @@ class BaseClient {
   }
 
   async getTokenCountForResponse(response) {
-    if (this.options.debug) {
-      console.debug('`recordTokenUsage` not implemented.', response);
-    }
+    logger.debug('`[BaseClient] recordTokenUsage` not implemented.', response);
   }
 
   async recordTokenUsage({ promptTokens, completionTokens }) {
-    if (this.options.debug) {
-      console.debug('`recordTokenUsage` not implemented.', { promptTokens, completionTokens });
-    }
+    logger.debug('`[BaseClient] recordTokenUsage` not implemented.', {
+      promptTokens,
+      completionTokens,
+    });
   }
 
   getBuildMessagesOptions() {
@@ -62,7 +64,7 @@ class BaseClient {
   }
 
   async setMessageOptions(opts = {}) {
-    if (opts && typeof opts === 'object') {
+    if (opts && opts.replaceOptions) {
       this.setOptions(opts);
     }
 
@@ -72,7 +74,7 @@ class BaseClient {
     const saveOptions = this.getSaveOptions();
     this.abortController = opts.abortController ?? new AbortController();
     const conversationId = opts.conversationId ?? crypto.randomUUID();
-    const parentMessageId = opts.parentMessageId ?? '00000000-0000-0000-0000-000000000000';
+    const parentMessageId = opts.parentMessageId ?? Constants.NO_PARENT;
     const userMessageId = opts.overrideParentMessageId ?? crypto.randomUUID();
     let responseMessageId = opts.responseMessageId ?? crypto.randomUUID();
     let head = isEdited ? responseMessageId : parentMessageId;
@@ -194,14 +196,14 @@ class BaseClient {
       const update = {};
 
       if (messageId === tokenCountMap.summaryMessage?.messageId) {
-        this.options.debug && console.debug(`Adding summary props to ${messageId}.`);
+        logger.debug(`[BaseClient] Adding summary props to ${messageId}.`);
 
         update.summary = tokenCountMap.summaryMessage.content;
         update.summaryTokenCount = tokenCountMap.summaryMessage.tokenCount;
       }
 
       if (message.tokenCount && !update.summaryTokenCount) {
-        this.options.debug && console.debug(`Skipping ${messageId}: already had a token count.`);
+        logger.debug(`[BaseClient] Skipping ${messageId}: already had a token count.`);
         continue;
       }
 
@@ -278,19 +280,17 @@ class BaseClient {
     if (instructions) {
       ({ tokenCount, ..._instructions } = instructions);
     }
-    this.options.debug && _instructions && console.debug('instructions tokenCount', tokenCount);
+    _instructions && logger.debug('[BaseClient] instructions tokenCount: ' + tokenCount);
     let payload = this.addInstructions(formattedMessages, _instructions);
     let orderedWithInstructions = this.addInstructions(orderedMessages, instructions);
 
     let { context, remainingContextTokens, messagesToRefine, summaryIndex } =
       await this.getMessagesWithinTokenLimit(orderedWithInstructions);
 
-    this.options.debug &&
-      console.debug(
-        'remainingContextTokens, this.maxContextTokens (1/2)',
-        remainingContextTokens,
-        this.maxContextTokens,
-      );
+    logger.debug('[BaseClient] Context Count (1/2)', {
+      remainingContextTokens,
+      maxContextTokens: this.maxContextTokens,
+    });
 
     let summaryMessage;
     let summaryTokenCount;
@@ -308,10 +308,9 @@ class BaseClient {
 
     if (diff > 0) {
       payload = payload.slice(diff);
-      this.options.debug &&
-        console.debug(
-          `Difference between original payload (${length}) and context (${context.length}): ${diff}`,
-        );
+      logger.debug(
+        `[BaseClient] Difference between original payload (${length}) and context (${context.length}): ${diff}`,
+      );
     }
 
     const latestMessage = orderedWithInstructions[orderedWithInstructions.length - 1];
@@ -338,12 +337,10 @@ class BaseClient {
     // Make sure to only continue summarization logic if the summary message was generated
     shouldSummarize = summaryMessage && shouldSummarize;
 
-    this.options.debug &&
-      console.debug(
-        'remainingContextTokens, this.maxContextTokens (2/2)',
-        remainingContextTokens,
-        this.maxContextTokens,
-      );
+    logger.debug('[BaseClient] Context Count (2/2)', {
+      remainingContextTokens,
+      maxContextTokens: this.maxContextTokens,
+    });
 
     let tokenCountMap = orderedWithInstructions.reduce((map, message, index) => {
       const { messageId } = message;
@@ -361,19 +358,13 @@ class BaseClient {
 
     const promptTokens = this.maxContextTokens - remainingContextTokens;
 
-    if (this.options.debug) {
-      console.debug('<-------------------------PAYLOAD/TOKEN COUNT MAP------------------------->');
-      console.debug('Payload:', payload);
-      console.debug('Token Count Map:', tokenCountMap);
-      console.debug(
-        'Prompt Tokens',
-        promptTokens,
-        'remainingContextTokens',
-        remainingContextTokens,
-        'this.maxContextTokens',
-        this.maxContextTokens,
-      );
-    }
+    logger.debug('[BaseClient] tokenCountMap:', tokenCountMap);
+    logger.debug('[BaseClient]', {
+      promptTokens,
+      remainingContextTokens,
+      payloadSize: payload.length,
+      maxContextTokens: this.maxContextTokens,
+    });
 
     return { payload, tokenCountMap, promptTokens, messages: orderedWithInstructions };
   }
@@ -417,14 +408,14 @@ class BaseClient {
       // this only matters when buildMessages is utilizing the parentMessageId, and may vary on implementation
       isEdited ? head : userMessage.messageId,
       this.getBuildMessagesOptions(opts),
+      opts,
     );
 
     if (tokenCountMap) {
-      console.dir(tokenCountMap, { depth: null });
+      logger.debug('[BaseClient] tokenCountMap', tokenCountMap);
       if (tokenCountMap[userMessage.messageId]) {
         userMessage.tokenCount = tokenCountMap[userMessage.messageId];
-        console.log('userMessage.tokenCount', userMessage.tokenCount);
-        console.log('userMessage', userMessage);
+        logger.debug('[BaseClient] userMessage', userMessage);
       }
 
       this.handleTokenCountMap(tokenCountMap);
@@ -434,7 +425,10 @@ class BaseClient {
       await this.saveMessageToDatabase(userMessage, saveOptions, user);
     }
 
-    if (isEnabled(process.env.CHECK_BALANCE)) {
+    if (
+      isEnabled(process.env.CHECK_BALANCE) &&
+      supportsBalanceCheck[this.options.endpointType ?? this.options.endpoint]
+    ) {
       await checkBalance({
         req: this.options.req,
         res: this.options.res,
@@ -442,13 +436,16 @@ class BaseClient {
           user: this.user,
           tokenType: 'prompt',
           amount: promptTokens,
-          debug: this.options.debug,
           model: this.modelOptions.model,
+          endpoint: this.options.endpoint,
+          endpointTokenConfig: this.options.endpointTokenConfig,
         },
       });
     }
 
     const completion = await this.sendCompletion(payload, opts);
+    this.abortController.requestCompleted = true;
+
     const responseMessage = {
       messageId: responseMessageId,
       conversationId,
@@ -459,6 +456,7 @@ class BaseClient {
       sender: this.sender,
       text: addSpaceIfNeeded(generation) + completion,
       promptTokens,
+      ...(this.metadata ?? {}),
     };
 
     if (
@@ -481,9 +479,7 @@ class BaseClient {
   }
 
   async loadHistory(conversationId, parentMessageId = null) {
-    if (this.options.debug) {
-      console.debug('Loading history for conversation', conversationId, parentMessageId);
-    }
+    logger.debug('[BaseClient] Loading history:', { conversationId, parentMessageId });
 
     const messages = (await getMessages({ conversationId })) ?? [];
 
@@ -496,37 +492,45 @@ class BaseClient {
       mapMethod = this.getMessageMapMethod();
     }
 
-    const orderedMessages = this.constructor.getMessagesForConversation({
+    let _messages = this.constructor.getMessagesForConversation({
       messages,
       parentMessageId,
       mapMethod,
     });
 
+    _messages = await this.addPreviousAttachments(_messages);
+
     if (!this.shouldSummarize) {
-      return orderedMessages;
+      return _messages;
     }
 
     // Find the latest message with a 'summary' property
-    for (let i = orderedMessages.length - 1; i >= 0; i--) {
-      if (orderedMessages[i]?.summary) {
-        this.previous_summary = orderedMessages[i];
+    for (let i = _messages.length - 1; i >= 0; i--) {
+      if (_messages[i]?.summary) {
+        this.previous_summary = _messages[i];
         break;
       }
     }
 
-    if (this.options.debug && this.previous_summary) {
+    if (this.previous_summary) {
       const { messageId, summary, tokenCount, summaryTokenCount } = this.previous_summary;
-      console.debug('Previous summary:', { messageId, summary, tokenCount, summaryTokenCount });
+      logger.debug('[BaseClient] Previous summary:', {
+        messageId,
+        summary,
+        tokenCount,
+        summaryTokenCount,
+      });
     }
 
-    return orderedMessages;
+    return _messages;
   }
 
   async saveMessageToDatabase(message, endpointOptions, user = null) {
-    await saveMessage({ ...message, user, unfinished: false, cancelled: false });
+    await saveMessage({ ...message, endpoint: this.options.endpoint, user, unfinished: false });
     await saveConvo(user, {
       conversationId: message.conversationId,
       endpoint: this.options.endpoint,
+      endpointType: this.options.endpointType,
       ...endpointOptions,
     });
   }
@@ -548,7 +552,7 @@ class BaseClient {
    *
    * Each message object should have an 'id' or 'messageId' property and may have a 'parentMessageId' property.
    * The 'parentMessageId' is the ID of the message that the current message is a reply to.
-   * If 'parentMessageId' is not present, null, or is '00000000-0000-0000-0000-000000000000',
+   * If 'parentMessageId' is not present, null, or is Constants.NO_PARENT,
    * the message is considered a root message.
    *
    * @param {Object} options - The options for the function.
@@ -603,9 +607,7 @@ class BaseClient {
       }
 
       currentMessageId =
-        message.parentMessageId === '00000000-0000-0000-0000-000000000000'
-          ? null
-          : message.parentMessageId;
+        message.parentMessageId === Constants.NO_PARENT ? null : message.parentMessageId;
     }
 
     orderedMessages.reverse();
@@ -624,6 +626,11 @@ class BaseClient {
    * An additional 3 tokens need to be added for assistant label priming after all messages have been counted.
    * In our implementation, this is accounted for in the getMessagesWithinTokenLimit method.
    *
+   * The content parts example was adapted from the following example:
+   * https://github.com/openai/openai-cookbook/pull/881/files
+   *
+   * Note: image token calculation is to be done elsewhere where we have access to the image metadata
+   *
    * @param {Object} message
    */
   getTokenCountForMessage(message) {
@@ -636,14 +643,34 @@ class BaseClient {
       tokensPerName = -1;
     }
 
+    const processValue = (value) => {
+      if (Array.isArray(value)) {
+        for (let item of value) {
+          if (!item || !item.type || item.type === 'image_url') {
+            continue;
+          }
+
+          const nestedValue = item[item.type];
+
+          if (!nestedValue) {
+            continue;
+          }
+
+          processValue(nestedValue);
+        }
+      } else {
+        numTokens += this.getTokenCount(value);
+      }
+    };
+
     let numTokens = tokensPerMessage;
     for (let [key, value] of Object.entries(message)) {
-      numTokens += this.getTokenCount(value);
+      processValue(value);
+
       if (key === 'name') {
         numTokens += tokensPerName;
       }
     }
-
     return numTokens;
   }
 
@@ -654,6 +681,54 @@ class BaseClient {
 
     return await this.sendCompletion(payload, opts);
   }
+
+  /**
+   *
+   * @param {TMessage[]} _messages
+   * @returns {Promise<TMessage[]>}
+   */
+  async addPreviousAttachments(_messages) {
+    if (!this.options.resendFiles) {
+      return _messages;
+    }
+
+    /**
+     *
+     * @param {TMessage} message
+     */
+    const processMessage = async (message) => {
+      if (!this.message_file_map) {
+        /** @type {Record<string, MongoFile[]> */
+        this.message_file_map = {};
+      }
+
+      const fileIds = message.files.map((file) => file.file_id);
+      const files = await getFiles({
+        file_id: { $in: fileIds },
+      });
+
+      await this.addImageURLs(message, files);
+
+      this.message_file_map[message.messageId] = files;
+      return message;
+    };
+
+    const promises = [];
+
+    for (const message of _messages) {
+      if (!message.files) {
+        promises.push(message);
+        continue;
+      }
+
+      promises.push(processMessage(message));
+    }
+
+    const messages = await Promise.all(promises);
+
+    this.checkVisionRequest(Object.values(this.message_file_map ?? {}).flat());
+    return messages;
+  }
 }
 
 module.exports = BaseClient;

api/app/clients/ChatGPTClient.js

@@ -1,9 +1,16 @@
-const crypto = require('crypto');
 const Keyv = require('keyv');
+const crypto = require('crypto');
+const {
+  EModelEndpoint,
+  resolveHeaders,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
 const { Agent, ProxyAgent } = require('undici');
 const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
+const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils');
 
 const CHATGPT_MODEL = 'gpt-3.5-turbo';
 const tokenizersCache = {};
@@ -144,7 +151,8 @@ class ChatGPTClient extends BaseClient {
     if (!abortController) {
       abortController = new AbortController();
     }
-    const modelOptions = { ...this.modelOptions };
+
+    let modelOptions = { ...this.modelOptions };
     if (typeof onProgress === 'function') {
       modelOptions.stream = true;
     }
@@ -159,50 +167,171 @@ class ChatGPTClient extends BaseClient {
     }
 
     const { debug } = this.options;
-    const url = this.completionsUrl;
+    let baseURL = this.completionsUrl;
     if (debug) {
       console.debug();
-      console.debug(url);
+      console.debug(baseURL);
       console.debug(modelOptions);
       console.debug();
     }
+
     const opts = {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify(modelOptions),
       dispatcher: new Agent({
         bodyTimeout: 0,
         headersTimeout: 0,
       }),
     };
 
-    if (this.apiKey && this.options.azure) {
-      opts.headers['api-key'] = this.apiKey;
+    if (this.isVisionModel) {
+      modelOptions.max_tokens = 4000;
+    }
+
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const isAzure = this.azure || this.options.azure;
+    if (
+      (isAzure && this.isVisionModel && azureConfig) ||
+      (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    ) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+      opts.headers = resolveHeaders(headers);
+      this.langchainProxy = extractBaseURL(baseURL);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+      this.azure = !serverless && azureOptions;
+      this.azureEndpoint =
+        !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+    }
+
+    if (this.options.headers) {
+      opts.headers = { ...opts.headers, ...this.options.headers };
+    }
+
+    if (isAzure) {
+      // Azure does not accept `model` in the body, so we need to remove it.
+      delete modelOptions.model;
+
+      baseURL = this.langchainProxy
+        ? constructAzureURL({
+          baseURL: this.langchainProxy,
+          azureOptions: this.azure,
+        })
+        : this.azureEndpoint.split(/(?<!\/)\/(chat|completion)\//)[0];
+
+      if (this.options.forcePrompt) {
+        baseURL += '/completions';
+      } else {
+        baseURL += '/chat/completions';
+      }
+
+      opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
+      opts.headers = { ...opts.headers, 'api-key': this.apiKey };
     } else if (this.apiKey) {
       opts.headers.Authorization = `Bearer ${this.apiKey}`;
     }
 
+    if (process.env.OPENAI_ORGANIZATION) {
+      opts.headers['OpenAI-Organization'] = process.env.OPENAI_ORGANIZATION;
+    }
+
     if (this.useOpenRouter) {
       opts.headers['HTTP-Referer'] = 'https://librechat.ai';
       opts.headers['X-Title'] = 'LibreChat';
     }
 
-    if (this.options.headers) {
-      opts.headers = { ...opts.headers, ...this.options.headers };
-    }
-
     if (this.options.proxy) {
       opts.dispatcher = new ProxyAgent(this.options.proxy);
     }
 
+    /* hacky fixes for Mistral AI API:
+      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
+      - If there is only one message and it's a system message, change the role to user
+      */
+    if (baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+      const { messages } = modelOptions;
+
+      const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+      if (systemMessageIndex > 0) {
+        const [systemMessage] = messages.splice(systemMessageIndex, 1);
+        messages.unshift(systemMessage);
+      }
+
+      modelOptions.messages = messages;
+
+      if (messages.length === 1 && messages[0].role === 'system') {
+        modelOptions.messages[0].role = 'user';
+      }
+    }
+
+    if (this.options.addParams && typeof this.options.addParams === 'object') {
+      modelOptions = {
+        ...modelOptions,
+        ...this.options.addParams,
+      };
+      logger.debug('[ChatGPTClient] chatCompletion: added params', {
+        addParams: this.options.addParams,
+        modelOptions,
+      });
+    }
+
+    if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
+      this.options.dropParams.forEach((param) => {
+        delete modelOptions[param];
+      });
+      logger.debug('[ChatGPTClient] chatCompletion: dropped params', {
+        dropParams: this.options.dropParams,
+        modelOptions,
+      });
+    }
+
+    if (baseURL.includes('v1') && !baseURL.includes('/completions') && !this.isChatCompletion) {
+      baseURL = baseURL.split('v1')[0] + 'v1/completions';
+    } else if (
+      baseURL.includes('v1') &&
+      !baseURL.includes('/chat/completions') &&
+      this.isChatCompletion
+    ) {
+      baseURL = baseURL.split('v1')[0] + 'v1/chat/completions';
+    }
+
+    const BASE_URL = new URL(baseURL);
+    if (opts.defaultQuery) {
+      Object.entries(opts.defaultQuery).forEach(([key, value]) => {
+        BASE_URL.searchParams.append(key, value);
+      });
+      delete opts.defaultQuery;
+    }
+
+    const completionsURL = BASE_URL.toString();
+    opts.body = JSON.stringify(modelOptions);
+
     if (modelOptions.stream) {
       // eslint-disable-next-line no-async-promise-executor
       return new Promise(async (resolve, reject) => {
         try {
           let done = false;
-          await fetchEventSource(url, {
+          await fetchEventSource(completionsURL, {
             ...opts,
             signal: abortController.signal,
             async onopen(response) {
@@ -230,7 +359,6 @@ class ChatGPTClient extends BaseClient {
               // workaround for private API not sending [DONE] event
               if (!done) {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
               }
             },
@@ -243,14 +371,13 @@ class ChatGPTClient extends BaseClient {
             },
             onmessage(message) {
               if (debug) {
-                // console.debug(message);
+                console.debug(message);
               }
               if (!message.data || message.event === 'ping') {
                 return;
               }
               if (message.data === '[DONE]') {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
                 done = true;
                 return;
@@ -263,7 +390,7 @@ class ChatGPTClient extends BaseClient {
         }
       });
     }
-    const response = await fetch(url, {
+    const response = await fetch(completionsURL, {
       ...opts,
       signal: abortController.signal,
     });
@@ -548,7 +675,7 @@ ${botMessage.message}
     if (isChatGptModel) {
       return { prompt: [instructionsPayload, messagePayload], context };
     }
-    return { prompt, context };
+    return { prompt, context, promptTokens: currentTokenCount };
   }
 
   getTokenCount(text) {

api/app/clients/GoogleClient.js

@@ -1,23 +1,61 @@
-const BaseClient = require('./BaseClient');
 const { google } = require('googleapis');
 const { Agent, ProxyAgent } = require('undici');
+const { GoogleVertexAI } = require('langchain/llms/googlevertexai');
+const { ChatGoogleGenerativeAI } = require('@langchain/google-genai');
+const { ChatGoogleVertexAI } = require('langchain/chat_models/googlevertexai');
+const { AIMessage, HumanMessage, SystemMessage } = require('langchain/schema');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
+const {
+  validateVisionModel,
+  getResponseSender,
+  endpointSettings,
+  EModelEndpoint,
+  AuthKeys,
+} = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images');
+const { formatMessage, createContextHandlers } = require('./prompts');
+const { getModelMaxTokens } = require('~/utils');
+const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
 
+const loc = 'us-central1';
+const publisher = 'google';
+const endpointPrefix = `https://${loc}-aiplatform.googleapis.com`;
+// const apiEndpoint = loc + '-aiplatform.googleapis.com';
 const tokenizersCache = {};
 
+const settings = endpointSettings[EModelEndpoint.google];
+
 class GoogleClient extends BaseClient {
   constructor(credentials, options = {}) {
     super('apiKey', options);
-    this.client_email = credentials.client_email;
-    this.project_id = credentials.project_id;
-    this.private_key = credentials.private_key;
-    this.sender = 'PaLM2';
+    let creds = {};
+
+    if (typeof credentials === 'string') {
+      creds = JSON.parse(credentials);
+    } else if (credentials) {
+      creds = credentials;
+    }
+
+    const serviceKey = creds[AuthKeys.GOOGLE_SERVICE_KEY] ?? {};
+    this.serviceKey =
+      serviceKey && typeof serviceKey === 'string' ? JSON.parse(serviceKey) : serviceKey ?? {};
+    this.client_email = this.serviceKey.client_email;
+    this.private_key = this.serviceKey.private_key;
+    this.project_id = this.serviceKey.project_id;
+    this.access_token = null;
+
+    this.apiKey = creds[AuthKeys.GOOGLE_API_KEY];
+
+    if (options.skipSetOptions) {
+      return;
+    }
     this.setOptions(options);
   }
 
-  /* Google/PaLM2 specific methods */
+  /* Google specific methods */
   constructUrl() {
-    return `https://us-central1-aiplatform.googleapis.com/v1/projects/${this.project_id}/locations/us-central1/publishers/google/models/${this.modelOptions.model}:predict`;
+    return `${endpointPrefix}/v1/projects/${this.project_id}/locations/${loc}/publishers/${publisher}/models/${this.modelOptions.model}:serverStreamingPredict`;
   }
 
   async getClient() {
@@ -26,8 +64,7 @@ class GoogleClient extends BaseClient {
 
     jwtClient.authorize((err) => {
       if (err) {
-        console.error('Error: jwtClient failed to authorize');
-        console.error(err.message);
+        logger.error('jwtClient failed to authorize', err);
         throw err;
       }
     });
@@ -35,6 +72,22 @@ class GoogleClient extends BaseClient {
     return jwtClient;
   }
 
+  async getAccessToken() {
+    const scopes = ['https://www.googleapis.com/auth/cloud-platform'];
+    const jwtClient = new google.auth.JWT(this.client_email, null, this.private_key, scopes);
+
+    return new Promise((resolve, reject) => {
+      jwtClient.authorize((err, tokens) => {
+        if (err) {
+          logger.error('jwtClient failed to authorize', err);
+          reject(err);
+        } else {
+          resolve(tokens.access_token);
+        }
+      });
+    });
+  }
+
   /* Required Client methods */
   setOptions(options) {
     if (this.options && !this.options.replaceOptions) {
@@ -53,30 +106,44 @@ class GoogleClient extends BaseClient {
       this.options = options;
     }
 
-    this.options.examples = this.options.examples.filter(
-      (obj) => obj.input.content !== '' && obj.output.content !== '',
-    );
+    this.options.examples = (this.options.examples ?? [])
+      .filter((ex) => ex)
+      .filter((obj) => obj.input.content !== '' && obj.output.content !== '');
 
     const modelOptions = this.options.modelOptions || {};
     this.modelOptions = {
       ...modelOptions,
       // set some good defaults (check for undefined in some cases because they may be 0)
-      model: modelOptions.model || 'chat-bison',
-      temperature: typeof modelOptions.temperature === 'undefined' ? 0.2 : modelOptions.temperature, // 0 - 1, 0.2 is recommended
-      topP: typeof modelOptions.topP === 'undefined' ? 0.95 : modelOptions.topP, // 0 - 1, default: 0.95
-      topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40
+      model: modelOptions.model || settings.model.default,
+      temperature:
+        typeof modelOptions.temperature === 'undefined'
+          ? settings.temperature.default
+          : modelOptions.temperature,
+      topP: typeof modelOptions.topP === 'undefined' ? settings.topP.default : modelOptions.topP,
+      topK: typeof modelOptions.topK === 'undefined' ? settings.topK.default : modelOptions.topK,
       // stop: modelOptions.stop // no stop method for now
     };
 
-    this.isChatModel = this.modelOptions.model.startsWith('chat-');
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
+
+    // TODO: as of 12/14/23, only gemini models are "Generative AI" models provided by Google
+    this.isGenerativeModel = this.modelOptions.model.includes('gemini');
+    const { isGenerativeModel } = this;
+    this.isChatModel = !isGenerativeModel && this.modelOptions.model.includes('chat');
     const { isChatModel } = this;
-    this.isTextModel = this.modelOptions.model.startsWith('text-');
+    this.isTextModel =
+      !isGenerativeModel && !isChatModel && /code|text/.test(this.modelOptions.model);
     const { isTextModel } = this;
 
-    this.maxContextTokens = this.options.maxContextTokens || (isTextModel ? 8000 : 4096);
+    this.maxContextTokens = getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google);
     // The max prompt tokens is determined by the max context tokens minus the max response tokens.
     // Earlier messages will be dropped until the prompt is within the limit.
-    this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1024;
+    this.maxResponseTokens = this.modelOptions.maxOutputTokens || settings.maxOutputTokens.default;
+
+    if (this.maxContextTokens > 32000) {
+      this.maxContextTokens = this.maxContextTokens - this.maxResponseTokens;
+    }
+
     this.maxPromptTokens =
       this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens;
 
@@ -88,10 +155,18 @@ class GoogleClient extends BaseClient {
       );
     }
 
+    this.sender =
+      this.options.sender ??
+      getResponseSender({
+        model: this.modelOptions.model,
+        endpoint: EModelEndpoint.google,
+        modelLabel: this.options.modelLabel,
+      });
+
     this.userLabel = this.options.userLabel || 'User';
     this.modelLabel = this.options.modelLabel || 'Assistant';
 
-    if (isChatModel) {
+    if (isChatModel || isGenerativeModel) {
       // Use these faux tokens to help the AI understand the context since we are building the chat log ourselves.
       // Trying to use "<|im_start|>" causes the AI to still generate "<" or "<|" at the end sometimes for some reason,
       // without tripping the stop sequences, so I'm using "||>" instead.
@@ -99,8 +174,8 @@ class GoogleClient extends BaseClient {
       this.endToken = '';
       this.gptEncoder = this.constructor.getTokenizer('cl100k_base');
     } else if (isTextModel) {
-      this.startToken = '<|im_start|>';
-      this.endToken = '<|im_end|>';
+      this.startToken = '||>';
+      this.endToken = '';
       this.gptEncoder = this.constructor.getTokenizer('text-davinci-003', true, {
         '<|im_start|>': 100264,
         '<|im_end|>': 100265,
@@ -138,22 +213,123 @@ class GoogleClient extends BaseClient {
     return this;
   }
 
-  getMessageMapMethod() {
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    /* Validation vision request */
+    this.defaultVisionModel = this.options.visionModel ?? 'gemini-pro-vision';
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.google];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      availableModels?.includes(this.defaultVisionModel) &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+
+    if (this.isVisionModel && !attachments) {
+      this.modelOptions.model = 'gemini-pro';
+      this.isVisionModel = false;
+    }
+  }
+
+  formatMessages() {
     return ((message) => ({
       author: message?.author ?? (message.isCreatedByUser ? this.userLabel : this.modelLabel),
       content: message?.content ?? message.text,
     })).bind(this);
   }
 
-  buildMessages(messages = []) {
-    const formattedMessages = messages.map(this.getMessageMapMethod());
+  /**
+   *
+   * Adds image URLs to the message object and returns the files
+   *
+   * @param {TMessage[]} messages
+   * @param {MongoFile[]} files
+   * @returns {Promise<MongoFile[]>}
+   */
+  async addImageURLs(message, attachments) {
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      EModelEndpoint.google,
+    );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
+  async buildVisionMessages(messages = [], parentMessageId) {
+    const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    this.contextHandlers = createContextHandlers(this.options.req, latestMessage.text);
+
+    if (this.contextHandlers) {
+      for (const file of attachments) {
+        if (file.embedded) {
+          this.contextHandlers?.processFile(file);
+          continue;
+        }
+      }
+
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + this.options.promptPrefix;
+    }
+
+    const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId);
+
+    const files = await this.addImageURLs(latestMessage, attachments);
+
+    this.options.attachments = files;
+
+    latestMessage.text = prompt;
+
+    const payload = {
+      instances: [
+        {
+          messages: [new HumanMessage(formatMessage({ message: latestMessage }))],
+        },
+      ],
+      parameters: this.modelOptions,
+    };
+    return { prompt: payload };
+  }
+
+  async buildMessages(messages = [], parentMessageId) {
+    if (!this.isGenerativeModel && !this.project_id) {
+      throw new Error(
+        '[GoogleClient] a Service Account JSON Key is required for PaLM 2 and Codey models (Vertex AI)',
+      );
+    } else if (this.isGenerativeModel && (!this.apiKey || this.apiKey === 'user_provided')) {
+      throw new Error(
+        '[GoogleClient] an API Key is required for Gemini models (Generative Language API)',
+      );
+    }
+
+    if (this.options.attachments && this.isGenerativeModel) {
+      return this.buildVisionMessages(messages, parentMessageId);
+    }
+
+    if (this.isTextModel) {
+      return this.buildMessagesPrompt(messages, parentMessageId);
+    }
+
     let payload = {
       instances: [
         {
-          messages: formattedMessages,
+          messages: messages
+            .map(this.formatMessages())
+            .map((msg) => ({ ...msg, role: msg.author === 'User' ? 'user' : 'assistant' }))
+            .map((message) => formatMessage({ message, langChain: true })),
         },
       ],
-      parameters: this.options.modelOptions,
+      parameters: this.modelOptions,
     };
 
     if (this.options.promptPrefix) {
@@ -164,34 +340,171 @@ class GoogleClient extends BaseClient {
       payload.instances[0].examples = this.options.examples;
     }
 
-    /* TO-DO: text model needs more context since it can't process an array of messages */
-    if (this.isTextModel) {
-      payload.instances = [
-        {
-          prompt: messages[messages.length - 1].content,
-        },
-      ];
-    }
-
-    if (this.options.debug) {
-      console.debug('GoogleClient buildMessages');
-      console.dir(payload, { depth: null });
-    }
+    logger.debug('[GoogleClient] buildMessages', payload);
 
     return { prompt: payload };
   }
 
-  async getCompletion(payload, abortController = null) {
+  async buildMessagesPrompt(messages, parentMessageId) {
+    const orderedMessages = this.constructor.getMessagesForConversation({
+      messages,
+      parentMessageId,
+    });
+
+    logger.debug('[GoogleClient]', {
+      orderedMessages,
+      parentMessageId,
+    });
+
+    const formattedMessages = orderedMessages.map((message) => ({
+      author: message.isCreatedByUser ? this.userLabel : this.modelLabel,
+      content: message?.content ?? message.text,
+    }));
+
+    let lastAuthor = '';
+    let groupedMessages = [];
+
+    for (let message of formattedMessages) {
+      // If last author is not same as current author, add to new group
+      if (lastAuthor !== message.author) {
+        groupedMessages.push({
+          author: message.author,
+          content: [message.content],
+        });
+        lastAuthor = message.author;
+        // If same author, append content to the last group
+      } else {
+        groupedMessages[groupedMessages.length - 1].content.push(message.content);
+      }
+    }
+
+    let identityPrefix = '';
+    if (this.options.userLabel) {
+      identityPrefix = `\nHuman's name: ${this.options.userLabel}`;
+    }
+
+    if (this.options.modelLabel) {
+      identityPrefix = `${identityPrefix}\nYou are ${this.options.modelLabel}`;
+    }
+
+    let promptPrefix = (this.options.promptPrefix || '').trim();
+    if (promptPrefix) {
+      // If the prompt prefix doesn't end with the end token, add it.
+      if (!promptPrefix.endsWith(`${this.endToken}`)) {
+        promptPrefix = `${promptPrefix.trim()}${this.endToken}\n\n`;
+      }
+      promptPrefix = `\nContext:\n${promptPrefix}`;
+    }
+
+    if (identityPrefix) {
+      promptPrefix = `${identityPrefix}${promptPrefix}`;
+    }
+
+    // Prompt AI to respond, empty if last message was from AI
+    let isEdited = lastAuthor === this.modelLabel;
+    const promptSuffix = isEdited ? '' : `${promptPrefix}\n\n${this.modelLabel}:\n`;
+    let currentTokenCount = isEdited
+      ? this.getTokenCount(promptPrefix)
+      : this.getTokenCount(promptSuffix);
+
+    let promptBody = '';
+    const maxTokenCount = this.maxPromptTokens;
+
+    const context = [];
+
+    // Iterate backwards through the messages, adding them to the prompt until we reach the max token count.
+    // Do this within a recursive async function so that it doesn't block the event loop for too long.
+    // Also, remove the next message when the message that puts us over the token limit is created by the user.
+    // Otherwise, remove only the exceeding message. This is due to Anthropic's strict payload rule to start with "Human:".
+    const nextMessage = {
+      remove: false,
+      tokenCount: 0,
+      messageString: '',
+    };
+
+    const buildPromptBody = async () => {
+      if (currentTokenCount < maxTokenCount && groupedMessages.length > 0) {
+        const message = groupedMessages.pop();
+        const isCreatedByUser = message.author === this.userLabel;
+        // Use promptPrefix if message is edited assistant'
+        const messagePrefix =
+          isCreatedByUser || !isEdited
+            ? `\n\n${message.author}:`
+            : `${promptPrefix}\n\n${message.author}:`;
+        const messageString = `${messagePrefix}\n${message.content}${this.endToken}\n`;
+        let newPromptBody = `${messageString}${promptBody}`;
+
+        context.unshift(message);
+
+        const tokenCountForMessage = this.getTokenCount(messageString);
+        const newTokenCount = currentTokenCount + tokenCountForMessage;
+
+        if (!isCreatedByUser) {
+          nextMessage.messageString = messageString;
+          nextMessage.tokenCount = tokenCountForMessage;
+        }
+
+        if (newTokenCount > maxTokenCount) {
+          if (!promptBody) {
+            // This is the first message, so we can't add it. Just throw an error.
+            throw new Error(
+              `Prompt is too long. Max token count is ${maxTokenCount}, but prompt is ${newTokenCount} tokens long.`,
+            );
+          }
+
+          // Otherwise, ths message would put us over the token limit, so don't add it.
+          // if created by user, remove next message, otherwise remove only this message
+          if (isCreatedByUser) {
+            nextMessage.remove = true;
+          }
+
+          return false;
+        }
+        promptBody = newPromptBody;
+        currentTokenCount = newTokenCount;
+
+        // Switch off isEdited after using it for the first time
+        if (isEdited) {
+          isEdited = false;
+        }
+
+        // wait for next tick to avoid blocking the event loop
+        await new Promise((resolve) => setImmediate(resolve));
+        return buildPromptBody();
+      }
+      return true;
+    };
+
+    await buildPromptBody();
+
+    if (nextMessage.remove) {
+      promptBody = promptBody.replace(nextMessage.messageString, '');
+      currentTokenCount -= nextMessage.tokenCount;
+      context.shift();
+    }
+
+    let prompt = `${promptBody}${promptSuffix}`.trim();
+
+    // Add 2 tokens for metadata after all messages have been counted.
+    currentTokenCount += 2;
+
+    // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
+    this.modelOptions.maxOutputTokens = Math.min(
+      this.maxContextTokens - currentTokenCount,
+      this.maxResponseTokens,
+    );
+
+    return { prompt, context };
+  }
+
+  async _getCompletion(payload, abortController = null) {
     if (!abortController) {
       abortController = new AbortController();
     }
     const { debug } = this.options;
     const url = this.completionsUrl;
     if (debug) {
-      console.debug();
-      console.debug(url);
-      console.debug(this.modelOptions);
-      console.debug();
+      logger.debug('GoogleClient _getCompletion', { url, payload });
     }
     const opts = {
       method: 'POST',
@@ -208,10 +521,88 @@ class GoogleClient extends BaseClient {
 
     const client = await this.getClient();
     const res = await client.request({ url, method: 'POST', data: payload });
-    console.dir(res.data, { depth: null });
+    logger.debug('GoogleClient _getCompletion', { res });
     return res.data;
   }
 
+  createLLM(clientOptions) {
+    if (this.isGenerativeModel) {
+      return new ChatGoogleGenerativeAI({ ...clientOptions, apiKey: this.apiKey });
+    }
+
+    return this.isTextModel
+      ? new GoogleVertexAI(clientOptions)
+      : new ChatGoogleVertexAI(clientOptions);
+  }
+
+  async getCompletion(_payload, options = {}) {
+    const { onProgress, abortController } = options;
+    const { parameters, instances } = _payload;
+    const { messages: _messages, context, examples: _examples } = instances?.[0] ?? {};
+
+    let examples;
+
+    let clientOptions = { ...parameters, maxRetries: 2 };
+
+    if (!this.isGenerativeModel) {
+      clientOptions['authOptions'] = {
+        credentials: {
+          ...this.serviceKey,
+        },
+        projectId: this.project_id,
+      };
+    }
+
+    if (!parameters) {
+      clientOptions = { ...clientOptions, ...this.modelOptions };
+    }
+
+    if (this.isGenerativeModel) {
+      clientOptions.modelName = clientOptions.model;
+      delete clientOptions.model;
+    }
+
+    if (_examples && _examples.length) {
+      examples = _examples
+        .map((ex) => {
+          const { input, output } = ex;
+          if (!input || !output) {
+            return undefined;
+          }
+          return {
+            input: new HumanMessage(input.content),
+            output: new AIMessage(output.content),
+          };
+        })
+        .filter((ex) => ex);
+
+      clientOptions.examples = examples;
+    }
+
+    const model = this.createLLM(clientOptions);
+
+    let reply = '';
+    const messages = this.isTextModel ? _payload.trim() : _messages;
+
+    if (!this.isVisionModel && context && messages?.length > 0) {
+      messages.unshift(new SystemMessage(context));
+    }
+
+    const stream = await model.stream(messages, {
+      signal: abortController.signal,
+      timeout: 7000,
+    });
+
+    for await (const chunk of stream) {
+      await this.generateTextStream(chunk?.content ?? chunk, onProgress, {
+        delay: this.isGenerativeModel ? 12 : 8,
+      });
+      reply += chunk?.content ?? chunk;
+    }
+
+    return reply;
+  }
+
   getSaveOptions() {
     return {
       promptPrefix: this.options.promptPrefix,
@@ -221,38 +612,12 @@ class GoogleClient extends BaseClient {
   }
 
   getBuildMessagesOptions() {
-    // console.log('GoogleClient doesn\'t use getBuildMessagesOptions');
+    // logger.debug('GoogleClient doesn\'t use getBuildMessagesOptions');
   }
 
   async sendCompletion(payload, opts = {}) {
-    console.log('GoogleClient: sendcompletion', payload, opts);
     let reply = '';
-    let blocked = false;
-    try {
-      const result = await this.getCompletion(payload, opts.abortController);
-      blocked = result?.predictions?.[0]?.safetyAttributes?.blocked;
-      reply =
-        result?.predictions?.[0]?.candidates?.[0]?.content ||
-        result?.predictions?.[0]?.content ||
-        '';
-      if (blocked === true) {
-        reply = `Google blocked a proper response to your message:\n${JSON.stringify(
-          result.predictions[0].safetyAttributes,
-        )}${reply.length > 0 ? `\nAI Response:\n${reply}` : ''}`;
-      }
-      if (this.options.debug) {
-        console.debug('result');
-        console.debug(result);
-      }
-    } catch (err) {
-      console.error('Error: failed to send completion to Google');
-      console.error(err.message);
-    }
-
-    if (!blocked) {
-      await this.generateTextStream(reply, opts.onProgress, { delay: 0.5 });
-    }
-
+    reply = await this.getCompletion(payload, opts);
     return reply.trim();
   }
 

api/app/clients/OpenAIClient.js

@@ -1,17 +1,33 @@
 const OpenAI = require('openai');
 const { HttpsProxyAgent } = require('https-proxy-agent');
+const {
+  ImageDetail,
+  EModelEndpoint,
+  resolveHeaders,
+  ImageDetailCost,
+  getResponseSender,
+  validateVisionModel,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
-const { getModelMaxTokens, genAzureChatCompletion, extractBaseURL } = require('../../utils');
-const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts');
-const spendTokens = require('../../models/spendTokens');
+const {
+  extractBaseURL,
+  constructAzureURL,
+  getModelMaxTokens,
+  genAzureChatCompletion,
+} = require('~/utils');
+const { truncateText, formatMessage, createContextHandlers, CUT_OFF_PROMPT } = require('./prompts');
+const { encodeAndFormat } = require('~/server/services/Files/images/encode');
 const { handleOpenAIErrors } = require('./tools/util');
-const { isEnabled } = require('../../server/utils');
+const spendTokens = require('~/models/spendTokens');
 const { createLLM, RunManager } = require('./llm');
 const ChatGPTClient = require('./ChatGPTClient');
+const { isEnabled } = require('~/server/utils');
 const { summaryBuffer } = require('./memory');
 const { runTitleChain } = require('./chains');
 const { tokenSplit } = require('./document');
 const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
 
 // Cache to store Tiktoken instances
 const tokenizersCache = {};
@@ -24,15 +40,17 @@ class OpenAIClient extends BaseClient {
     this.ChatGPTClient = new ChatGPTClient();
     this.buildPrompt = this.ChatGPTClient.buildPrompt.bind(this);
     this.getCompletion = this.ChatGPTClient.getCompletion.bind(this);
-    this.sender = options.sender ?? 'ChatGPT';
     this.contextStrategy = options.contextStrategy
       ? options.contextStrategy.toLowerCase()
       : 'discard';
     this.shouldSummarize = this.contextStrategy === 'summarize';
+    /** @type {AzureOptions} */
     this.azure = options.azure || false;
     this.setOptions(options);
+    this.metadata = {};
   }
 
+  // TODO: PluginsClient calls this 3x, unneeded
   setOptions(options) {
     if (this.options && !this.options.replaceOptions) {
       this.options.modelOptions = {
@@ -53,6 +71,7 @@ class OpenAIClient extends BaseClient {
     }
 
     const modelOptions = this.options.modelOptions || {};
+
     if (!this.modelOptions) {
       this.modelOptions = {
         ...modelOptions,
@@ -72,6 +91,13 @@ class OpenAIClient extends BaseClient {
       };
     }
 
+    this.defaultVisionModel = this.options.visionModel ?? 'gpt-4-vision-preview';
+    if (typeof this.options.attachments?.then === 'function') {
+      this.options.attachments.then((attachments) => this.checkVisionRequest(attachments));
+    } else {
+      this.checkVisionRequest(this.options.attachments);
+    }
+
     const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {};
     if (OPENROUTER_API_KEY && !this.azure) {
       this.apiKey = OPENROUTER_API_KEY;
@@ -79,20 +105,33 @@ class OpenAIClient extends BaseClient {
     }
 
     const { reverseProxyUrl: reverseProxy } = this.options;
+
+    if (
+      !this.useOpenRouter &&
+      reverseProxy &&
+      reverseProxy.includes('https://openrouter.ai/api/v1')
+    ) {
+      this.useOpenRouter = true;
+    }
+
     this.FORCE_PROMPT =
       isEnabled(OPENAI_FORCE_PROMPT) ||
       (reverseProxy && reverseProxy.includes('completions') && !reverseProxy.includes('chat'));
 
+    if (typeof this.options.forcePrompt === 'boolean') {
+      this.FORCE_PROMPT = this.options.forcePrompt;
+    }
+
     if (this.azure && process.env.AZURE_OPENAI_DEFAULT_MODEL) {
-      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model);
+      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model, this);
       this.modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL;
     } else if (this.azure) {
-      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model);
+      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model, this);
     }
 
     const { model } = this.modelOptions;
 
-    this.isChatCompletion = this.useOpenRouter || !!reverseProxy || model.includes('gpt-');
+    this.isChatCompletion = this.useOpenRouter || !!reverseProxy || model.includes('gpt');
     this.isChatGptModel = this.isChatCompletion;
     if (
       model.includes('text-davinci') ||
@@ -105,14 +144,20 @@ class OpenAIClient extends BaseClient {
     const { isChatGptModel } = this;
     this.isUnofficialChatGptModel =
       model.startsWith('text-chat') || model.startsWith('text-davinci-002-render');
-    this.maxContextTokens = getModelMaxTokens(model) ?? 4095; // 1 less than maximum
+
+    this.maxContextTokens =
+      getModelMaxTokens(
+        model,
+        this.options.endpointType ?? this.options.endpoint,
+        this.options.endpointTokenConfig,
+      ) ?? 4095; // 1 less than maximum
 
     if (this.shouldSummarize) {
       this.maxContextTokens = Math.floor(this.maxContextTokens / 2);
     }
 
     if (this.options.debug) {
-      console.debug('maxContextTokens', this.maxContextTokens);
+      logger.debug('[OpenAIClient] maxContextTokens', this.maxContextTokens);
     }
 
     this.maxResponseTokens = this.modelOptions.max_tokens || 1024;
@@ -127,12 +172,22 @@ class OpenAIClient extends BaseClient {
       );
     }
 
+    this.sender =
+      this.options.sender ??
+      getResponseSender({
+        model: this.modelOptions.model,
+        endpoint: this.options.endpoint,
+        endpointType: this.options.endpointType,
+        chatGptLabel: this.options.chatGptLabel,
+        modelDisplayLabel: this.options.modelDisplayLabel,
+      });
+
     this.userLabel = this.options.userLabel || 'User';
     this.chatGptLabel = this.options.chatGptLabel || 'Assistant';
 
     this.setupTokens();
 
-    if (!this.modelOptions.stop) {
+    if (!this.modelOptions.stop && !this.isVisionModel) {
       const stopTokens = [this.startToken];
       if (this.endToken && this.endToken !== this.startToken) {
         stopTokens.push(this.endToken);
@@ -145,10 +200,6 @@ class OpenAIClient extends BaseClient {
     if (reverseProxy) {
       this.completionsUrl = reverseProxy;
       this.langchainProxy = extractBaseURL(reverseProxy);
-      !this.langchainProxy &&
-        console.warn(`The reverse proxy URL ${reverseProxy} is not valid for Plugins.
-The url must follow OpenAI specs, for example: https://localhost:8080/v1/chat/completions
-If your reverse proxy is compatible to OpenAI specs in every other way, it may still work without plugins enabled.`);
     } else if (isChatGptModel) {
       this.completionsUrl = 'https://api.openai.com/v1/chat/completions';
     } else {
@@ -160,7 +211,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     }
 
     if (this.azureEndpoint && this.options.debug) {
-      console.debug('Using Azure endpoint');
+      logger.debug('Using Azure endpoint');
     }
 
     if (this.useOpenRouter) {
@@ -170,6 +221,34 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     return this;
   }
 
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
+   * - Sets `this.isVisionModel` to `true` if vision request.
+   * - Deletes `this.modelOptions.stop` if vision request.
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    const availableModels = this.options.modelsConfig?.[this.options.endpoint];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    const visionModelAvailable = availableModels?.includes(this.defaultVisionModel);
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      visionModelAvailable &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+
+    if (this.isVisionModel) {
+      delete this.modelOptions.stop;
+    }
+  }
+
   setupTokens() {
     if (this.isChatCompletion) {
       this.startToken = '||>';
@@ -239,8 +318,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       // Reset count
       tokenizerCallsCount = 1;
     } catch (error) {
-      console.log('Free and reset encoders error');
-      console.error(error);
+      logger.error('[OpenAIClient] Free and reset encoders error', error);
     }
   }
 
@@ -248,14 +326,18 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
   resetTokenizersIfNecessary() {
     if (tokenizerCallsCount >= 25) {
       if (this.options.debug) {
-        console.debug('freeAndResetAllEncoders: reached 25 encodings, resetting...');
+        logger.debug('[OpenAIClient] freeAndResetAllEncoders: reached 25 encodings, resetting...');
       }
       this.constructor.freeAndResetAllEncoders();
     }
     tokenizerCallsCount++;
   }
 
-  // Returns the token count of a given text. It also checks and resets the tokenizers if necessary.
+  /**
+   * Returns the token count of a given text. It also checks and resets the tokenizers if necessary.
+   * @param {string} text - The text to get the token count for.
+   * @returns {number} The token count of the given text.
+   */
   getTokenCount(text) {
     this.resetTokenizersIfNecessary();
     try {
@@ -268,10 +350,33 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     }
   }
 
+  /**
+   * Calculate the token cost for an image based on its dimensions and detail level.
+   *
+   * @param {Object} image - The image object.
+   * @param {number} image.width - The width of the image.
+   * @param {number} image.height - The height of the image.
+   * @param {'low'|'high'|string|undefined} [image.detail] - The detail level ('low', 'high', or other).
+   * @returns {number} The calculated token cost.
+   */
+  calculateImageTokenCost({ width, height, detail }) {
+    if (detail === 'low') {
+      return ImageDetailCost.LOW;
+    }
+
+    // Calculate the number of 512px squares
+    const numSquares = Math.ceil(width / 512) * Math.ceil(height / 512);
+
+    // Default to high detail cost calculation
+    return numSquares * ImageDetailCost.HIGH + ImageDetailCost.ADDITIONAL;
+  }
+
   getSaveOptions() {
     return {
       chatGptLabel: this.options.chatGptLabel,
       promptPrefix: this.options.promptPrefix,
+      resendFiles: this.options.resendFiles,
+      imageDetail: this.options.imageDetail,
       ...this.modelOptions,
     };
   }
@@ -284,10 +389,25 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     };
   }
 
+  /**
+   *
+   * Adds image URLs to the message object and returns the files
+   *
+   * @param {TMessage[]} messages
+   * @param {MongoFile[]} files
+   * @returns {Promise<MongoFile[]>}
+   */
+  async addImageURLs(message, attachments) {
+    const { files, image_urls } = await encodeAndFormat(this.options.req, attachments);
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
   async buildMessages(
     messages,
     parentMessageId,
     { isChatCompletion = false, promptPrefix = null },
+    opts,
   ) {
     let orderedMessages = this.constructor.getMessagesForConversation({
       messages,
@@ -307,8 +427,74 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     let promptTokens;
 
     promptPrefix = (promptPrefix || this.options.promptPrefix || '').trim();
+
+    if (this.options.attachments) {
+      const attachments = await this.options.attachments;
+
+      if (this.message_file_map) {
+        this.message_file_map[orderedMessages[orderedMessages.length - 1].messageId] = attachments;
+      } else {
+        this.message_file_map = {
+          [orderedMessages[orderedMessages.length - 1].messageId]: attachments,
+        };
+      }
+
+      const files = await this.addImageURLs(
+        orderedMessages[orderedMessages.length - 1],
+        attachments,
+      );
+
+      this.options.attachments = files;
+    }
+
+    if (this.message_file_map) {
+      this.contextHandlers = createContextHandlers(
+        this.options.req,
+        orderedMessages[orderedMessages.length - 1].text,
+      );
+    }
+
+    const formattedMessages = orderedMessages.map((message, i) => {
+      const formattedMessage = formatMessage({
+        message,
+        userName: this.options?.name,
+        assistantName: this.options?.chatGptLabel,
+      });
+
+      const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+
+      /* If tokens were never counted, or, is a Vision request and the message has files, count again */
+      if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {
+        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
+      }
+
+      /* If message has files, calculate image token cost */
+      if (this.message_file_map && this.message_file_map[message.messageId]) {
+        const attachments = this.message_file_map[message.messageId];
+        for (const file of attachments) {
+          if (file.embedded) {
+            this.contextHandlers?.processFile(file);
+            continue;
+          }
+
+          orderedMessages[i].tokenCount += this.calculateImageTokenCost({
+            width: file.width,
+            height: file.height,
+            detail: this.options.imageDetail ?? ImageDetail.auto,
+          });
+        }
+      }
+
+      return formattedMessage;
+    });
+
+    if (this.contextHandlers) {
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      promptPrefix = this.augmentedPrompt + promptPrefix;
+    }
+
     if (promptPrefix) {
-      promptPrefix = `Instructions:\n${promptPrefix}`;
+      promptPrefix = `Instructions:\n${promptPrefix.trim()}`;
       instructions = {
         role: 'system',
         name: 'instructions',
@@ -320,20 +506,6 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       }
     }
 
-    const formattedMessages = orderedMessages.map((message, i) => {
-      const formattedMessage = formatMessage({
-        message,
-        userName: this.options?.name,
-        assistantName: this.options?.chatGptLabel,
-      });
-
-      if (this.contextStrategy && !orderedMessages[i].tokenCount) {
-        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
-      }
-
-      return formattedMessage;
-    });
-
     // TODO: need to handle interleaving instructions better
     if (this.contextStrategy) {
       ({ payload, tokenCountMap, promptTokens, messages } = await this.handleContextStrategy({
@@ -354,8 +526,8 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       result.tokenCountMap = tokenCountMap;
     }
 
-    if (promptTokens >= 0 && typeof this.options.getReqData === 'function') {
-      this.options.getReqData({ promptTokens });
+    if (promptTokens >= 0 && typeof opts?.getReqData === 'function') {
+      opts.getReqData({ promptTokens });
     }
 
     return result;
@@ -367,7 +539,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     let streamResult = null;
     this.modelOptions.user = this.user;
     const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null;
-    const useOldMethod = !!(this.azure || invalidBaseUrl || !this.isChatCompletion);
+    const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion || typeof Bun !== 'undefined');
     if (typeof opts.onProgress === 'function' && useOldMethod) {
       await this.getCompletion(
         payload,
@@ -376,11 +548,6 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
             return;
           }
 
-          if (this.options.debug) {
-            // console.debug('progressMessage');
-            // console.dir(progressMessage, { depth: null });
-          }
-
           if (progressMessage.choices) {
             streamResult = progressMessage;
           }
@@ -400,9 +567,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
           if (!token) {
             return;
           }
-          if (this.options.debug) {
-            // console.debug(token);
-          }
+
           if (token === this.endToken) {
             return;
           }
@@ -411,10 +576,9 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
         },
         opts.abortController || new AbortController(),
       );
-    } else if (typeof opts.onProgress === 'function') {
+    } else if (typeof opts.onProgress === 'function' || this.options.useChatCompletion) {
       reply = await this.chatCompletion({
         payload,
-        clientOptions: opts,
         onProgress: opts.onProgress,
         abortController: opts.abortController,
       });
@@ -424,9 +588,9 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
         null,
         opts.abortController || new AbortController(),
       );
-      if (this.options.debug) {
-        console.debug(JSON.stringify(result));
-      }
+
+      logger.debug('[OpenAIClient] sendCompletion: result', result);
+
       if (this.isChatCompletion) {
         reply = result.choices[0].message.content;
       } else {
@@ -434,11 +598,11 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       }
     }
 
-    if (streamResult && typeof opts.addMetadata === 'function') {
+    if (streamResult) {
       const { finish_reason } = streamResult.choices[0];
-      opts.addMetadata({ finish_reason });
+      this.metadata = { finish_reason };
     }
-    return reply.trim();
+    return (reply ?? '').trim();
   }
 
   initializeLLM({
@@ -452,6 +616,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     context,
     tokenBuffer,
     initialMessageCount,
+    conversationId,
   }) {
     const modelOptions = {
       modelName: modelName ?? model,
@@ -481,6 +646,16 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       };
     }
 
+    const { headers } = this.options;
+    if (headers && typeof headers === 'object' && !Array.isArray(headers)) {
+      configOptions.baseOptions = {
+        headers: resolveHeaders({
+          ...headers,
+          ...configOptions?.baseOptions?.headers,
+        }),
+      };
+    }
+
     if (this.options.proxy) {
       configOptions.httpAgent = new HttpsProxyAgent(this.options.proxy);
       configOptions.httpsAgent = new HttpsProxyAgent(this.options.proxy);
@@ -499,7 +674,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       callbacks: runManager.createCallbacks({
         context,
         tokenBuffer,
-        conversationId: this.conversationId,
+        conversationId: this.conversationId ?? conversationId,
         initialMessageCount,
       }),
     });
@@ -507,7 +682,21 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     return llm;
   }
 
-  async titleConvo({ text, responseText = '' }) {
+  /**
+   * Generates a concise title for a conversation based on the user's input text and response.
+   * Uses either specified method or starts with the OpenAI `functions` method (using LangChain).
+   * If the `functions` method fails, it falls back to the `completion` method,
+   * which involves sending a chat completion request with specific instructions for title generation.
+   *
+   * @param {Object} params - The parameters for the conversation title generation.
+   * @param {string} params.text - The user's input.
+   * @param {string} [params.conversationId] - The current conversationId, if not already defined on client initialization.
+   * @param {string} [params.responseText=''] - The AI's immediate response to the user.
+   *
+   * @returns {Promise<string | 'New Chat'>} A promise that resolves to the generated conversation title.
+   *                            In case of failure, it will return the default title, "New Chat".
+   */
+  async titleConvo({ text, conversationId, responseText = '' }) {
     let title = 'New Chat';
     const convo = `||>User:
 "${truncateText(text)}"
@@ -516,30 +705,58 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
 
     const { OPENAI_TITLE_MODEL } = process.env ?? {};
 
+    const model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+
     const modelOptions = {
-      model: OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo',
+      // TODO: remove the gpt fallback and make it specific to endpoint
+      model,
       temperature: 0.2,
       presence_penalty: 0,
       frequency_penalty: 0,
       max_tokens: 16,
     };
 
-    try {
-      this.abortController = new AbortController();
-      const llm = this.initializeLLM({ ...modelOptions, context: 'title', tokenBuffer: 150 });
-      title = await runTitleChain({ llm, text, convo, signal: this.abortController.signal });
-    } catch (e) {
-      if (e?.message?.toLowerCase()?.includes('abort')) {
-        this.options.debug && console.debug('Aborted title generation');
-        return;
-      }
-      console.log('There was an issue generating title with LangChain, trying the old method...');
-      this.options.debug && console.error(e.message, e);
-      modelOptions.model = OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const resetTitleOptions = !!(
+      (this.azure && azureConfig) ||
+      (azureConfig && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    );
+
+    if (resetTitleOptions) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+
+      this.options.headers = resolveHeaders(headers);
+      this.options.reverseProxyUrl = baseURL ?? null;
+      this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      this.options.forcePrompt = azureConfig.groupMap[groupName].forcePrompt;
+      this.azure = !serverless && azureOptions;
+    }
+
+    const titleChatCompletion = async () => {
+      modelOptions.model = model;
+
       if (this.azure) {
         modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL ?? modelOptions.model;
-        this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model);
+        this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this);
       }
+
       const instructionsPayload = [
         {
           role: 'system',
@@ -553,24 +770,64 @@ ${convo}
       ];
 
       try {
-        title = (await this.sendPayload(instructionsPayload, { modelOptions })).replaceAll('"', '');
+        title = (
+          await this.sendPayload(instructionsPayload, { modelOptions, useChatCompletion: true })
+        ).replaceAll('"', '');
       } catch (e) {
-        console.error(e);
-        console.log('There was another issue generating the title, see error above.');
+        logger.error(
+          '[OpenAIClient] There was an issue generating the title with the completion method',
+          e,
+        );
       }
+    };
+
+    if (this.options.titleMethod === 'completion') {
+      await titleChatCompletion();
+      logger.debug('[OpenAIClient] Convo Title: ' + title);
+      return title;
     }
 
-    console.log('CONVERSATION TITLE', title);
+    try {
+      this.abortController = new AbortController();
+      const llm = this.initializeLLM({
+        ...modelOptions,
+        conversationId,
+        context: 'title',
+        tokenBuffer: 150,
+      });
+      title = await runTitleChain({ llm, text, convo, signal: this.abortController.signal });
+    } catch (e) {
+      if (e?.message?.toLowerCase()?.includes('abort')) {
+        logger.debug('[OpenAIClient] Aborted title generation');
+        return;
+      }
+      logger.error(
+        '[OpenAIClient] There was an issue generating title with LangChain, trying completion method...',
+        e,
+      );
+
+      await titleChatCompletion();
+    }
+
+    logger.debug('[OpenAIClient] Convo Title: ' + title);
     return title;
   }
 
   async summarizeMessages({ messagesToRefine, remainingContextTokens }) {
-    this.options.debug && console.debug('Summarizing messages...');
+    logger.debug('[OpenAIClient] Summarizing messages...');
     let context = messagesToRefine;
     let prompt;
 
+    // TODO: remove the gpt fallback and make it specific to endpoint
     const { OPENAI_SUMMARY_MODEL = 'gpt-3.5-turbo' } = process.env ?? {};
-    const maxContextTokens = getModelMaxTokens(OPENAI_SUMMARY_MODEL) ?? 4095;
+    const model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL;
+    const maxContextTokens =
+      getModelMaxTokens(
+        model,
+        this.options.endpointType ?? this.options.endpoint,
+        this.options.endpointTokenConfig,
+      ) ?? 4095; // 1 less than maximum
+
     // 3 tokens for the assistant label, and 98 for the summarizer prompt (101)
     let promptBuffer = 101;
 
@@ -588,8 +845,9 @@ ${convo}
     }
 
     if (context.length === 0) {
-      this.options.debug &&
-        console.debug('Summary context is empty, using latest message within token limit');
+      logger.debug(
+        '[OpenAIClient] Summary context is empty, using latest message within token limit',
+      );
 
       promptBuffer = 32;
       const { text, ...latestMessage } = messagesToRefine[messagesToRefine.length - 1];
@@ -616,10 +874,10 @@ ${convo}
     // by recreating the summary prompt (single message) to avoid LangChain handling
 
     const initialPromptTokens = this.maxContextTokens - remainingContextTokens;
-    this.options.debug && console.debug(`initialPromptTokens: ${initialPromptTokens}`);
+    logger.debug('[OpenAIClient] initialPromptTokens', initialPromptTokens);
 
     const llm = this.initializeLLM({
-      model: OPENAI_SUMMARY_MODEL,
+      model,
       temperature: 0.2,
       context: 'summary',
       tokenBuffer: initialPromptTokens,
@@ -642,9 +900,9 @@ ${convo}
       const summaryTokenCount = this.getTokenCountForMessage(summaryMessage);
 
       if (this.options.debug) {
-        console.debug('summaryMessage:', summaryMessage);
-        console.debug(
-          `remainingContextTokens: ${remainingContextTokens}, after refining: ${
+        logger.debug('[OpenAIClient] summaryTokenCount', summaryTokenCount);
+        logger.debug(
+          `[OpenAIClient] Summarization complete: remainingContextTokens: ${remainingContextTokens}, after refining: ${
             remainingContextTokens - summaryTokenCount
           }`,
         );
@@ -653,7 +911,7 @@ ${convo}
       return { summaryMessage, summaryTokenCount };
     } catch (e) {
       if (e?.message?.toLowerCase()?.includes('abort')) {
-        this.options.debug && console.debug('Aborted summarization');
+        logger.debug('[OpenAIClient] Aborted summarization');
         const { run, runId } = this.runManager.getRunByConversationId(this.conversationId);
         if (run && run.error) {
           const { error } = run;
@@ -661,23 +919,19 @@ ${convo}
           throw new Error(error);
         }
       }
-      console.error('Error summarizing messages');
-      this.options.debug && console.error(e);
+      logger.error('[OpenAIClient] Error summarizing messages', e);
       return {};
     }
   }
 
   async recordTokenUsage({ promptTokens, completionTokens }) {
-    if (this.options.debug) {
-      console.debug('promptTokens', promptTokens);
-      console.debug('completionTokens', completionTokens);
-    }
     await spendTokens(
       {
         user: this.user,
         model: this.modelOptions.model,
         context: 'message',
         conversationId: this.conversationId,
+        endpointTokenConfig: this.options.endpointTokenConfig,
       },
       { promptTokens, completionTokens },
     );
@@ -690,7 +944,7 @@ ${convo}
     });
   }
 
-  async chatCompletion({ payload, onProgress, clientOptions, abortController = null }) {
+  async chatCompletion({ payload, onProgress, abortController = null }) {
     let error = null;
     const errorCallback = (err) => (error = err);
     let intermediateReply = '';
@@ -698,25 +952,22 @@ ${convo}
       if (!abortController) {
         abortController = new AbortController();
       }
-      const modelOptions = { ...this.modelOptions };
+
+      let modelOptions = { ...this.modelOptions };
+
       if (typeof onProgress === 'function') {
         modelOptions.stream = true;
       }
       if (this.isChatCompletion) {
         modelOptions.messages = payload;
       } else {
-        // TODO: unreachable code. Need to implement completions call for non-chat models
         modelOptions.prompt = payload;
       }
 
-      const { debug } = this.options;
-      const url = extractBaseURL(this.completionsUrl);
-      if (debug) {
-        console.debug('baseURL', url);
-        console.debug('modelOptions', modelOptions);
-      }
+      const baseURL = extractBaseURL(this.completionsUrl);
+      logger.debug('[OpenAIClient] chatCompletion', { baseURL, modelOptions });
       const opts = {
-        baseURL: url,
+        baseURL,
       };
 
       if (this.useOpenRouter) {
@@ -734,12 +985,111 @@ ${convo}
         opts.httpAgent = new HttpsProxyAgent(this.options.proxy);
       }
 
+      if (this.isVisionModel) {
+        modelOptions.max_tokens = 4000;
+      }
+
+      /** @type {TAzureConfig | undefined} */
+      const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+      if (
+        (this.azure && this.isVisionModel && azureConfig) ||
+        (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+      ) {
+        const { modelGroupMap, groupMap } = azureConfig;
+        const {
+          azureOptions,
+          baseURL,
+          headers = {},
+          serverless,
+        } = mapModelToAzureConfig({
+          modelName: modelOptions.model,
+          modelGroupMap,
+          groupMap,
+        });
+        opts.defaultHeaders = resolveHeaders(headers);
+        this.langchainProxy = extractBaseURL(baseURL);
+        this.apiKey = azureOptions.azureOpenAIApiKey;
+
+        const groupName = modelGroupMap[modelOptions.model].group;
+        this.options.addParams = azureConfig.groupMap[groupName].addParams;
+        this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+        // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+        this.azure = !serverless && azureOptions;
+        this.azureEndpoint =
+          !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+      }
+
+      if (this.azure || this.options.azure) {
+        // Azure does not accept `model` in the body, so we need to remove it.
+        delete modelOptions.model;
+
+        opts.baseURL = this.langchainProxy
+          ? constructAzureURL({
+            baseURL: this.langchainProxy,
+            azureOptions: this.azure,
+          })
+          : this.azureEndpoint.split(/(?<!\/)\/(chat|completion)\//)[0];
+
+        opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
+        opts.defaultHeaders = { ...opts.defaultHeaders, 'api-key': this.apiKey };
+      }
+
+      if (process.env.OPENAI_ORGANIZATION) {
+        opts.organization = process.env.OPENAI_ORGANIZATION;
+      }
+
       let chatCompletion;
+      /** @type {OpenAI} */
       const openai = new OpenAI({
         apiKey: this.apiKey,
         ...opts,
       });
 
+      /* hacky fixes for Mistral AI API:
+      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
+      - If there is only one message and it's a system message, change the role to user
+      */
+      if (opts.baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+        const { messages } = modelOptions;
+
+        const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+        if (systemMessageIndex > 0) {
+          const [systemMessage] = messages.splice(systemMessageIndex, 1);
+          messages.unshift(systemMessage);
+        }
+
+        modelOptions.messages = messages;
+
+        if (messages.length === 1 && messages[0].role === 'system') {
+          modelOptions.messages[0].role = 'user';
+        }
+      }
+
+      if (this.options.addParams && typeof this.options.addParams === 'object') {
+        modelOptions = {
+          ...modelOptions,
+          ...this.options.addParams,
+        };
+        logger.debug('[OpenAIClient] chatCompletion: added params', {
+          addParams: this.options.addParams,
+          modelOptions,
+        });
+      }
+
+      if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
+        this.options.dropParams.forEach((param) => {
+          delete modelOptions[param];
+        });
+        logger.debug('[OpenAIClient] chatCompletion: dropped params', {
+          dropParams: this.options.dropParams,
+          modelOptions,
+        });
+      }
+
+      let UnexpectedRoleError = false;
       if (modelOptions.stream) {
         const stream = await openai.beta.chat.completions
           .stream({
@@ -751,6 +1101,22 @@ ${convo}
           })
           .on('error', (err) => {
             handleOpenAIErrors(err, errorCallback, 'stream');
+          })
+          .on('finalChatCompletion', (finalChatCompletion) => {
+            const finalMessage = finalChatCompletion?.choices?.[0]?.message;
+            if (finalMessage && finalMessage?.role !== 'assistant') {
+              finalChatCompletion.choices[0].message.role = 'assistant';
+            }
+
+            if (finalMessage && !finalMessage?.content?.trim()) {
+              finalChatCompletion.choices[0].message.content = intermediateReply;
+            }
+          })
+          .on('finalMessage', (message) => {
+            if (message?.role !== 'assistant') {
+              stream.messages.push({ role: 'assistant', content: intermediateReply });
+              UnexpectedRoleError = true;
+            }
           });
 
         for await (const chunk of stream) {
@@ -763,9 +1129,11 @@ ${convo}
           }
         }
 
-        chatCompletion = await stream.finalChatCompletion().catch((err) => {
-          handleOpenAIErrors(err, errorCallback, 'finalChatCompletion');
-        });
+        if (!UnexpectedRoleError) {
+          chatCompletion = await stream.finalChatCompletion().catch((err) => {
+            handleOpenAIErrors(err, errorCallback, 'finalChatCompletion');
+          });
+        }
       }
       // regular completion
       else {
@@ -778,15 +1146,29 @@ ${convo}
           });
       }
 
-      if (!chatCompletion && error) {
+      if (!chatCompletion && UnexpectedRoleError) {
+        throw new Error(
+          'OpenAI error: Invalid final message: OpenAI expects final message to include role=assistant',
+        );
+      } else if (!chatCompletion && error) {
         throw new Error(error);
       } else if (!chatCompletion) {
         throw new Error('Chat completion failed');
       }
 
       const { message, finish_reason } = chatCompletion.choices[0];
-      if (chatCompletion && typeof clientOptions.addMetadata === 'function') {
-        clientOptions.addMetadata({ finish_reason });
+      if (chatCompletion) {
+        this.metadata = { finish_reason };
+      }
+
+      logger.debug('[OpenAIClient] chatCompletion response', chatCompletion);
+
+      if (!message?.content?.trim() && intermediateReply.length) {
+        logger.debug(
+          '[OpenAIClient] chatCompletion: using intermediateReply due to empty message.content',
+          { intermediateReply },
+        );
+        return intermediateReply;
       }
 
       return message.content;
@@ -795,26 +1177,30 @@ ${convo}
         err?.message?.includes('abort') ||
         (err instanceof OpenAI.APIError && err?.message?.includes('abort'))
       ) {
-        return '';
+        return intermediateReply;
       }
       if (
+        err?.message?.includes(
+          'OpenAI error: Invalid final message: OpenAI expects final message to include role=assistant',
+        ) ||
+        err?.message?.includes(
+          'stream ended without producing a ChatCompletionMessage with role=assistant',
+        ) ||
+        err?.message?.includes('The server had an error processing your request') ||
         err?.message?.includes('missing finish_reason') ||
+        err?.message?.includes('missing role') ||
         (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason'))
       ) {
-        await abortController.abortCompletion();
+        logger.error('[OpenAIClient] Known OpenAI error:', err);
         return intermediateReply;
       } else if (err instanceof OpenAI.APIError) {
-        console.log(err.name);
-        console.log(err.status);
-        console.log(err.headers);
         if (intermediateReply) {
           return intermediateReply;
         } else {
           throw err;
         }
       } else {
-        console.warn('[OpenAIClient.chatCompletion] Unhandled error type');
-        console.error(err);
+        logger.error('[OpenAIClient.chatCompletion] Unhandled error type', err);
         throw err;
       }
     }

api/app/clients/PluginsClient.js

@@ -3,12 +3,15 @@ const { CallbackManager } = require('langchain/callbacks');
 const { BufferMemory, ChatMessageHistory } = require('langchain/memory');
 const { initializeCustomAgent, initializeFunctionsAgent } = require('./agents');
 const { addImages, buildErrorInput, buildPromptPrefix } = require('./output_parsers');
-const checkBalance = require('../../models/checkBalance');
+const { processFileURL } = require('~/server/services/Files/process');
+const { EModelEndpoint } = require('librechat-data-provider');
 const { formatLangChainMessages } = require('./prompts');
-const { isEnabled } = require('../../server/utils');
-const { extractBaseURL } = require('../../utils');
+const checkBalance = require('~/models/checkBalance');
 const { SelfReflectionTool } = require('./tools');
+const { isEnabled } = require('~/server/utils');
+const { extractBaseURL } = require('~/utils');
 const { loadTools } = require('./tools/util');
+const { logger } = require('~/config');
 
 class PluginsClient extends OpenAIClient {
   constructor(apiKey, options = {}) {
@@ -28,18 +31,10 @@ class PluginsClient extends OpenAIClient {
 
     super.setOptions(options);
 
-    if (this.functionsAgent && this.agentOptions.model && !this.useOpenRouter) {
-      this.agentOptions.model = this.getFunctionModelName(this.agentOptions.model);
-    }
-
     this.isGpt3 = this.modelOptions?.model?.includes('gpt-3');
 
     if (this.options.reverseProxyUrl) {
       this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl);
-      !this.langchainProxy &&
-        console.warn(`The reverse proxy URL ${this.options.reverseProxyUrl} is not valid for Plugins.
-The url must follow OpenAI specs, for example: https://localhost:8080/v1/chat/completions
-If your reverse proxy is compatible to OpenAI specs in every other way, it may still work without plugins enabled.`);
     }
   }
 
@@ -57,7 +52,9 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
   }
 
   getFunctionModelName(input) {
-    if (input.includes('gpt-3.5-turbo')) {
+    if (/-(?!0314)\d{4}/.test(input)) {
+      return input;
+    } else if (input.includes('gpt-3.5-turbo')) {
       return 'gpt-3.5-turbo';
     } else if (input.includes('gpt-4')) {
       return 'gpt-4';
@@ -86,17 +83,15 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       initialMessageCount: this.currentMessages.length + 1,
     });
 
-    if (this.options.debug) {
-      console.debug(
-        `<-----Agent Model: ${model.modelName} | Temp: ${model.temperature} | Functions: ${this.functionsAgent}----->`,
-      );
-    }
+    logger.debug(
+      `[PluginsClient] Agent Model: ${model.modelName} | Temp: ${model.temperature} | Functions: ${this.functionsAgent}`,
+    );
 
     // Map Messages to Langchain format
     const pastMessages = formatLangChainMessages(this.currentMessages.slice(0, -1), {
       userName: this.options?.name,
     });
-    this.options.debug && console.debug('pastMessages: ', pastMessages);
+    logger.debug('[PluginsClient] pastMessages: ' + pastMessages.length);
 
     // TODO: use readOnly memory, TokenBufferMemory? (both unavailable in LangChainJS)
     const memory = new BufferMemory({
@@ -114,7 +109,8 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
         signal: this.abortController.signal,
         openAIApiKey: this.openAIApiKey,
         conversationId: this.conversationId,
-        debug: this.options?.debug,
+        fileStrategy: this.options.req.app.locals.fileStrategy,
+        processFileURL,
         message,
       },
     });
@@ -125,19 +121,16 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       return;
     }
 
-    if (this.options.debug) {
-      console.debug('Requested Tools');
-      console.debug(this.options.tools);
-      console.debug('Loaded Tools');
-      console.debug(this.tools.map((tool) => tool.name));
-    }
+    logger.debug('[PluginsClient] Requested Tools', this.options.tools);
+    logger.debug(
+      '[PluginsClient] Loaded Tools',
+      this.tools.map((tool) => tool.name),
+    );
 
     const handleAction = (action, runId, callback = null) => {
       this.saveLatestAction(action);
 
-      if (this.options.debug) {
-        console.debug('Latest Agent Action ', this.actions[this.actions.length - 1]);
-      }
+      logger.debug('[PluginsClient] Latest Agent Action ', this.actions[this.actions.length - 1]);
 
       if (typeof callback === 'function') {
         callback(action, runId);
@@ -166,9 +159,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       }),
     });
 
-    if (this.options.debug) {
-      console.debug('Loaded agent.');
-    }
+    logger.debug('[PluginsClient] Loaded agent.');
   }
 
   async executorCall(message, { signal, stream, onToolStart, onToolEnd }) {
@@ -184,12 +175,10 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       });
       const input = attempts > 1 ? errorInput : message;
 
-      if (this.options.debug) {
-        console.debug(`Attempt ${attempts} of ${maxAttempts}`);
-      }
+      logger.debug(`[PluginsClient] Attempt ${attempts} of ${maxAttempts}`);
 
-      if (this.options.debug && errorMessage.length > 0) {
-        console.debug('Caught error, input:', input);
+      if (errorMessage.length > 0) {
+        logger.debug('[PluginsClient] Caught error, input: ' + JSON.stringify(input));
       }
 
       try {
@@ -212,10 +201,10 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
         ]);
         break; // Exit the loop if the function call is successful
       } catch (err) {
-        console.error(err);
+        logger.error('[PluginsClient] executorCall error:', err);
         if (attempts === maxAttempts) {
           const { run } = this.runManager.getRunByConversationId(this.conversationId);
-          const defaultOutput = `Encountered an error while attempting to respond. Error: ${err.message}`;
+          const defaultOutput = `Encountered an error while attempting to respond: ${err.message}`;
           this.result.output = run && run.error ? run.error : defaultOutput;
           this.result.errorMessage = run && run.error ? run.error : err.message;
           this.result.intermediateSteps = this.actions;
@@ -227,8 +216,11 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
 
   async handleResponseMessage(responseMessage, saveOptions, user) {
     const { output, errorMessage, ...result } = this.result;
-    this.options.debug &&
-      console.debug('[handleResponseMessage] Output:', { output, errorMessage, ...result });
+    logger.debug('[PluginsClient][handleResponseMessage] Output:', {
+      output,
+      errorMessage,
+      ...result,
+    });
     const { error } = responseMessage;
     if (!error) {
       responseMessage.tokenCount = this.getTokenCountForResponse(responseMessage);
@@ -252,7 +244,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       this.setOptions(opts);
       return super.sendMessage(message, opts);
     }
-    this.options.debug && console.log('Plugins sendMessage', message, opts);
+    logger.debug('[PluginsClient] sendMessage', { message, opts });
     const {
       user,
       isEdited,
@@ -282,10 +274,10 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
     );
 
     if (tokenCountMap) {
-      console.dir(tokenCountMap, { depth: null });
+      logger.debug('[PluginsClient] tokenCountMap', { tokenCountMap });
       if (tokenCountMap[userMessage.messageId]) {
         userMessage.tokenCount = tokenCountMap[userMessage.messageId];
-        console.log('userMessage.tokenCount', userMessage.tokenCount);
+        logger.debug('[PluginsClient] userMessage.tokenCount', userMessage.tokenCount);
       }
       this.handleTokenCountMap(tokenCountMap);
     }
@@ -306,6 +298,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
           amount: promptTokens,
           debug: this.options.debug,
           model: this.modelOptions.model,
+          endpoint: EModelEndpoint.openAI,
         },
       });
     }
@@ -370,10 +363,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       return await this.handleResponseMessage(responseMessage, saveOptions, user);
     }
 
-    if (this.options.debug) {
-      console.debug('Plugins completion phase: this.result');
-      console.debug(this.result);
-    }
+    logger.debug('[PluginsClient] Completion phase: this.result', this.result);
 
     const promptPrefix = buildPromptPrefix({
       result: this.result,
@@ -381,28 +371,20 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
       functionsAgent: this.functionsAgent,
     });
 
-    if (this.options.debug) {
-      console.debug('Plugins: promptPrefix');
-      console.debug(promptPrefix);
-    }
+    logger.debug('[PluginsClient]', { promptPrefix });
 
     payload = await this.buildCompletionPrompt({
       messages: this.currentMessages,
       promptPrefix,
     });
 
-    if (this.options.debug) {
-      console.debug('buildCompletionPrompt Payload');
-      console.debug(payload);
-    }
+    logger.debug('[PluginsClient] buildCompletionPrompt Payload', payload);
     responseMessage.text = await this.sendCompletion(payload, opts);
     return await this.handleResponseMessage(responseMessage, saveOptions, user);
   }
 
   async buildCompletionPrompt({ messages, promptPrefix: _promptPrefix }) {
-    if (this.options.debug) {
-      console.debug('buildCompletionPrompt messages', messages);
-    }
+    logger.debug('[PluginsClient] buildCompletionPrompt messages', messages);
 
     const orderedMessages = messages;
     let promptPrefix = _promptPrefix.trim();

api/app/clients/TextStream.js

@@ -1,4 +1,5 @@
 const { Readable } = require('stream');
+const { logger } = require('~/config');
 
 class TextStream extends Readable {
   constructor(text, options = {}) {
@@ -38,7 +39,7 @@ class TextStream extends Readable {
       });
 
       this.on('end', () => {
-        // console.log('Stream ended');
+        // logger.debug('[processTextStream] Stream ended');
         resolve();
       });
 
@@ -50,7 +51,7 @@ class TextStream extends Readable {
     try {
       await streamPromise;
     } catch (err) {
-      console.error('Error processing text stream:', err);
+      logger.error('[processTextStream] Error in text stream:', err);
       // Handle the error appropriately, e.g., return an error message or throw an error
     }
   }

api/app/clients/agents/CustomAgent/outputParser.js

@@ -1,4 +1,5 @@
 const { ZeroShotAgentOutputParser } = require('langchain/agents');
+const { logger } = require('~/config');
 
 class CustomOutputParser extends ZeroShotAgentOutputParser {
   constructor(fields) {
@@ -64,9 +65,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
     const match = this.actionValues.exec(text); // old v2
 
     if (!match) {
-      console.log(
-        '\n\n<----------------------HIT NO MATCH PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT NO MATCH PARSING ERROR---------------------->\n\n' +
+          match,
       );
       const thoughts = text.replace(/[tT]hought:/, '').split('\n');
       // return {
@@ -84,9 +85,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
     let selectedTool = match?.[1].trim().toLowerCase();
 
     if (match && selectedTool === 'n/a') {
-      console.log(
-        '\n\n<----------------------HIT N/A PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT N/A PARSING ERROR---------------------->\n\n' +
+          match,
       );
       return {
         tool: 'self-reflection',
@@ -97,25 +98,25 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
 
     let toolIsValid = this.checkIfValidTool(selectedTool);
     if (match && !toolIsValid) {
-      console.log(
-        '\n\n<----------------Tool invalid: Re-assigning Selected Tool---------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------[CustomOutputParser] Tool invalid: Re-assigning Selected Tool---------------->\n\n' +
+          match,
       );
       selectedTool = this.getValidTool(selectedTool);
     }
 
     if (match && !selectedTool) {
-      console.log(
-        '\n\n<----------------------HIT INVALID TOOL PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT INVALID TOOL PARSING ERROR---------------------->\n\n' +
+          match,
       );
       selectedTool = 'self-reflection';
     }
 
     if (match && !match[2]) {
-      console.log(
-        '\n\n<----------------------HIT NO ACTION INPUT PARSING ERROR---------------------->\n\n',
-        match,
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT NO ACTION INPUT PARSING ERROR---------------------->\n\n' +
+          match,
       );
 
       // In case there is no action input, let's double-check if there is an action input in 'text' variable
@@ -139,7 +140,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
     }
 
     if (match && selectedTool.length > this.longestToolName.length) {
-      console.log('\n\n<----------------------HIT LONG PARSING ERROR---------------------->\n\n');
+      logger.debug(
+        '\n\n<----------------------[CustomOutputParser] HIT LONG PARSING ERROR---------------------->\n\n',
+      );
 
       let action, input, thought;
       let firstIndex = Infinity;
@@ -156,9 +159,9 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
       // In case there is no action input, let's double-check if there is an action input in 'text' variable
       const actionInputMatch = this.actionInputRegex.exec(text);
       if (action && actionInputMatch) {
-        console.log(
-          '\n\n<------Matched Action Input in Long Parsing Error------>\n\n',
-          actionInputMatch,
+        logger.debug(
+          '\n\n<------[CustomOutputParser] Matched Action Input in Long Parsing Error------>\n\n' +
+            actionInputMatch,
         );
         return {
           tool: action,
@@ -185,15 +188,14 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
 
         const inputMatch = this.actionValues.exec(returnValues.log); //new
         if (inputMatch) {
-          console.log('inputMatch');
-          console.dir(inputMatch, { depth: null });
+          logger.debug('[CustomOutputParser] inputMatch', inputMatch);
           returnValues.toolInput = inputMatch[1].replaceAll('"', '').trim();
           returnValues.log = returnValues.log.replace(this.actionValues, '');
         }
 
         return returnValues;
       } else {
-        console.log('No valid tool mentioned.', this.tools, text);
+        logger.debug('[CustomOutputParser] No valid tool mentioned.', this.tools, text);
         return {
           tool: 'self-reflection',
           toolInput: 'Hypothetical actions: \n"' + text + '"\n',
@@ -202,8 +204,8 @@ class CustomOutputParser extends ZeroShotAgentOutputParser {
       }
 
       // if (action && input) {
-      //   console.log('Action:', action);
-      //   console.log('Input:', input);
+      //   logger.debug('Action:', action);
+      //   logger.debug('Input:', input);
       // }
     }
 

api/app/clients/agents/Functions/FunctionsAgent.js

@@ -7,6 +7,8 @@ const {
   SystemMessagePromptTemplate,
   HumanMessagePromptTemplate,
 } = require('langchain/prompts');
+const { logger } = require('~/config');
+
 const PREFIX = 'You are a helpful AI assistant.';
 
 function parseOutput(message) {
@@ -112,7 +114,7 @@ class FunctionsAgent extends Agent {
       valuesForLLM,
       callbackManager,
     );
-    console.log('message', message);
+    logger.debug('[FunctionsAgent] plan message', message);
     return parseOutput(message);
   }
 }

api/app/clients/callbacks/createStartHandler.js

@@ -1,7 +1,9 @@
 const { promptTokensEstimate } = require('openai-chat-tokens');
-const checkBalance = require('../../../models/checkBalance');
-const { isEnabled } = require('../../../server/utils');
-const { formatFromLangChain } = require('../prompts');
+const { EModelEndpoint, supportsBalanceCheck } = require('librechat-data-provider');
+const { formatFromLangChain } = require('~/app/clients/prompts');
+const checkBalance = require('~/models/checkBalance');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
 
 const createStartHandler = ({
   context,
@@ -15,9 +17,15 @@ const createStartHandler = ({
     const { model, functions, function_call } = invocation_params;
     const messages = _messages[0].map(formatFromLangChain);
 
-    if (manager.debug) {
-      console.log(`handleChatModelStart: ${context}`);
-      console.dir({ model, functions, function_call }, { depth: null });
+    logger.debug(`[createStartHandler] handleChatModelStart: ${context}`, {
+      model,
+      function_call,
+    });
+
+    if (context !== 'title') {
+      logger.debug(`[createStartHandler] handleChatModelStart: ${context}`, {
+        functions,
+      });
     }
 
     const payload = { messages };
@@ -34,13 +42,15 @@ const createStartHandler = ({
     }
 
     prelimPromptTokens += promptTokensEstimate(payload);
-    if (manager.debug) {
-      console.log('Prelim Prompt Tokens & Token Buffer', prelimPromptTokens, tokenBuffer);
-    }
+    logger.debug('[createStartHandler]', {
+      prelimPromptTokens,
+      tokenBuffer,
+    });
     prelimPromptTokens += tokenBuffer;
 
     try {
-      if (isEnabled(process.env.CHECK_BALANCE)) {
+      // TODO: if plugins extends to non-OpenAI models, this will need to be updated
+      if (isEnabled(process.env.CHECK_BALANCE) && supportsBalanceCheck[EModelEndpoint.openAI]) {
         const generations =
           initialMessageCount && messages.length > initialMessageCount
             ? messages.slice(initialMessageCount)
@@ -55,11 +65,12 @@ const createStartHandler = ({
             debug: manager.debug,
             generations,
             model,
+            endpoint: EModelEndpoint.openAI,
           },
         });
       }
     } catch (err) {
-      console.error(`[${context}] checkBalance error`, err);
+      logger.error(`[createStartHandler][${context}] checkBalance error`, err);
       manager.abortController.abort();
       if (context === 'summary' || context === 'plugins') {
         manager.addRun(runId, { conversationId, error: err.message });

api/app/clients/chains/runTitleChain.js

@@ -1,6 +1,7 @@
 const { z } = require('zod');
 const { langPrompt, createTitlePrompt, escapeBraces, getSnippet } = require('../prompts');
 const { createStructuredOutputChainFromZod } = require('langchain/chains/openai_functions');
+const { logger } = require('~/config');
 
 const langSchema = z.object({
   language: z.string().describe('The language of the input text (full noun, no abbreviations).'),
@@ -30,8 +31,7 @@ const runTitleChain = async ({ llm, text, convo, signal, callbacks }) => {
   try {
     snippet = getSnippet(text);
   } catch (e) {
-    console.log('Error getting snippet of text for titleChain');
-    console.log(e);
+    logger.error('[runTitleChain] Error getting snippet of text for titleChain', e);
   }
   const languageChain = createLanguageChain({ llm, callbacks });
   const titleChain = createTitleChain({ llm, callbacks, convo: escapeBraces(convo) });

api/app/clients/llm/RunManager.js

@@ -1,5 +1,6 @@
-const { createStartHandler } = require('../callbacks');
-const spendTokens = require('../../../models/spendTokens');
+const { createStartHandler } = require('~/app/clients/callbacks');
+const spendTokens = require('~/models/spendTokens');
+const { logger } = require('~/config');
 
 class RunManager {
   constructor(fields) {
@@ -35,7 +36,7 @@ class RunManager {
     if (this.runs.has(runId)) {
       this.runs.delete(runId);
     } else {
-      console.error(`Run with ID ${runId} does not exist.`);
+      logger.error(`[api/app/clients/llm/RunManager] Run with ID ${runId} does not exist.`);
     }
   }
 
@@ -57,10 +58,19 @@ class RunManager {
       {
         handleChatModelStart: createStartHandler({ ...metadata, manager: this }),
         handleLLMEnd: async (output, runId, _parentRunId) => {
-          if (this.debug) {
-            console.log(`handleLLMEnd: ${JSON.stringify(metadata)}`);
-            console.dir({ output, runId, _parentRunId }, { depth: null });
+          const { llmOutput, ..._output } = output;
+          logger.debug(`[RunManager] handleLLMEnd: ${JSON.stringify(metadata)}`, {
+            runId,
+            _parentRunId,
+            llmOutput,
+          });
+
+          if (metadata.context !== 'title') {
+            logger.debug('[RunManager] handleLLMEnd:', {
+              output: _output,
+            });
           }
+
           const { tokenUsage } = output.llmOutput;
           const run = this.getRunById(runId);
           this.removeRun(runId);
@@ -74,8 +84,7 @@ class RunManager {
           await spendTokens(txData, tokenUsage);
         },
         handleLLMError: async (err) => {
-          this.debug && console.log(`handleLLMError: ${JSON.stringify(metadata)}`);
-          this.debug && console.error(err);
+          logger.error(`[RunManager] handleLLMError: ${JSON.stringify(metadata)}`, err);
           if (metadata.context === 'title') {
             return;
           } else if (metadata.context === 'plugins') {

api/app/clients/llm/createLLM.js

@@ -1,38 +1,6 @@
 const { ChatOpenAI } = require('langchain/chat_models/openai');
-const { sanitizeModelName } = require('../../../utils');
-const { isEnabled } = require('../../../server/utils');
-
-/**
- * @typedef {Object} ModelOptions
- * @property {string} modelName - The name of the model.
- * @property {number} [temperature] - The temperature setting for the model.
- * @property {number} [presence_penalty] - The presence penalty setting.
- * @property {number} [frequency_penalty] - The frequency penalty setting.
- * @property {number} [max_tokens] - The maximum number of tokens to generate.
- */
-
-/**
- * @typedef {Object} ConfigOptions
- * @property {string} [basePath] - The base path for the API requests.
- * @property {Object} [baseOptions] - Base options for the API requests, including headers.
- * @property {Object} [httpAgent] - The HTTP agent for the request.
- * @property {Object} [httpsAgent] - The HTTPS agent for the request.
- */
-
-/**
- * @typedef {Object} Callbacks
- * @property {Function} [handleChatModelStart] - A callback function for handleChatModelStart
- * @property {Function} [handleLLMEnd] - A callback function for handleLLMEnd
- * @property {Function} [handleLLMError] - A callback function for handleLLMError
- */
-
-/**
- * @typedef {Object} AzureOptions
- * @property {string} [azureOpenAIApiKey] - The Azure OpenAI API key.
- * @property {string} [azureOpenAIApiInstanceName] - The Azure OpenAI API instance name.
- * @property {string} [azureOpenAIApiDeploymentName] - The Azure OpenAI API deployment name.
- * @property {string} [azureOpenAIApiVersion] - The Azure OpenAI API version.
- */
+const { sanitizeModelName, constructAzureURL } = require('~/utils');
+const { isEnabled } = require('~/server/utils');
 
 /**
  * Creates a new instance of a language model (LLM) for chat interactions.
@@ -68,6 +36,7 @@ function createLLM({
     apiKey: openAIApiKey,
   };
 
+  /**  @type {AzureOptions} */
   let azureOptions = {};
   if (azure) {
     const useModelName = isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME);
@@ -85,17 +54,24 @@ function createLLM({
     modelOptions.modelName = process.env.AZURE_OPENAI_DEFAULT_MODEL;
   }
 
-  // console.debug('createLLM: configOptions');
-  // console.debug(configOptions);
+  if (azure && configOptions.basePath) {
+    const azureURL = constructAzureURL({
+      baseURL: configOptions.basePath,
+      azureOptions,
+    });
+    azureOptions.azureOpenAIBasePath = azureURL.split(
+      `/${azureOptions.azureOpenAIApiDeploymentName}`,
+    )[0];
+  }
 
   return new ChatOpenAI(
     {
       streaming,
-      verbose: true,
       credentials,
       configuration,
       ...azureOptions,
       ...modelOptions,
+      ...credentials,
       callbacks,
     },
     configOptions,

api/app/clients/memory/summaryBuffer.js

@@ -1,6 +1,7 @@
 const { ConversationSummaryBufferMemory, ChatMessageHistory } = require('langchain/memory');
 const { formatLangChainMessages, SUMMARY_PROMPT } = require('../prompts');
 const { predictNewSummary } = require('../chains');
+const { logger } = require('~/config');
 
 const createSummaryBufferMemory = ({ llm, prompt, messages, ...rest }) => {
   const chatHistory = new ChatMessageHistory(messages);
@@ -22,9 +23,8 @@ const summaryBuffer = async ({
   prompt = SUMMARY_PROMPT,
   signal,
 }) => {
-  if (debug && previous_summary) {
-    console.log('<-----------PREVIOUS SUMMARY----------->\n\n');
-    console.log(previous_summary);
+  if (previous_summary) {
+    logger.debug('[summaryBuffer]', { previous_summary });
   }
 
   const formattedMessages = formatLangChainMessages(context, formatOptions);
@@ -46,8 +46,7 @@ const summaryBuffer = async ({
   const messages = await chatPromptMemory.chatHistory.getMessages();
 
   if (debug) {
-    console.log('<-----------SUMMARY BUFFER MESSAGES----------->\n\n');
-    console.log(JSON.stringify(messages));
+    logger.debug('[summaryBuffer]', { summary_buffer_messages: messages.length });
   }
 
   const predictSummary = await predictNewSummary({
@@ -58,8 +57,7 @@ const summaryBuffer = async ({
   });
 
   if (debug) {
-    console.log('<-----------SUMMARY----------->\n\n');
-    console.log(JSON.stringify(predictSummary));
+    logger.debug('[summaryBuffer]', { summary: predictSummary });
   }
 
   return { role: 'system', content: predictSummary };

api/app/clients/output_parsers/addImages.js

@@ -1,3 +1,5 @@
+const { logger } = require('~/config');
+
 /**
  * The `addImages` function corrects any erroneous image URLs in the `responseMessage.text`
  * and appends image observations from `intermediateSteps` if they are not already present.
@@ -20,7 +22,7 @@
  *
  * addImages(intermediateSteps, responseMessage);
  *
- * console.log(responseMessage.text);
+ * logger.debug(responseMessage.text);
  * // Outputs: 'Some text with ![desc](/images/test.png)\n![desc](/images/test.png)'
  *
  * @returns {void}
@@ -58,12 +60,10 @@ function addImages(intermediateSteps, responseMessage) {
     if (!observation || !observation.includes('![')) {
       return;
     }
-    const observedImagePath = observation.match(/\(\/images\/.*\.\w*\)/g);
+    const observedImagePath = observation.match(/!\[.*\]\([^)]*\)/g);
     if (observedImagePath && !responseMessage.text.includes(observedImagePath[0])) {
       responseMessage.text += '\n' + observation;
-      if (process.env.DEBUG_PLUGINS) {
-        console.debug('[addImages] added image from intermediateSteps');
-      }
+      logger.debug('[addImages] added image from intermediateSteps:', observation);
     }
   });
 }

api/app/clients/prompts/createContextHandlers.js

@@ -0,0 +1,159 @@
+const axios = require('axios');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
+
+const footer = `Use the context as your learned knowledge to better answer the user.
+
+In your response, remember to follow these guidelines:
+- If you don't know the answer, simply say that you don't know.
+- If you are unsure how to answer, ask for clarification.
+- Avoid mentioning that you obtained the information from the context.
+
+Answer appropriately in the user's language.
+`;
+
+function createContextHandlers(req, userMessageContent) {
+  if (!process.env.RAG_API_URL) {
+    return;
+  }
+
+  const queryPromises = [];
+  const processedFiles = [];
+  const processedIds = new Set();
+  const jwtToken = req.headers.authorization.split(' ')[1];
+  const useFullContext = isEnabled(process.env.RAG_USE_FULL_CONTEXT);
+
+  const query = async (file) => {
+    if (useFullContext) {
+      return axios.get(`${process.env.RAG_API_URL}/documents/${file.file_id}/context`, {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+        },
+      });
+    }
+
+    return axios.post(
+      `${process.env.RAG_API_URL}/query`,
+      {
+        file_id: file.file_id,
+        query: userMessageContent,
+        k: 4,
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+  };
+
+  const processFile = async (file) => {
+    if (file.embedded && !processedIds.has(file.file_id)) {
+      try {
+        const promise = query(file);
+        queryPromises.push(promise);
+        processedFiles.push(file);
+        processedIds.add(file.file_id);
+      } catch (error) {
+        logger.error(`Error processing file ${file.filename}:`, error);
+      }
+    }
+  };
+
+  const createContext = async () => {
+    try {
+      if (!queryPromises.length || !processedFiles.length) {
+        return '';
+      }
+
+      const oneFile = processedFiles.length === 1;
+      const header = `The user has attached ${oneFile ? 'a' : processedFiles.length} file${
+        !oneFile ? 's' : ''
+      } to the conversation:`;
+
+      const files = `${
+        oneFile
+          ? ''
+          : `
+      <files>`
+      }${processedFiles
+        .map(
+          (file) => `
+              <file>
+                <filename>${file.filename}</filename>
+                <type>${file.type}</type>
+              </file>`,
+        )
+        .join('')}${
+        oneFile
+          ? ''
+          : `
+        </files>`
+      }`;
+
+      const resolvedQueries = await Promise.all(queryPromises);
+
+      const context = resolvedQueries
+        .map((queryResult, index) => {
+          const file = processedFiles[index];
+          let contextItems = queryResult.data;
+
+          const generateContext = (currentContext) =>
+            `
+          <file>
+            <filename>${file.filename}</filename>
+            <context>${currentContext}
+            </context>
+          </file>`;
+
+          if (useFullContext) {
+            return generateContext(`\n${contextItems}`);
+          }
+
+          contextItems = queryResult.data
+            .map((item) => {
+              const pageContent = item[0].page_content;
+              return `
+            <contextItem>
+              <![CDATA[${pageContent?.trim()}]]>
+            </contextItem>`;
+            })
+            .join('');
+
+          return generateContext(contextItems);
+        })
+        .join('');
+
+      if (useFullContext) {
+        const prompt = `${header}
+          ${context}
+          ${footer}`;
+
+        return prompt;
+      }
+
+      const prompt = `${header}
+        ${files}
+
+        A semantic search was executed with the user's message as the query, retrieving the following context inside <context></context> XML tags.
+
+        <context>${context}
+        </context>
+
+        ${footer}`;
+
+      return prompt;
+    } catch (error) {
+      logger.error('Error creating context:', error);
+      throw error;
+    }
+  };
+
+  return {
+    processFile,
+    createContext,
+  };
+}
+
+module.exports = createContextHandlers;

api/app/clients/prompts/createVisionPrompt.js

@@ -0,0 +1,34 @@
+/**
+ * Generates a prompt instructing the user to describe an image in detail, tailored to different types of visual content.
+ * @param {boolean} pluralized - Whether to pluralize the prompt for multiple images.
+ * @returns {string} - The generated vision prompt.
+ */
+const createVisionPrompt = (pluralized = false) => {
+  return `Please describe the image${
+    pluralized ? 's' : ''
+  } in detail, covering relevant aspects such as:
+
+  For photographs, illustrations, or artwork:
+  - The main subject(s) and their appearance, positioning, and actions
+  - The setting, background, and any notable objects or elements
+  - Colors, lighting, and overall mood or atmosphere
+  - Any interesting details, textures, or patterns
+  - The style, technique, or medium used (if discernible)
+  
+  For screenshots or images containing text:
+  - The content and purpose of the text
+  - The layout, formatting, and organization of the information
+  - Any notable visual elements, such as logos, icons, or graphics
+  - The overall context or message conveyed by the screenshot
+  
+  For graphs, charts, or data visualizations:
+  - The type of graph or chart (e.g., bar graph, line chart, pie chart)
+  - The variables being compared or analyzed
+  - Any trends, patterns, or outliers in the data
+  - The axis labels, scales, and units of measurement
+  - The title, legend, and any additional context provided
+  
+  Be as specific and descriptive as possible while maintaining clarity and concision.`;
+};
+
+module.exports = createVisionPrompt;

api/app/clients/prompts/formatGoogleInputs.js

@@ -0,0 +1,42 @@
+/**
+ * Formats an object to match the struct_val, list_val, string_val, float_val, and int_val format.
+ *
+ * @param {Object} obj - The object to be formatted.
+ * @returns {Object} The formatted object.
+ *
+ * Handles different types:
+ * - Arrays are wrapped in list_val and each element is processed.
+ * - Objects are recursively processed.
+ * - Strings are wrapped in string_val.
+ * - Numbers are wrapped in float_val or int_val depending on whether they are floating-point or integers.
+ */
+function formatGoogleInputs(obj) {
+  const formattedObj = {};
+
+  for (const key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      const value = obj[key];
+
+      // Handle arrays
+      if (Array.isArray(value)) {
+        formattedObj[key] = { list_val: value.map((item) => formatGoogleInputs(item)) };
+      }
+      // Handle objects
+      else if (typeof value === 'object' && value !== null) {
+        formattedObj[key] = formatGoogleInputs(value);
+      }
+      // Handle numbers
+      else if (typeof value === 'number') {
+        formattedObj[key] = Number.isInteger(value) ? { int_val: value } : { float_val: value };
+      }
+      // Handle other types (e.g., strings)
+      else {
+        formattedObj[key] = { string_val: [value] };
+      }
+    }
+  }
+
+  return { struct_val: formattedObj };
+}
+
+module.exports = formatGoogleInputs;

api/app/clients/prompts/formatGoogleInputs.spec.js

@@ -0,0 +1,274 @@
+const formatGoogleInputs = require('./formatGoogleInputs');
+
+describe('formatGoogleInputs', () => {
+  it('formats message correctly', () => {
+    const input = {
+      messages: [
+        {
+          content: 'hi',
+          author: 'user',
+        },
+      ],
+      context: 'context',
+      examples: [
+        {
+          input: {
+            author: 'user',
+            content: 'user input',
+          },
+          output: {
+            author: 'bot',
+            content: 'bot output',
+          },
+        },
+      ],
+      parameters: {
+        temperature: 0.2,
+        topP: 0.8,
+        topK: 40,
+        maxOutputTokens: 1024,
+      },
+    };
+
+    const expectedOutput = {
+      struct_val: {
+        messages: {
+          list_val: [
+            {
+              struct_val: {
+                content: {
+                  string_val: ['hi'],
+                },
+                author: {
+                  string_val: ['user'],
+                },
+              },
+            },
+          ],
+        },
+        context: {
+          string_val: ['context'],
+        },
+        examples: {
+          list_val: [
+            {
+              struct_val: {
+                input: {
+                  struct_val: {
+                    author: {
+                      string_val: ['user'],
+                    },
+                    content: {
+                      string_val: ['user input'],
+                    },
+                  },
+                },
+                output: {
+                  struct_val: {
+                    author: {
+                      string_val: ['bot'],
+                    },
+                    content: {
+                      string_val: ['bot output'],
+                    },
+                  },
+                },
+              },
+            },
+          ],
+        },
+        parameters: {
+          struct_val: {
+            temperature: {
+              float_val: 0.2,
+            },
+            topP: {
+              float_val: 0.8,
+            },
+            topK: {
+              int_val: 40,
+            },
+            maxOutputTokens: {
+              int_val: 1024,
+            },
+          },
+        },
+      },
+    };
+
+    const result = formatGoogleInputs(input);
+    expect(JSON.stringify(result)).toEqual(JSON.stringify(expectedOutput));
+  });
+
+  it('formats real payload parts', () => {
+    const input = {
+      instances: [
+        {
+          context: 'context',
+          examples: [
+            {
+              input: {
+                author: 'user',
+                content: 'user input',
+              },
+              output: {
+                author: 'bot',
+                content: 'user output',
+              },
+            },
+          ],
+          messages: [
+            {
+              author: 'user',
+              content: 'hi',
+            },
+          ],
+        },
+      ],
+      parameters: {
+        candidateCount: 1,
+        maxOutputTokens: 1024,
+        temperature: 0.2,
+        topP: 0.8,
+        topK: 40,
+      },
+    };
+    const expectedOutput = {
+      struct_val: {
+        instances: {
+          list_val: [
+            {
+              struct_val: {
+                context: { string_val: ['context'] },
+                examples: {
+                  list_val: [
+                    {
+                      struct_val: {
+                        input: {
+                          struct_val: {
+                            author: { string_val: ['user'] },
+                            content: { string_val: ['user input'] },
+                          },
+                        },
+                        output: {
+                          struct_val: {
+                            author: { string_val: ['bot'] },
+                            content: { string_val: ['user output'] },
+                          },
+                        },
+                      },
+                    },
+                  ],
+                },
+                messages: {
+                  list_val: [
+                    {
+                      struct_val: {
+                        author: { string_val: ['user'] },
+                        content: { string_val: ['hi'] },
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          ],
+        },
+        parameters: {
+          struct_val: {
+            candidateCount: { int_val: 1 },
+            maxOutputTokens: { int_val: 1024 },
+            temperature: { float_val: 0.2 },
+            topP: { float_val: 0.8 },
+            topK: { int_val: 40 },
+          },
+        },
+      },
+    };
+
+    const result = formatGoogleInputs(input);
+    expect(JSON.stringify(result)).toEqual(JSON.stringify(expectedOutput));
+  });
+
+  it('helps create valid payload parts', () => {
+    const instances = {
+      context: 'context',
+      examples: [
+        {
+          input: {
+            author: 'user',
+            content: 'user input',
+          },
+          output: {
+            author: 'bot',
+            content: 'user output',
+          },
+        },
+      ],
+      messages: [
+        {
+          author: 'user',
+          content: 'hi',
+        },
+      ],
+    };
+
+    const expectedInstances = {
+      struct_val: {
+        context: { string_val: ['context'] },
+        examples: {
+          list_val: [
+            {
+              struct_val: {
+                input: {
+                  struct_val: {
+                    author: { string_val: ['user'] },
+                    content: { string_val: ['user input'] },
+                  },
+                },
+                output: {
+                  struct_val: {
+                    author: { string_val: ['bot'] },
+                    content: { string_val: ['user output'] },
+                  },
+                },
+              },
+            },
+          ],
+        },
+        messages: {
+          list_val: [
+            {
+              struct_val: {
+                author: { string_val: ['user'] },
+                content: { string_val: ['hi'] },
+              },
+            },
+          ],
+        },
+      },
+    };
+
+    const parameters = {
+      candidateCount: 1,
+      maxOutputTokens: 1024,
+      temperature: 0.2,
+      topP: 0.8,
+      topK: 40,
+    };
+    const expectedParameters = {
+      struct_val: {
+        candidateCount: { int_val: 1 },
+        maxOutputTokens: { int_val: 1024 },
+        temperature: { float_val: 0.2 },
+        topP: { float_val: 0.8 },
+        topK: { int_val: 40 },
+      },
+    };
+
+    const instancesResult = formatGoogleInputs(instances);
+    const parametersResult = formatGoogleInputs(parameters);
+    expect(JSON.stringify(instancesResult)).toEqual(JSON.stringify(expectedInstances));
+    expect(JSON.stringify(parametersResult)).toEqual(JSON.stringify(expectedParameters));
+  });
+});

api/app/clients/prompts/formatMessages.js

@@ -1,5 +1,28 @@
+const { EModelEndpoint } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
 
+/**
+ * Formats a message to OpenAI Vision API payload format.
+ *
+ * @param {Object} params - The parameters for formatting.
+ * @param {Object} params.message - The message object to format.
+ * @param {string} [params.message.role] - The role of the message sender (must be 'user').
+ * @param {string} [params.message.content] - The text content of the message.
+ * @param {EModelEndpoint} [params.endpoint] - Identifier for specific endpoint handling
+ * @param {Array<string>} [params.image_urls] - The image_urls to attach to the message.
+ * @returns {(Object)} - The formatted message.
+ */
+const formatVisionMessage = ({ message, image_urls, endpoint }) => {
+  if (endpoint === EModelEndpoint.anthropic) {
+    message.content = [...image_urls, { type: 'text', text: message.content }];
+    return message;
+  }
+
+  message.content = [{ type: 'text', text: message.content }, ...image_urls];
+
+  return message;
+};
+
 /**
  * Formats a message to OpenAI payload format based on the provided options.
  *
@@ -10,12 +33,14 @@ const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
  * @param {string} [params.message.sender] - The sender of the message.
  * @param {string} [params.message.text] - The text content of the message.
  * @param {string} [params.message.content] - The content of the message.
+ * @param {Array<string>} [params.message.image_urls] - The image_urls attached to the message for Vision API.
  * @param {string} [params.userName] - The name of the user.
  * @param {string} [params.assistantName] - The name of the assistant.
+ * @param {string} [params.endpoint] - Identifier for specific endpoint handling
  * @param {boolean} [params.langChain=false] - Whether to return a LangChain message object.
  * @returns {(Object|HumanMessage|AIMessage|SystemMessage)} - The formatted message.
  */
-const formatMessage = ({ message, userName, assistantName, langChain = false }) => {
+const formatMessage = ({ message, userName, assistantName, endpoint, langChain = false }) => {
   let { role: _role, _name, sender, text, content: _content, lc_id } = message;
   if (lc_id && lc_id[2] && !langChain) {
     const roleMapping = {
@@ -32,6 +57,15 @@ const formatMessage = ({ message, userName, assistantName, langChain = false })
     content,
   };
 
+  const { image_urls } = message;
+  if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
+    return formatVisionMessage({
+      message: formattedMessage,
+      image_urls: message.image_urls,
+      endpoint,
+    });
+  }
+
   if (_name) {
     formattedMessage.name = _name;
   }

api/app/clients/prompts/formatMessages.spec.js

@@ -1,5 +1,6 @@
-const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
+const { Constants } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
+const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
 
 describe('formatMessage', () => {
   it('formats user message', () => {
@@ -54,7 +55,6 @@ describe('formatMessage', () => {
         _id: '6512cdfb92cbf69fea615331',
         messageId: 'b620bf73-c5c3-4a38-b724-76886aac24c4',
         __v: 0,
-        cancelled: false,
         conversationId: '5c23d24f-941f-4aab-85df-127b596c8aa5',
         createdAt: Date.now(),
         error: false,
@@ -62,7 +62,7 @@ describe('formatMessage', () => {
         isCreatedByUser: true,
         isEdited: false,
         model: null,
-        parentMessageId: '00000000-0000-0000-0000-000000000000',
+        parentMessageId: Constants.NO_PARENT,
         sender: 'User',
         text: 'hi',
         tokenCount: 5,

api/app/clients/prompts/index.js

@@ -4,6 +4,8 @@ const handleInputs = require('./handleInputs');
 const instructions = require('./instructions');
 const titlePrompts = require('./titlePrompts');
 const truncateText = require('./truncateText');
+const createVisionPrompt = require('./createVisionPrompt');
+const createContextHandlers = require('./createContextHandlers');
 
 module.exports = {
   ...formatMessages,
@@ -12,4 +14,6 @@ module.exports = {
   ...instructions,
   ...titlePrompts,
   truncateText,
+  createVisionPrompt,
+  createContextHandlers,
 };

api/app/clients/prompts/titlePrompts.js

@@ -27,7 +27,60 @@ ${convo}`,
   return titlePrompt;
 };
 
+const titleFunctionPrompt = `In this environment you have access to a set of tools you can use to generate the conversation title.
+  
+You may call them like this:
+<function_calls>
+<invoke>
+<tool_name>$TOOL_NAME</tool_name>
+<parameters>
+<$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME>
+...
+</parameters>
+</invoke>
+</function_calls>
+
+Here are the tools available:
+<tools>
+<tool_description>
+<tool_name>submit_title</tool_name>
+<description>
+Submit a brief title in the conversation's language, following the parameter description closely.
+</description>
+<parameters>
+<parameter>
+<name>title</name>
+<type>string</type>
+<description>A concise, 5-word-or-less title for the conversation, using its same language, with no punctuation. Apply title case conventions appropriate for the language. For English, use AP Stylebook Title Case. Never directly mention the language name or the word "title"</description>
+</parameter>
+</parameters>
+</tool_description>
+</tools>`;
+
+/**
+ * Parses titles from title functions based on the provided prompt.
+ * @param {string} prompt - The prompt containing the title function.
+ * @returns {string} The parsed title. "New Chat" if no title is found.
+ */
+function parseTitleFromPrompt(prompt) {
+  const titleRegex = /<title>(.+?)<\/title>/;
+  const titleMatch = prompt.match(titleRegex);
+
+  if (titleMatch && titleMatch[1]) {
+    const title = titleMatch[1].trim();
+
+    // // Capitalize the first letter of each word; Note: unnecessary due to title case prompting
+    // const capitalizedTitle = title.replace(/\b\w/g, (char) => char.toUpperCase());
+
+    return title;
+  }
+
+  return 'New Chat';
+}
+
 module.exports = {
   langPrompt,
   createTitlePrompt,
+  titleFunctionPrompt,
+  parseTitleFromPrompt,
 };

api/app/clients/specs/BaseClient.test.js

@@ -1,19 +1,34 @@
+const { Constants } = require('librechat-data-provider');
 const { initializeFakeClient } = require('./FakeClient');
 
 jest.mock('../../../lib/db/connectDb');
-jest.mock('../../../models', () => {
-  return function () {
-    return {
-      save: jest.fn(),
-      deleteConvos: jest.fn(),
-      getConvo: jest.fn(),
-      getMessages: jest.fn(),
-      saveMessage: jest.fn(),
-      updateMessage: jest.fn(),
-      saveConvo: jest.fn(),
-    };
-  };
-});
+jest.mock('~/models', () => ({
+  User: jest.fn(),
+  Key: jest.fn(),
+  Session: jest.fn(),
+  Balance: jest.fn(),
+  Transaction: jest.fn(),
+  getMessages: jest.fn().mockResolvedValue([]),
+  saveMessage: jest.fn(),
+  updateMessage: jest.fn(),
+  deleteMessagesSince: jest.fn(),
+  deleteMessages: jest.fn(),
+  getConvoTitle: jest.fn(),
+  getConvo: jest.fn(),
+  saveConvo: jest.fn(),
+  deleteConvos: jest.fn(),
+  getPreset: jest.fn(),
+  getPresets: jest.fn(),
+  savePreset: jest.fn(),
+  deletePresets: jest.fn(),
+  findFileById: jest.fn(),
+  createFile: jest.fn(),
+  updateFile: jest.fn(),
+  deleteFile: jest.fn(),
+  deleteFiles: jest.fn(),
+  getFiles: jest.fn(),
+  updateFileUsage: jest.fn(),
+}));
 
 jest.mock('langchain/chat_models/openai', () => {
   return {
@@ -293,7 +308,7 @@ describe('BaseClient', () => {
     const unorderedMessages = [
       { id: '3', parentMessageId: '2', text: 'Message 3' },
       { id: '2', parentMessageId: '1', text: 'Message 2' },
-      { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+      { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
     ];
 
     it('should return ordered messages based on parentMessageId', () => {
@@ -302,7 +317,7 @@ describe('BaseClient', () => {
         parentMessageId: '3',
       });
       expect(result).toEqual([
-        { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+        { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
         { id: '2', parentMessageId: '1', text: 'Message 2' },
         { id: '3', parentMessageId: '2', text: 'Message 3' },
       ]);
@@ -529,9 +544,9 @@ describe('BaseClient', () => {
       );
     });
 
-    test('setOptions is called with the correct arguments', async () => {
+    test('setOptions is called with the correct arguments only when replaceOptions is set to true', async () => {
       TestClient.setOptions = jest.fn();
-      const opts = { conversationId: '123', parentMessageId: '456' };
+      const opts = { conversationId: '123', parentMessageId: '456', replaceOptions: true };
       await TestClient.sendMessage('Hello, world!', opts);
       expect(TestClient.setOptions).toHaveBeenCalledWith(opts);
       TestClient.setOptions.mockClear();

api/app/clients/specs/FakeClient.js

@@ -42,7 +42,6 @@ class FakeClient extends BaseClient {
 
     this.maxContextTokens = getModelMaxTokens(this.modelOptions.model) ?? 4097;
   }
-  getCompletion() {}
   buildMessages() {}
   getTokenCount(str) {
     return str.length;
@@ -86,6 +85,19 @@ const initializeFakeClient = (apiKey, options, fakeMessages) => {
     return 'Mock response text';
   });
 
+  // eslint-disable-next-line no-unused-vars
+  TestClient.getCompletion = jest.fn().mockImplementation(async (..._args) => {
+    return {
+      choices: [
+        {
+          message: {
+            content: 'Mock response text',
+          },
+        },
+      ],
+    };
+  });
+
   TestClient.buildMessages = jest.fn(async (messages, parentMessageId) => {
     const orderedMessages = TestClient.constructor.getMessagesForConversation({
       messages,

api/app/clients/specs/OpenAIClient.test.js

@@ -1,8 +1,138 @@
 require('dotenv').config();
+const OpenAI = require('openai');
+const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
+const { genAzureChatCompletion } = require('~/utils/azureUtils');
 const OpenAIClient = require('../OpenAIClient');
-
 jest.mock('meilisearch');
 
+jest.mock('~/lib/db/connectDb');
+jest.mock('~/models', () => ({
+  User: jest.fn(),
+  Key: jest.fn(),
+  Session: jest.fn(),
+  Balance: jest.fn(),
+  Transaction: jest.fn(),
+  getMessages: jest.fn().mockResolvedValue([]),
+  saveMessage: jest.fn(),
+  updateMessage: jest.fn(),
+  deleteMessagesSince: jest.fn(),
+  deleteMessages: jest.fn(),
+  getConvoTitle: jest.fn(),
+  getConvo: jest.fn(),
+  saveConvo: jest.fn(),
+  deleteConvos: jest.fn(),
+  getPreset: jest.fn(),
+  getPresets: jest.fn(),
+  savePreset: jest.fn(),
+  deletePresets: jest.fn(),
+  findFileById: jest.fn(),
+  createFile: jest.fn(),
+  updateFile: jest.fn(),
+  deleteFile: jest.fn(),
+  deleteFiles: jest.fn(),
+  getFiles: jest.fn(),
+  updateFileUsage: jest.fn(),
+}));
+
+jest.mock('langchain/chat_models/openai', () => {
+  return {
+    ChatOpenAI: jest.fn().mockImplementation(() => {
+      return {};
+    }),
+  };
+});
+
+jest.mock('openai');
+
+jest.spyOn(OpenAI, 'constructor').mockImplementation(function (...options) {
+  // We can add additional logic here if needed
+  return new OpenAI(...options);
+});
+
+const finalChatCompletion = jest.fn().mockResolvedValue({
+  choices: [
+    {
+      message: { role: 'assistant', content: 'Mock message content' },
+      finish_reason: 'Mock finish reason',
+    },
+  ],
+});
+
+const stream = jest.fn().mockImplementation(() => {
+  let isDone = false;
+  let isError = false;
+  let errorCallback = null;
+
+  const onEventHandlers = {
+    abort: () => {
+      // Mock abort behavior
+    },
+    error: (callback) => {
+      errorCallback = callback; // Save the error callback for later use
+    },
+    finalMessage: (callback) => {
+      callback({ role: 'assistant', content: 'Mock Response' });
+      isDone = true; // Set stream to done
+    },
+  };
+
+  const mockStream = {
+    on: jest.fn((event, callback) => {
+      if (onEventHandlers[event]) {
+        onEventHandlers[event](callback);
+      }
+      return mockStream;
+    }),
+    finalChatCompletion,
+    controller: { abort: jest.fn() },
+    triggerError: () => {
+      isError = true;
+      if (errorCallback) {
+        errorCallback(new Error('Mock error'));
+      }
+    },
+    [Symbol.asyncIterator]: () => {
+      return {
+        next: () => {
+          if (isError) {
+            return Promise.reject(new Error('Mock error'));
+          }
+          if (isDone) {
+            return Promise.resolve({ done: true });
+          }
+          const chunk = { choices: [{ delta: { content: 'Mock chunk' } }] };
+          return Promise.resolve({ value: chunk, done: false });
+        },
+      };
+    },
+  };
+  return mockStream;
+});
+
+const create = jest.fn().mockResolvedValue({
+  choices: [
+    {
+      message: { content: 'Mock message content' },
+      finish_reason: 'Mock finish reason',
+    },
+  ],
+});
+
+OpenAI.mockImplementation(() => ({
+  beta: {
+    chat: {
+      completions: {
+        stream,
+      },
+    },
+  },
+  chat: {
+    completions: {
+      create,
+    },
+  },
+}));
+
 describe('OpenAIClient', () => {
   let client, client2;
   const model = 'gpt-4';
@@ -12,6 +142,21 @@ describe('OpenAIClient', () => {
     { role: 'assistant', sender: 'Assistant', text: 'Hi', messageId: '2' },
   ];
 
+  const defaultOptions = {
+    // debug: true,
+    openaiApiKey: 'new-api-key',
+    modelOptions: {
+      model,
+      temperature: 0.7,
+    },
+  };
+
+  const defaultAzureOptions = {
+    azureOpenAIApiInstanceName: 'your-instance-name',
+    azureOpenAIApiDeploymentName: 'your-deployment-name',
+    azureOpenAIApiVersion: '2020-07-01-preview',
+  };
+
   beforeAll(() => {
     jest.spyOn(console, 'warn').mockImplementation(() => {});
   });
@@ -21,14 +166,7 @@ describe('OpenAIClient', () => {
   });
 
   beforeEach(() => {
-    const options = {
-      // debug: true,
-      openaiApiKey: 'new-api-key',
-      modelOptions: {
-        model,
-        temperature: 0.7,
-      },
-    };
+    const options = { ...defaultOptions };
     client = new OpenAIClient('test-api-key', options);
     client2 = new OpenAIClient('test-api-key', options);
     client.summarizeMessages = jest.fn().mockResolvedValue({
@@ -40,6 +178,7 @@ describe('OpenAIClient', () => {
       .fn()
       .mockResolvedValue({ prompt: messages.map((m) => m.text).join('\n') });
     client.constructor.freeAndResetAllEncoders();
+    client.getMessages = jest.fn().mockResolvedValue([]);
   });
 
   describe('setOptions', () => {
@@ -94,7 +233,7 @@ describe('OpenAIClient', () => {
 
       client.setOptions({ reverseProxyUrl: 'https://example.com/completions' });
       expect(client.completionsUrl).toBe('https://example.com/completions');
-      expect(client.langchainProxy).toBe(null);
+      expect(client.langchainProxy).toBe('https://example.com/completions');
     });
   });
 
@@ -407,5 +546,120 @@ describe('OpenAIClient', () => {
         expect(totalTokens).toBe(testCase.expected);
       });
     });
+
+    const vision_request = [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'describe what is in this image?',
+          },
+          {
+            type: 'image_url',
+            image_url: {
+              url: 'https://venturebeat.com/wp-content/uploads/2019/03/openai-1.png',
+              detail: 'high',
+            },
+          },
+        ],
+      },
+    ];
+
+    const expectedTokens = 14;
+    const visionModel = 'gpt-4-vision-preview';
+
+    it(`should return ${expectedTokens} tokens for model ${visionModel} (Vision Request)`, () => {
+      client.modelOptions.model = visionModel;
+      client.selectTokenizer();
+      // 3 tokens for assistant label
+      let totalTokens = 3;
+      for (let message of vision_request) {
+        totalTokens += client.getTokenCountForMessage(message);
+      }
+      expect(totalTokens).toBe(expectedTokens);
+    });
+  });
+
+  describe('sendMessage/getCompletion/chatCompletion', () => {
+    afterEach(() => {
+      delete process.env.AZURE_OPENAI_DEFAULT_MODEL;
+      delete process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME;
+      delete process.env.OPENROUTER_API_KEY;
+    });
+
+    it('should call getCompletion and fetchEventSource when using a text/instruct model', async () => {
+      const model = 'text-davinci-003';
+      const onProgress = jest.fn().mockImplementation(() => ({}));
+
+      const testClient = new OpenAIClient('test-api-key', {
+        ...defaultOptions,
+        modelOptions: { model },
+      });
+
+      const getCompletion = jest.spyOn(testClient, 'getCompletion');
+      await testClient.sendMessage('Hi mom!', { onProgress });
+
+      expect(getCompletion).toHaveBeenCalled();
+      expect(getCompletion.mock.calls.length).toBe(1);
+
+      const currentDateString = new Date().toLocaleDateString('en-us', {
+        year: 'numeric',
+        month: 'long',
+        day: 'numeric',
+      });
+
+      expect(getCompletion.mock.calls[0][0]).toBe(
+        `||>Instructions:\nYou are ChatGPT, a large language model trained by OpenAI. Respond conversationally.\nCurrent date: ${currentDateString}\n\n||>User:\nHi mom!\n||>Assistant:\n`,
+      );
+
+      expect(fetchEventSource).toHaveBeenCalled();
+      expect(fetchEventSource.mock.calls.length).toBe(1);
+
+      // Check if the first argument (url) is correct
+      const firstCallArgs = fetchEventSource.mock.calls[0];
+
+      const expectedURL = 'https://api.openai.com/v1/completions';
+      expect(firstCallArgs[0]).toBe(expectedURL);
+
+      const requestBody = JSON.parse(firstCallArgs[1].body);
+      expect(requestBody).toHaveProperty('model');
+      expect(requestBody.model).toBe(model);
+    });
+
+    it('[Azure OpenAI] should call chatCompletion and OpenAI.stream with correct args', async () => {
+      // Set a default model
+      process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt4-turbo';
+
+      const onProgress = jest.fn().mockImplementation(() => ({}));
+      client.azure = defaultAzureOptions;
+      const chatCompletion = jest.spyOn(client, 'chatCompletion');
+      await client.sendMessage('Hi mom!', {
+        replaceOptions: true,
+        ...defaultOptions,
+        modelOptions: { model: 'gpt4-turbo', stream: true },
+        onProgress,
+        azure: defaultAzureOptions,
+      });
+
+      expect(chatCompletion).toHaveBeenCalled();
+      expect(chatCompletion.mock.calls.length).toBe(1);
+
+      const chatCompletionArgs = chatCompletion.mock.calls[0][0];
+      const { payload } = chatCompletionArgs;
+
+      expect(payload[0].role).toBe('user');
+      expect(payload[0].content).toBe('Hi mom!');
+
+      // Azure OpenAI does not use the model property, and will error if it's passed
+      // This check ensures the model property is not present
+      const streamArgs = stream.mock.calls[0][0];
+      expect(streamArgs).not.toHaveProperty('model');
+
+      // Check if the baseURL is correct
+      const constructorArgs = OpenAI.mock.calls[0][0];
+      const expectedURL = genAzureChatCompletion(defaultAzureOptions).split('/chat')[0];
+      expect(constructorArgs.baseURL).toBe(expectedURL);
+    });
   });
 });

api/app/clients/specs/PluginsClient.test.js

@@ -1,9 +1,10 @@
+const crypto = require('crypto');
+const { Constants } = require('librechat-data-provider');
 const { HumanChatMessage, AIChatMessage } = require('langchain/schema');
 const PluginsClient = require('../PluginsClient');
-const crypto = require('crypto');
 
-jest.mock('../../../lib/db/connectDb');
-jest.mock('../../../models/Conversation', () => {
+jest.mock('~/lib/db/connectDb');
+jest.mock('~/models/Conversation', () => {
   return function () {
     return {
       save: jest.fn(),
@@ -12,6 +13,12 @@ jest.mock('../../../models/Conversation', () => {
   };
 });
 
+const defaultAzureOptions = {
+  azureOpenAIApiInstanceName: 'your-instance-name',
+  azureOpenAIApiDeploymentName: 'your-deployment-name',
+  azureOpenAIApiVersion: '2020-07-01-preview',
+};
+
 describe('PluginsClient', () => {
   let TestAgent;
   let options = {
@@ -60,7 +67,7 @@ describe('PluginsClient', () => {
         TestAgent.setOptions(opts);
       }
       const conversationId = opts.conversationId || crypto.randomUUID();
-      const parentMessageId = opts.parentMessageId || '00000000-0000-0000-0000-000000000000';
+      const parentMessageId = opts.parentMessageId || Constants.NO_PARENT;
       const userMessageId = opts.overrideParentMessageId || crypto.randomUUID();
       this.pastMessages = await TestAgent.loadHistory(
         conversationId,
@@ -144,4 +151,73 @@ describe('PluginsClient', () => {
       expect(chatMessages[0].text).toEqual(userMessage);
     });
   });
+
+  describe('getFunctionModelName', () => {
+    let client;
+
+    beforeEach(() => {
+      client = new PluginsClient('dummy_api_key');
+    });
+
+    test('should return the input when it includes a dash followed by four digits', () => {
+      expect(client.getFunctionModelName('-1234')).toBe('-1234');
+      expect(client.getFunctionModelName('gpt-4-5678-preview')).toBe('gpt-4-5678-preview');
+    });
+
+    test('should return the input for all function-capable models (`0613` models and above)', () => {
+      expect(client.getFunctionModelName('gpt-4-0613')).toBe('gpt-4-0613');
+      expect(client.getFunctionModelName('gpt-4-32k-0613')).toBe('gpt-4-32k-0613');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-0613')).toBe('gpt-3.5-turbo-0613');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-16k-0613')).toBe('gpt-3.5-turbo-16k-0613');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-1106')).toBe('gpt-3.5-turbo-1106');
+      expect(client.getFunctionModelName('gpt-4-1106-preview')).toBe('gpt-4-1106-preview');
+      expect(client.getFunctionModelName('gpt-4-1106')).toBe('gpt-4-1106');
+    });
+
+    test('should return the corresponding model if input is non-function capable (`0314` models)', () => {
+      expect(client.getFunctionModelName('gpt-4-0314')).toBe('gpt-4');
+      expect(client.getFunctionModelName('gpt-4-32k-0314')).toBe('gpt-4');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-0314')).toBe('gpt-3.5-turbo');
+      expect(client.getFunctionModelName('gpt-3.5-turbo-16k-0314')).toBe('gpt-3.5-turbo');
+    });
+
+    test('should return "gpt-3.5-turbo" when the input includes "gpt-3.5-turbo"', () => {
+      expect(client.getFunctionModelName('test gpt-3.5-turbo model')).toBe('gpt-3.5-turbo');
+    });
+
+    test('should return "gpt-4" when the input includes "gpt-4"', () => {
+      expect(client.getFunctionModelName('testing gpt-4')).toBe('gpt-4');
+    });
+
+    test('should return "gpt-3.5-turbo" for input that does not meet any specific condition', () => {
+      expect(client.getFunctionModelName('random string')).toBe('gpt-3.5-turbo');
+      expect(client.getFunctionModelName('')).toBe('gpt-3.5-turbo');
+    });
+  });
+  describe('Azure OpenAI tests specific to Plugins', () => {
+    // TODO: add more tests for Azure OpenAI integration with Plugins
+    // let client;
+    // beforeEach(() => {
+    //   client = new PluginsClient('dummy_api_key');
+    // });
+
+    test('should not call getFunctionModelName when azure options are set', () => {
+      const spy = jest.spyOn(PluginsClient.prototype, 'getFunctionModelName');
+      const model = 'gpt-4-turbo';
+
+      // note, without the azure change in PR #1766, `getFunctionModelName` is called twice
+      const testClient = new PluginsClient('dummy_api_key', {
+        agentOptions: {
+          model,
+          agent: 'functions',
+        },
+        azure: defaultAzureOptions,
+      });
+
+      expect(spy).not.toHaveBeenCalled();
+      expect(testClient.agentOptions.model).toBe(model);
+
+      spy.mockRestore();
+    });
+  });
 });

api/app/clients/tools/AIPluginTool.js

@@ -1,238 +0,0 @@
-const { Tool } = require('langchain/tools');
-const yaml = require('js-yaml');
-
-/*
-export interface AIPluginToolParams {
-  name: string;
-  description: string;
-  apiSpec: string;
-  openaiSpec: string;
-  model: BaseLanguageModel;
-}
-
-export interface PathParameter {
-  name: string;
-  description: string;
-}
-
-export interface Info {
-  title: string;
-  description: string;
-  version: string;
-}
-export interface PathMethod {
-  summary: string;
-  operationId: string;
-  parameters?: PathParameter[];
-}
-
-interface ApiSpec {
-  openapi: string;
-  info: Info;
-  paths: { [key: string]: { [key: string]: PathMethod } };
-}
-*/
-
-function isJson(str) {
-  try {
-    JSON.parse(str);
-  } catch (e) {
-    return false;
-  }
-  return true;
-}
-
-function convertJsonToYamlIfApplicable(spec) {
-  if (isJson(spec)) {
-    const jsonData = JSON.parse(spec);
-    return yaml.dump(jsonData);
-  }
-  return spec;
-}
-
-function extractShortVersion(openapiSpec) {
-  openapiSpec = convertJsonToYamlIfApplicable(openapiSpec);
-  try {
-    const fullApiSpec = yaml.load(openapiSpec);
-    const shortApiSpec = {
-      openapi: fullApiSpec.openapi,
-      info: fullApiSpec.info,
-      paths: {},
-    };
-
-    for (let path in fullApiSpec.paths) {
-      shortApiSpec.paths[path] = {};
-      for (let method in fullApiSpec.paths[path]) {
-        shortApiSpec.paths[path][method] = {
-          summary: fullApiSpec.paths[path][method].summary,
-          operationId: fullApiSpec.paths[path][method].operationId,
-          parameters: fullApiSpec.paths[path][method].parameters?.map((parameter) => ({
-            name: parameter.name,
-            description: parameter.description,
-          })),
-        };
-      }
-    }
-
-    return yaml.dump(shortApiSpec);
-  } catch (e) {
-    console.log(e);
-    return '';
-  }
-}
-function printOperationDetails(operationId, openapiSpec) {
-  openapiSpec = convertJsonToYamlIfApplicable(openapiSpec);
-  let returnText = '';
-  try {
-    let doc = yaml.load(openapiSpec);
-    let servers = doc.servers;
-    let paths = doc.paths;
-    let components = doc.components;
-
-    for (let path in paths) {
-      for (let method in paths[path]) {
-        let operation = paths[path][method];
-        if (operation.operationId === operationId) {
-          returnText += `The API request to do for operationId "${operationId}" is:\n`;
-          returnText += `Method: ${method.toUpperCase()}\n`;
-
-          let url = servers[0].url + path;
-          returnText += `Path: ${url}\n`;
-
-          returnText += 'Parameters:\n';
-          if (operation.parameters) {
-            for (let param of operation.parameters) {
-              let required = param.required ? '' : ' (optional),';
-              returnText += `- ${param.name} (${param.in},${required} ${param.schema.type}): ${param.description}\n`;
-            }
-          } else {
-            returnText += ' None\n';
-          }
-          returnText += '\n';
-
-          let responseSchema = operation.responses['200'].content['application/json'].schema;
-
-          // Check if schema is a reference
-          if (responseSchema.$ref) {
-            // Extract schema name from reference
-            let schemaName = responseSchema.$ref.split('/').pop();
-            // Look up schema in components
-            responseSchema = components.schemas[schemaName];
-          }
-
-          returnText += 'Response schema:\n';
-          returnText += '- Type: ' + responseSchema.type + '\n';
-          returnText += '- Additional properties:\n';
-          returnText += '  - Type: ' + responseSchema.additionalProperties?.type + '\n';
-          if (responseSchema.additionalProperties?.properties) {
-            returnText += '  - Properties:\n';
-            for (let prop in responseSchema.additionalProperties.properties) {
-              returnText += `    - ${prop} (${responseSchema.additionalProperties.properties[prop].type}): Description not provided in OpenAPI spec\n`;
-            }
-          }
-        }
-      }
-    }
-    if (returnText === '') {
-      returnText += `No operation with operationId "${operationId}" found.`;
-    }
-    return returnText;
-  } catch (e) {
-    console.log(e);
-    return '';
-  }
-}
-
-class AIPluginTool extends Tool {
-  /*
-  private _name: string;
-  private _description: string;
-  apiSpec: string;
-  openaiSpec: string;
-  model: BaseLanguageModel;
-  */
-
-  get name() {
-    return this._name;
-  }
-
-  get description() {
-    return this._description;
-  }
-
-  constructor(params) {
-    super();
-    this._name = params.name;
-    this._description = params.description;
-    this.apiSpec = params.apiSpec;
-    this.openaiSpec = params.openaiSpec;
-    this.model = params.model;
-  }
-
-  async _call(input) {
-    let date = new Date();
-    let fullDate = `Date: ${date.getDate()}/${
-      date.getMonth() + 1
-    }/${date.getFullYear()}, Time: ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
-    const prompt = `${fullDate}\nQuestion: ${input} \n${this.apiSpec}.`;
-    console.log(prompt);
-    const gptResponse = await this.model.predict(prompt);
-    let operationId = gptResponse.match(/operationId: (.*)/)?.[1];
-    if (!operationId) {
-      return 'No operationId found in the response';
-    }
-    if (operationId == 'No API path found to answer the question') {
-      return 'No API path found to answer the question';
-    }
-
-    let openApiData = printOperationDetails(operationId, this.openaiSpec);
-
-    return openApiData;
-  }
-
-  static async fromPluginUrl(url, model) {
-    const aiPluginRes = await fetch(url, {});
-    if (!aiPluginRes.ok) {
-      throw new Error(`Failed to fetch plugin from ${url} with status ${aiPluginRes.status}`);
-    }
-    const aiPluginJson = await aiPluginRes.json();
-    const apiUrlRes = await fetch(aiPluginJson.api.url, {});
-    if (!apiUrlRes.ok) {
-      throw new Error(
-        `Failed to fetch API spec from ${aiPluginJson.api.url} with status ${apiUrlRes.status}`,
-      );
-    }
-    const apiUrlJson = await apiUrlRes.text();
-    const shortApiSpec = extractShortVersion(apiUrlJson);
-    return new AIPluginTool({
-      name: aiPluginJson.name_for_model.toLowerCase(),
-      description: `A \`tool\` to learn the API documentation for ${aiPluginJson.name_for_model.toLowerCase()}, after which you can use 'http_request' to make the actual API call. Short description of how to use the API's results: ${
-        aiPluginJson.description_for_model
-      })`,
-      apiSpec: `
-As an AI, your task is to identify the operationId of the relevant API path based on the condensed OpenAPI specifications provided.
-
-Please note:
-
-1. Do not imagine URLs. Only use the information provided in the condensed OpenAPI specifications.
-
-2. Do not guess the operationId. Identify it strictly based on the API paths and their descriptions.
-
-Your output should only include:
-- operationId: The operationId of the relevant API path
-
-If you cannot find a suitable API path based on the OpenAPI specifications, please answer only "operationId: No API path found to answer the question".
-
-Now, based on the question above and the condensed OpenAPI specifications given below, identify the operationId:
-
-\`\`\`
-${shortApiSpec}
-\`\`\`
-`,
-      openaiSpec: apiUrlJson,
-      model: model,
-    });
-  }
-}
-
-module.exports = AIPluginTool;

api/app/clients/tools/AzureAiSearch.js

@@ -0,0 +1,98 @@
+const { z } = require('zod');
+const { StructuredTool } = require('langchain/tools');
+const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
+const { logger } = require('~/config');
+
+class AzureAISearch extends StructuredTool {
+  // Constants for default values
+  static DEFAULT_API_VERSION = '2023-11-01';
+  static DEFAULT_QUERY_TYPE = 'simple';
+  static DEFAULT_TOP = 5;
+
+  // Helper function for initializing properties
+  _initializeField(field, envVar, defaultValue) {
+    return field || process.env[envVar] || defaultValue;
+  }
+
+  constructor(fields = {}) {
+    super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
+
+    // Initialize properties using helper function
+    this.serviceEndpoint = this._initializeField(
+      fields.AZURE_AI_SEARCH_SERVICE_ENDPOINT,
+      'AZURE_AI_SEARCH_SERVICE_ENDPOINT',
+    );
+    this.indexName = this._initializeField(
+      fields.AZURE_AI_SEARCH_INDEX_NAME,
+      'AZURE_AI_SEARCH_INDEX_NAME',
+    );
+    this.apiKey = this._initializeField(fields.AZURE_AI_SEARCH_API_KEY, 'AZURE_AI_SEARCH_API_KEY');
+    this.apiVersion = this._initializeField(
+      fields.AZURE_AI_SEARCH_API_VERSION,
+      'AZURE_AI_SEARCH_API_VERSION',
+      AzureAISearch.DEFAULT_API_VERSION,
+    );
+    this.queryType = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE',
+      AzureAISearch.DEFAULT_QUERY_TYPE,
+    );
+    this.top = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_TOP,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_TOP',
+      AzureAISearch.DEFAULT_TOP,
+    );
+    this.select = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_SELECT,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_SELECT',
+    );
+
+    // Check for required fields
+    if (!this.serviceEndpoint || !this.indexName || !this.apiKey) {
+      throw new Error(
+        'Missing AZURE_AI_SEARCH_SERVICE_ENDPOINT, AZURE_AI_SEARCH_INDEX_NAME, or AZURE_AI_SEARCH_API_KEY environment variable.',
+      );
+    }
+
+    // Create SearchClient
+    this.client = new SearchClient(
+      this.serviceEndpoint,
+      this.indexName,
+      new AzureKeyCredential(this.apiKey),
+      { apiVersion: this.apiVersion },
+    );
+
+    // Define schema
+    this.schema = z.object({
+      query: z.string().describe('Search word or phrase to Azure AI Search'),
+    });
+  }
+
+  // Improved error handling and logging
+  async _call(data) {
+    const { query } = data;
+    try {
+      const searchOption = {
+        queryType: this.queryType,
+        top: this.top,
+      };
+      if (this.select) {
+        searchOption.select = this.select.split(',');
+      }
+      const searchResults = await this.client.search(query, searchOption);
+      const resultDocuments = [];
+      for await (const result of searchResults.results) {
+        resultDocuments.push(result.document);
+      }
+      return JSON.stringify(resultDocuments);
+    } catch (error) {
+      logger.error('Azure AI Search request failed', error);
+      return 'There was an error with Azure AI Search.';
+    }
+  }
+}
+
+module.exports = AzureAISearch;

api/app/clients/tools/AzureCognitiveSearch.js

@@ -1,111 +0,0 @@
-const { Tool } = require('langchain/tools');
-const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
-
-class AzureCognitiveSearch extends Tool {
-  constructor(fields = {}) {
-    super();
-    this.serviceEndpoint =
-      fields.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || this.getServiceEndpoint();
-    this.indexName = fields.AZURE_COGNITIVE_SEARCH_INDEX_NAME || this.getIndexName();
-    this.apiKey = fields.AZURE_COGNITIVE_SEARCH_API_KEY || this.getApiKey();
-
-    this.apiVersion = fields.AZURE_COGNITIVE_SEARCH_API_VERSION || this.getApiVersion();
-
-    this.queryType = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || this.getQueryType();
-    this.top = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP || this.getTop();
-    this.select = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT || this.getSelect();
-
-    this.client = new SearchClient(
-      this.serviceEndpoint,
-      this.indexName,
-      new AzureKeyCredential(this.apiKey),
-      {
-        apiVersion: this.apiVersion,
-      },
-    );
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'azure-cognitive-search';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'azure-cognitive-search\' tool to retrieve search results relevant to your input';
-
-  getServiceEndpoint() {
-    const serviceEndpoint = process.env.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || '';
-    if (!serviceEndpoint) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT environment variable.');
-    }
-    return serviceEndpoint;
-  }
-
-  getIndexName() {
-    const indexName = process.env.AZURE_COGNITIVE_SEARCH_INDEX_NAME || '';
-    if (!indexName) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_INDEX_NAME environment variable.');
-    }
-    return indexName;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.AZURE_COGNITIVE_SEARCH_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getApiVersion() {
-    return process.env.AZURE_COGNITIVE_SEARCH_API_VERSION || '2020-06-30';
-  }
-
-  getQueryType() {
-    return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || 'simple';
-  }
-
-  getTop() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP) {
-      return Number(process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP);
-    } else {
-      return 5;
-    }
-  }
-
-  getSelect() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT) {
-      return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT.split(',');
-    } else {
-      return null;
-    }
-  }
-
-  async _call(query) {
-    try {
-      const searchOption = {
-        queryType: this.queryType,
-        top: this.top,
-      };
-      if (this.select) {
-        searchOption.select = this.select;
-      }
-      const searchResults = await this.client.search(query, searchOption);
-      const resultDocuments = [];
-      for await (const result of searchResults.results) {
-        resultDocuments.push(result.document);
-      }
-      return JSON.stringify(resultDocuments);
-    } catch (error) {
-      console.error(`Azure Cognitive Search request failed: ${error}`);
-      return 'There was an error with Azure Cognitive Search.';
-    }
-  }
-}
-
-module.exports = AzureCognitiveSearch;

api/app/clients/tools/CodeInterpreter.js

@@ -1,52 +0,0 @@
-const { Tool } = require('langchain/tools');
-const WebSocket = require('ws');
-const { promisify } = require('util');
-const fs = require('fs');
-
-class CodeInterpreter extends Tool {
-  constructor() {
-    super();
-    this.name = 'code-interpreter';
-    this.description = `If there is plotting or any image related tasks, save the result as .png file. 
-    No need show the image or plot. USE print(variable_name) if you need output.You can run python codes with this plugin.You have to use print function in python code to get any result from this plugin. 
-    This does not support user input. Even if the code has input() function, change it to an appropriate value.
-    You can show the user the code with input() functions. But the code passed to the plug-in should not contain input().
-    You should provide properly formatted code to this plugin. If the code is executed successfully, the stdout will be returned to you. You have to print that to the user, and if the user had 
-    asked for an explanation, you have to provide one. If the output is "Error From here" or any other error message, 
-    tell the user "Python Engine Failed" and continue with whatever you are supposed to do.`;
-
-    // Create a promisified version of fs.unlink
-    this.unlinkAsync = promisify(fs.unlink);
-  }
-
-  async _call(input) {
-    const websocket = new WebSocket('ws://localhost:3380'); // Update with your WebSocket server URL
-
-    // Wait until the WebSocket connection is open
-    await new Promise((resolve) => {
-      websocket.onopen = resolve;
-    });
-
-    // Send the Python code to the server
-    websocket.send(input);
-
-    // Wait for the result from the server
-    const result = await new Promise((resolve) => {
-      websocket.onmessage = (event) => {
-        resolve(event.data);
-      };
-
-      // Handle WebSocket connection closed
-      websocket.onclose = () => {
-        resolve('Python Engine Failed');
-      };
-    });
-
-    // Close the WebSocket connection
-    websocket.close();
-
-    return result;
-  }
-}
-
-module.exports = CodeInterpreter;

api/app/clients/tools/DALL-E.js

@@ -1,52 +1,43 @@
-// From https://platform.openai.com/docs/api-reference/images/create
-// To use this tool, you must pass in a configured OpenAIApi object.
-const fs = require('fs');
 const path = require('path');
 const OpenAI = require('openai');
-// const { genAzureEndpoint } = require('../../../utils/genAzureEndpoints');
+const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
-const saveImageFromUrl = require('./saveImageFromUrl');
-const extractBaseURL = require('../../../utils/extractBaseURL');
-const { DALLE_REVERSE_PROXY, PROXY } = process.env;
+const { FileContext } = require('librechat-data-provider');
+const { getImageBasename } = require('~/server/services/Files/images');
+const extractBaseURL = require('~/utils/extractBaseURL');
+const { logger } = require('~/config');
 
 class OpenAICreateImage extends Tool {
   constructor(fields = {}) {
     super();
 
-    let apiKey = fields.DALLE_API_KEY || this.getApiKey();
+    this.userId = fields.userId;
+    this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
+    let apiKey = fields.DALLE2_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
+
     const config = { apiKey };
-    if (DALLE_REVERSE_PROXY) {
-      config.baseURL = extractBaseURL(DALLE_REVERSE_PROXY);
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
     }
 
-    if (PROXY) {
-      config.httpAgent = new HttpsProxyAgent(PROXY);
+    if (process.env.DALLE2_AZURE_API_VERSION && process.env.DALLE2_BASEURL) {
+      config.baseURL = process.env.DALLE2_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE2_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE2_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE2_API_KEY;
     }
 
-    // let azureKey = fields.AZURE_API_KEY || process.env.AZURE_API_KEY;
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
+    }
 
-    // if (azureKey) {
-    //   apiKey = azureKey;
-    //   const azureConfig = {
-    //     apiKey,
-    //     azureOpenAIApiInstanceName: process.env.AZURE_OPENAI_API_INSTANCE_NAME || fields.azureOpenAIApiInstanceName,
-    //     azureOpenAIApiDeploymentName: process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME || fields.azureOpenAIApiDeploymentName,
-    //     azureOpenAIApiVersion: process.env.AZURE_OPENAI_API_VERSION || fields.azureOpenAIApiVersion
-    //   };
-    //   config = {
-    //     apiKey,
-    //     basePath: genAzureEndpoint({
-    //       ...azureConfig,
-    //     }),
-    //     baseOptions: {
-    //       headers: { 'api-key': apiKey },
-    //       params: {
-    //         'api-version': azureConfig.azureOpenAIApiVersion // this might change. I got the current value from the sample code at https://oai.azure.com/portal/chat
-    //       }
-    //     }
-    //   };
-    // }
     this.openai = new OpenAI(config);
     this.name = 'dall-e';
     this.description = `You can generate images with 'dall-e'. This tool is exclusively for visual content.
@@ -56,10 +47,24 @@ Guidelines:
 - It's best to follow this format for image creation. Come up with the optional inputs yourself if none are given:
 "Subject: [subject], Style: [style], Color: [color], Details: [details], Emotion: [emotion]"
 - Generate images only once per human query unless explicitly requested by the user`;
+    this.description_for_model =
+      process.env.DALLE2_SYSTEM_PROMPT ??
+      `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
+// 1. Prompts must be in English. Translate to English if needed.
+// 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
+// 3. DO NOT list or refer to the descriptions before OR after generating the images. They should ONLY ever be written out ONCE, in the \`"prompts"\` field of the request. You do not need to ask for permission to generate, just do it!
+// 4. Always mention the image type (photo, oil painting, watercolor painting, illustration, cartoon, drawing, vector, render, etc.) at the beginning of the caption. Unless the captions suggests otherwise, make one of the images a photo.
+// 5. Diversify depictions of ALL images with people to always include always DESCENT and GENDER for EACH person using direct terms. Adjust only human descriptions.
+// - EXPLICITLY specify these attributes, not abstractly reference them. The attributes should be specified in a minimal way and should directly describe their physical form.
+// - Your choices should be grounded in reality. For example, all of a given OCCUPATION should not be the same gender or race. Additionally, focus on creating diverse, inclusive, and exploratory scenes via the properties you choose during rewrites.  Make choices that may be insightful or unique sometimes.
+// - Use "various" or "diverse" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
+// - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
+// The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
+// All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.`;
   }
 
   getApiKey() {
-    const apiKey = process.env.DALLE_API_KEY || '';
+    const apiKey = process.env.DALLE2_API_KEY ?? process.env.DALLE_API_KEY ?? '';
     if (!apiKey) {
       throw new Error('Missing DALLE_API_KEY environment variable.');
     }
@@ -73,22 +78,26 @@ Guidelines:
       .trim();
   }
 
-  getMarkdownImageUrl(imageName) {
-    const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
-      .replace(/\\/g, '/')
-      .replace('public/', '');
-    return `![generated image](/${imageUrl})`;
+  wrapInMarkdown(imageUrl) {
+    return `![generated image](${imageUrl})`;
   }
 
   async _call(input) {
-    const resp = await this.openai.images.generate({
-      prompt: this.replaceUnwantedChars(input),
-      // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
-      n: 1,
-      // size: '1024x1024'
-      size: '512x512',
-    });
+    let resp;
+
+    try {
+      resp = await this.openai.images.generate({
+        prompt: this.replaceUnwantedChars(input),
+        // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
+        n: 1,
+        // size: '1024x1024'
+        size: '512x512',
+      });
+    } catch (error) {
+      logger.error('[DALL-E] Problem generating the image:', error);
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
+Error Message: ${error.message}`;
+    }
 
     const theImageUrl = resp.data[0].url;
 
@@ -96,32 +105,35 @@ Guidelines:
       throw new Error('No image URL returned from OpenAI API.');
     }
 
-    const regex = /img-[\w\d]+.png/;
-    const match = theImageUrl.match(regex);
-    let imageName = '1.png';
+    const imageBasename = getImageBasename(theImageUrl);
+    const imageExt = path.extname(imageBasename);
 
-    if (match) {
-      imageName = match[0];
-      console.log(imageName); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
-    } else {
-      console.log('No image name found in the string.');
-    }
+    const extension = imageExt.startsWith('.') ? imageExt.slice(1) : imageExt;
+    const imageName = `img-${uuidv4()}.${extension}`;
 
-    this.outputPath = path.resolve(__dirname, '..', '..', '..', '..', 'client', 'public', 'images');
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
-
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
-    }
+    logger.debug('[DALL-E-2]', {
+      imageName,
+      imageBasename,
+      imageExt,
+      extension,
+      theImageUrl,
+      data: resp.data[0],
+    });
 
     try {
-      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
-      this.result = this.getMarkdownImageUrl(imageName);
+      const result = await this.processFileURL({
+        fileStrategy: this.fileStrategy,
+        userId: this.userId,
+        URL: theImageUrl,
+        fileName: imageName,
+        basePath: 'images',
+        context: FileContext.image_generation,
+      });
+
+      this.result = this.wrapInMarkdown(result.filepath);
     } catch (error) {
-      console.error('Error while saving the image:', error);
-      this.result = theImageUrl;
+      logger.error('Error while saving the image:', error);
+      this.result = `Failed to save the image locally. ${error.message}`;
     }
 
     return this.result;

api/app/clients/tools/GoogleSearch.js

@@ -1,120 +0,0 @@
-const { Tool } = require('langchain/tools');
-const { google } = require('googleapis');
-
-/**
- * Represents a tool that allows an agent to use the Google Custom Search API.
- * @extends Tool
- */
-class GoogleSearchAPI extends Tool {
-  constructor(fields = {}) {
-    super();
-    this.cx = fields.GOOGLE_CSE_ID || this.getCx();
-    this.apiKey = fields.GOOGLE_API_KEY || this.getApiKey();
-    this.customSearch = undefined;
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'google';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-  description_for_model =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-
-  getCx() {
-    const cx = process.env.GOOGLE_CSE_ID || '';
-    if (!cx) {
-      throw new Error('Missing GOOGLE_CSE_ID environment variable.');
-    }
-    return cx;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.GOOGLE_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing GOOGLE_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getCustomSearch() {
-    if (!this.customSearch) {
-      const version = 'v1';
-      this.customSearch = google.customsearch(version);
-    }
-    return this.customSearch;
-  }
-
-  resultsToReadableFormat(results) {
-    let output = 'Results:\n';
-
-    results.forEach((resultObj, index) => {
-      output += `Title: ${resultObj.title}\n`;
-      output += `Link: ${resultObj.link}\n`;
-      if (resultObj.snippet) {
-        output += `Snippet: ${resultObj.snippet}\n`;
-      }
-
-      if (index < results.length - 1) {
-        output += '\n';
-      }
-    });
-
-    return output;
-  }
-
-  /**
-   * Calls the tool with the provided input and returns a promise that resolves with a response from the Google Custom Search API.
-   * @param {string} input - The input to provide to the API.
-   * @returns {Promise<String>} A promise that resolves with a response from the Google Custom Search API.
-   */
-  async _call(input) {
-    try {
-      const metadataResults = [];
-      const response = await this.getCustomSearch().cse.list({
-        q: input,
-        cx: this.cx,
-        auth: this.apiKey,
-        num: 5, // Limit the number of results to 5
-      });
-
-      // return response.data;
-      // console.log(response.data);
-
-      if (!response.data.items || response.data.items.length === 0) {
-        return this.resultsToReadableFormat([
-          { title: 'No good Google Search Result was found', link: '' },
-        ]);
-      }
-
-      // const results = response.items.slice(0, numResults);
-      const results = response.data.items;
-
-      for (const result of results) {
-        const metadataResult = {
-          title: result.title || '',
-          link: result.link || '',
-        };
-        if (result.snippet) {
-          metadataResult.snippet = result.snippet;
-        }
-        metadataResults.push(metadataResult);
-      }
-
-      return this.resultsToReadableFormat(metadataResults);
-    } catch (error) {
-      console.log(`Error searching Google: ${error}`);
-      // throw error;
-      return 'There was an error searching Google.';
-    }
-  }
-}
-
-module.exports = GoogleSearchAPI;

api/app/clients/tools/HttpRequestTool.js

@@ -1,108 +0,0 @@
-const { Tool } = require('langchain/tools');
-
-// class RequestsGetTool extends Tool {
-//   constructor(headers = {}, { maxOutputLength } = {}) {
-//     super();
-//     this.name = 'requests_get';
-//     this.headers = headers;
-//     this.maxOutputLength = maxOutputLength || 2000;
-//     this.description = `A portal to the internet. Use this when you need to get specific content from a website.
-//  - Input should be a  url (i.e. https://www.google.com). The output will be the text response of the GET request.`;
-//   }
-
-//   async _call(input) {
-//     const res = await fetch(input, {
-//       headers: this.headers
-//     });
-//     const text = await res.text();
-//     return text.slice(0, this.maxOutputLength);
-//   }
-// }
-
-// class RequestsPostTool extends Tool {
-//   constructor(headers = {}, { maxOutputLength } = {}) {
-//     super();
-//     this.name = 'requests_post';
-//     this.headers = headers;
-//     this.maxOutputLength = maxOutputLength || Infinity;
-//     this.description = `Use this when you want to POST to a website.
-//  - Input should be a json string with two keys: "url" and "data".
-//  - The value of "url" should be a string, and the value of "data" should be a dictionary of
-//  - key-value pairs you want to POST to the url as a JSON body.
-//  - Be careful to always use double quotes for strings in the json string
-//  - The output will be the text response of the POST request.`;
-//   }
-
-//   async _call(input) {
-//     try {
-//       const { url, data } = JSON.parse(input);
-//       const res = await fetch(url, {
-//         method: 'POST',
-//         headers: this.headers,
-//         body: JSON.stringify(data)
-//       });
-//       const text = await res.text();
-//       return text.slice(0, this.maxOutputLength);
-//     } catch (error) {
-//       return `${error}`;
-//     }
-//   }
-// }
-
-class HttpRequestTool extends Tool {
-  constructor(headers = {}, { maxOutputLength = Infinity } = {}) {
-    super();
-    this.headers = headers;
-    this.name = 'http_request';
-    this.maxOutputLength = maxOutputLength;
-    this.description =
-      'Executes HTTP methods (GET, POST, PUT, DELETE, etc.). The input is an object with three keys: "url", "method", and "data". Even for GET or DELETE, include "data" key as an empty string. "method" is the HTTP method, and "url" is the desired endpoint. If POST or PUT, "data" should contain a stringified JSON representing the body to send. Only one url per use.';
-  }
-
-  async _call(input) {
-    try {
-      const urlPattern = /"url":\s*"([^"]*)"/;
-      const methodPattern = /"method":\s*"([^"]*)"/;
-      const dataPattern = /"data":\s*"([^"]*)"/;
-
-      const url = input.match(urlPattern)[1];
-      const method = input.match(methodPattern)[1];
-      let data = input.match(dataPattern)[1];
-
-      // Parse 'data' back to JSON if possible
-      try {
-        data = JSON.parse(data);
-      } catch (e) {
-        // If it's not a JSON string, keep it as is
-      }
-
-      let options = {
-        method: method,
-        headers: this.headers,
-      };
-
-      if (['POST', 'PUT', 'PATCH'].includes(method.toUpperCase()) && data) {
-        if (typeof data === 'object') {
-          options.body = JSON.stringify(data);
-        } else {
-          options.body = data;
-        }
-        options.headers['Content-Type'] = 'application/json';
-      }
-
-      const res = await fetch(url, options);
-
-      const text = await res.text();
-      if (text.includes('<html')) {
-        return 'This tool is not designed to browse web pages. Only use it for API calls.';
-      }
-
-      return text.slice(0, this.maxOutputLength);
-    } catch (error) {
-      console.log(error);
-      return `${error}`;
-    }
-  }
-}
-
-module.exports = HttpRequestTool;

api/app/clients/tools/StableDiffusion.js

@@ -1,9 +1,10 @@
 // Generates image using stable diffusion webui's api (automatic1111)
 const fs = require('fs');
-const { Tool } = require('langchain/tools');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { Tool } = require('langchain/tools');
+const { logger } = require('~/config');
 
 class StableDiffusionAPI extends Tool {
   constructor(fields) {
@@ -81,7 +82,7 @@ Guidelines:
         .toFile(this.outputPath + '/' + imageName);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
-      console.error('Error while saving the image:', error);
+      logger.error('[StableDiffusion] Error while saving the image:', error);
       // this.result = theImageUrl;
     }
 

api/app/clients/tools/Wolfram.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-useless-escape */
 const axios = require('axios');
 const { Tool } = require('langchain/tools');
+const { logger } = require('~/config');
 
 class WolframAlphaAPI extends Tool {
   constructor(fields) {
@@ -38,7 +39,7 @@ General guidelines:
       const response = await axios.get(url, { responseType: 'text' });
       return response.data;
     } catch (error) {
-      console.error(`Error fetching raw text: ${error}`);
+      logger.error('[WolframAlphaAPI] Error fetching raw text:', error);
       throw error;
     }
   }
@@ -68,11 +69,10 @@ General guidelines:
       return response;
     } catch (error) {
       if (error.response && error.response.data) {
-        console.log('Error data:', error.response.data);
+        logger.error('[WolframAlphaAPI] Error data:', error);
         return error.response.data;
       } else {
-        console.log('Error querying Wolfram Alpha', error.message);
-        // throw error;
+        logger.error('[WolframAlphaAPI] Error querying Wolfram Alpha', error);
         return 'There was an error querying Wolfram Alpha.';
       }
     }

api/app/clients/tools/dynamic/OpenAPIPlugin.js

@@ -1,11 +1,12 @@
 require('dotenv').config();
-const { z } = require('zod');
 const fs = require('fs');
-const yaml = require('js-yaml');
+const { z } = require('zod');
 const path = require('path');
-const { DynamicStructuredTool } = require('langchain/tools');
+const yaml = require('js-yaml');
 const { createOpenAPIChain } = require('langchain/chains');
+const { DynamicStructuredTool } = require('langchain/tools');
 const { ChatPromptTemplate, HumanMessagePromptTemplate } = require('langchain/prompts');
+const { logger } = require('~/config');
 
 function addLinePrefix(text, prefix = '// ') {
   return text
@@ -52,7 +53,7 @@ async function readSpecFile(filePath) {
     }
     return yaml.load(fileContents);
   } catch (e) {
-    console.error(e);
+    logger.error('[readSpecFile] error', e);
     return false;
   }
 }
@@ -83,54 +84,51 @@ async function getSpec(url) {
   return ValidSpecPath.parse(url);
 }
 
-async function createOpenAPIPlugin({ data, llm, user, message, memory, signal, verbose = false }) {
+async function createOpenAPIPlugin({ data, llm, user, message, memory, signal }) {
   let spec;
   try {
-    spec = await getSpec(data.api.url, verbose);
+    spec = await getSpec(data.api.url);
   } catch (error) {
-    verbose && console.debug('getSpec error', error);
+    logger.error('[createOpenAPIPlugin] getSpec error', error);
     return null;
   }
 
   if (!spec) {
-    verbose && console.debug('No spec found');
+    logger.warn('[createOpenAPIPlugin] No spec found');
     return null;
   }
 
   const headers = {};
   const { auth, name_for_model, description_for_model, description_for_human } = data;
   if (auth && AuthDefinition.parse(auth)) {
-    verbose && console.debug('auth detected', auth);
+    logger.debug('[createOpenAPIPlugin] auth detected', auth);
     const { openai } = auth.verification_tokens;
     if (AuthBearer.parse(auth)) {
       headers.authorization = `Bearer ${openai}`;
-      verbose && console.debug('added auth bearer', headers);
+      logger.debug('[createOpenAPIPlugin] added auth bearer', headers);
     }
   }
 
-  const chainOptions = {
-    llm,
-    verbose,
-  };
+  const chainOptions = { llm };
 
   if (data.headers && data.headers['librechat_user_id']) {
-    verbose && console.debug('id detected', headers);
+    logger.debug('[createOpenAPIPlugin] id detected', headers);
     headers[data.headers['librechat_user_id']] = user;
   }
 
   if (Object.keys(headers).length > 0) {
-    verbose && console.debug('headers detected', headers);
+    logger.debug('[createOpenAPIPlugin] headers detected', headers);
     chainOptions.headers = headers;
   }
 
   if (data.params) {
-    verbose && console.debug('params detected', data.params);
+    logger.debug('[createOpenAPIPlugin] params detected', data.params);
     chainOptions.params = data.params;
   }
 
   let history = '';
   if (memory) {
-    verbose && console.debug('openAPI chain: memory detected', memory);
+    logger.debug('[createOpenAPIPlugin] openAPI chain: memory detected', memory);
     const { history: chat_history } = await memory.loadMemoryVariables({});
     history = chat_history?.length > 0 ? `\n\n## Chat History:\n${chat_history}\n` : '';
   }

api/app/clients/tools/index.js

@@ -1,41 +1,44 @@
-const GoogleSearchAPI = require('./GoogleSearch');
-const HttpRequestTool = require('./HttpRequestTool');
-const AIPluginTool = require('./AIPluginTool');
-const OpenAICreateImage = require('./DALL-E');
-const DALLE3 = require('./structured/DALLE3');
-const StructuredSD = require('./structured/StableDiffusion');
-const StableDiffusionAPI = require('./StableDiffusion');
+const availableTools = require('./manifest.json');
+// Basic Tools
+const CodeBrew = require('./CodeBrew');
 const WolframAlphaAPI = require('./Wolfram');
-const StructuredWolfram = require('./structured/Wolfram');
+const AzureAiSearch = require('./AzureAiSearch');
+const OpenAICreateImage = require('./DALL-E');
+const StableDiffusionAPI = require('./StableDiffusion');
 const SelfReflectionTool = require('./SelfReflection');
-const AzureCognitiveSearch = require('./AzureCognitiveSearch');
-const StructuredACS = require('./structured/AzureCognitiveSearch');
+
+// Structured Tools
+const DALLE3 = require('./structured/DALLE3');
 const ChatTool = require('./structured/ChatTool');
 const E2BTools = require('./structured/E2BTools');
 const CodeSherpa = require('./structured/CodeSherpa');
+const StructuredSD = require('./structured/StableDiffusion');
+const StructuredACS = require('./structured/AzureAISearch');
 const CodeSherpaTools = require('./structured/CodeSherpaTools');
-const availableTools = require('./manifest.json');
-const CodeInterpreter = require('./CodeInterpreter');
-const CodeBrew = require('./CodeBrew');
+const GoogleSearchAPI = require('./structured/GoogleSearch');
+const StructuredWolfram = require('./structured/Wolfram');
+const TavilySearchResults = require('./structured/TavilySearchResults');
+const TraversaalSearch = require('./structured/TraversaalSearch');
 
 module.exports = {
   availableTools,
-  GoogleSearchAPI,
-  HttpRequestTool,
-  AIPluginTool,
-  OpenAICreateImage,
-  DALLE3,
-  StableDiffusionAPI,
-  StructuredSD,
-  WolframAlphaAPI,
-  StructuredWolfram,
-  SelfReflectionTool,
-  AzureCognitiveSearch,
-  StructuredACS,
-  E2BTools,
-  ChatTool,
-  CodeSherpa,
-  CodeSherpaTools,
-  CodeInterpreter,
+  // Basic Tools
   CodeBrew,
+  AzureAiSearch,
+  GoogleSearchAPI,
+  WolframAlphaAPI,
+  OpenAICreateImage,
+  StableDiffusionAPI,
+  SelfReflectionTool,
+  // Structured Tools
+  DALLE3,
+  ChatTool,
+  E2BTools,
+  CodeSherpa,
+  StructuredSD,
+  StructuredACS,
+  CodeSherpaTools,
+  StructuredWolfram,
+  TavilySearchResults,
+  TraversaalSearch,
 };

api/app/clients/tools/manifest.json

@@ -1,4 +1,17 @@
 [
+  {
+    "name": "Traversaal",
+    "pluginKey": "traversaal_search",
+    "description": "Traversaal is a robust search API tailored for LLM Agents. Get an API key here: https://api.traversaal.ai",
+    "icon": "https://traversaal.ai/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TRAVERSAAL_API_KEY",
+        "label": "Traversaal API Key",
+        "description": "Get your API key here: <a href=\"https://api.traversaal.ai\" target=\"_blank\">https://api.traversaal.ai</a>"
+      }
+    ]
+  },
   {
     "name": "Google",
     "pluginKey": "google",
@@ -89,7 +102,7 @@
     "icon": "https://i.imgur.com/u2TzXzH.png",
     "authConfig": [
       {
-        "authField": "DALLE_API_KEY",
+        "authField": "DALLE2_API_KEY||DALLE_API_KEY",
         "label": "OpenAI API Key",
         "description": "You can use DALL-E with your API Key from OpenAI."
       }
@@ -102,12 +115,25 @@
     "icon": "https://i.imgur.com/u2TzXzH.png",
     "authConfig": [
       {
-        "authField": "DALLE_API_KEY",
+        "authField": "DALLE3_API_KEY||DALLE_API_KEY",
         "label": "OpenAI API Key",
         "description": "You can use DALL-E with your API Key from OpenAI."
       }
     ]
   },
+  {
+    "name": "Tavily Search",
+    "pluginKey": "tavily_search_results_json",
+    "description": "Tavily Search is a robust search API tailored for LLM Agents. It seamlessly integrates with diverse data sources to ensure a superior, relevant search experience.",
+    "icon": "https://tavily.com/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TAVILY_API_KEY",
+        "label": "Tavily API Key",
+        "description": "Get your API key here: https://app.tavily.com/"
+      }
+    ]
+  },
   {
     "name": "Calculator",
     "pluginKey": "calculator",
@@ -143,38 +169,25 @@
     ]
   },
   {
-    "name": "Azure Cognitive Search",
-    "pluginKey": "azure-cognitive-search",
-    "description": "Use Azure Cognitive Search to find information",
+    "name": "Azure AI Search",
+    "pluginKey": "azure-ai-search",
+    "description": "Use Azure AI Search to find information",
     "icon": "https://i.imgur.com/E7crPze.png",
     "authConfig": [
       {
-        "authField": "AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT",
-        "label": "Azur Cognitive Search Endpoint",
-        "description": "You need to provide your Endpoint for Azure Cognitive Search."
+        "authField": "AZURE_AI_SEARCH_SERVICE_ENDPOINT",
+        "label": "Azure AI Search Endpoint",
+        "description": "You need to provide your Endpoint for Azure AI Search."
       },
       {
-        "authField": "AZURE_COGNITIVE_SEARCH_INDEX_NAME",
-        "label": "Azur Cognitive Search Index Name",
-        "description": "You need to provide your Index Name for Azure Cognitive Search."
+        "authField": "AZURE_AI_SEARCH_INDEX_NAME",
+        "label": "Azure AI Search Index Name",
+        "description": "You need to provide your Index Name for Azure AI Search."
       },
       {
-        "authField": "AZURE_COGNITIVE_SEARCH_API_KEY",
-        "label": "Azur Cognitive Search API Key",
-        "description": "You need to provideq your API Key for Azure Cognitive Search."
-      }
-    ]
-  },
-  {
-    "name": "Code Interpreter",
-    "pluginKey": "codeinterpreter",
-    "description": "[Experimental] Analyze files and run code online with ease. Requires dockerized python server in /pyserver/",
-    "icon": "/assets/code.png",
-    "authConfig": [
-      {
-        "authField": "OPENAI_API_KEY",
-        "label": "OpenAI API Key",
-        "description": "Gets Code from Open AI API"
+        "authField": "AZURE_AI_SEARCH_API_KEY",
+        "label": "Azure AI Search API Key",
+        "description": "You need to provideq your API Key for Azure AI Search."
       }
     ]
   },

api/app/clients/tools/saveImageFromUrl.js

@@ -1,39 +0,0 @@
-const axios = require('axios');
-const fs = require('fs');
-const path = require('path');
-
-async function saveImageFromUrl(url, outputPath, outputFilename) {
-  try {
-    // Fetch the image from the URL
-    const response = await axios({
-      url,
-      responseType: 'stream',
-    });
-
-    // Check if the output directory exists, if not, create it
-    if (!fs.existsSync(outputPath)) {
-      fs.mkdirSync(outputPath, { recursive: true });
-    }
-
-    // Ensure the output filename has a '.png' extension
-    const filenameWithPngExt = outputFilename.endsWith('.png')
-      ? outputFilename
-      : `${outputFilename}.png`;
-
-    // Create a writable stream for the output path
-    const outputFilePath = path.join(outputPath, filenameWithPngExt);
-    const writer = fs.createWriteStream(outputFilePath);
-
-    // Pipe the response data to the output file
-    response.data.pipe(writer);
-
-    return new Promise((resolve, reject) => {
-      writer.on('finish', resolve);
-      writer.on('error', reject);
-    });
-  } catch (error) {
-    console.error('Error while saving the image:', error);
-  }
-}
-
-module.exports = saveImageFromUrl;

api/app/clients/tools/structured/AzureAISearch.js

@@ -0,0 +1,104 @@
+const { z } = require('zod');
+const { StructuredTool } = require('langchain/tools');
+const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
+const { logger } = require('~/config');
+
+class AzureAISearch extends StructuredTool {
+  // Constants for default values
+  static DEFAULT_API_VERSION = '2023-11-01';
+  static DEFAULT_QUERY_TYPE = 'simple';
+  static DEFAULT_TOP = 5;
+
+  // Helper function for initializing properties
+  _initializeField(field, envVar, defaultValue) {
+    return field || process.env[envVar] || defaultValue;
+  }
+
+  constructor(fields = {}) {
+    super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
+    // Define schema
+    this.schema = z.object({
+      query: z.string().describe('Search word or phrase to Azure AI Search'),
+    });
+
+    // Initialize properties using helper function
+    this.serviceEndpoint = this._initializeField(
+      fields.AZURE_AI_SEARCH_SERVICE_ENDPOINT,
+      'AZURE_AI_SEARCH_SERVICE_ENDPOINT',
+    );
+    this.indexName = this._initializeField(
+      fields.AZURE_AI_SEARCH_INDEX_NAME,
+      'AZURE_AI_SEARCH_INDEX_NAME',
+    );
+    this.apiKey = this._initializeField(fields.AZURE_AI_SEARCH_API_KEY, 'AZURE_AI_SEARCH_API_KEY');
+    this.apiVersion = this._initializeField(
+      fields.AZURE_AI_SEARCH_API_VERSION,
+      'AZURE_AI_SEARCH_API_VERSION',
+      AzureAISearch.DEFAULT_API_VERSION,
+    );
+    this.queryType = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE',
+      AzureAISearch.DEFAULT_QUERY_TYPE,
+    );
+    this.top = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_TOP,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_TOP',
+      AzureAISearch.DEFAULT_TOP,
+    );
+    this.select = this._initializeField(
+      fields.AZURE_AI_SEARCH_SEARCH_OPTION_SELECT,
+      'AZURE_AI_SEARCH_SEARCH_OPTION_SELECT',
+    );
+
+    // Check for required fields
+    if (!this.override && (!this.serviceEndpoint || !this.indexName || !this.apiKey)) {
+      throw new Error(
+        'Missing AZURE_AI_SEARCH_SERVICE_ENDPOINT, AZURE_AI_SEARCH_INDEX_NAME, or AZURE_AI_SEARCH_API_KEY environment variable.',
+      );
+    }
+
+    if (this.override) {
+      return;
+    }
+
+    // Create SearchClient
+    this.client = new SearchClient(
+      this.serviceEndpoint,
+      this.indexName,
+      new AzureKeyCredential(this.apiKey),
+      { apiVersion: this.apiVersion },
+    );
+  }
+
+  // Improved error handling and logging
+  async _call(data) {
+    const { query } = data;
+    try {
+      const searchOption = {
+        queryType: this.queryType,
+        top: this.top,
+      };
+      if (this.select) {
+        searchOption.select = this.select.split(',');
+      }
+      const searchResults = await this.client.search(query, searchOption);
+      const resultDocuments = [];
+      for await (const result of searchResults.results) {
+        resultDocuments.push(result.document);
+      }
+      return JSON.stringify(resultDocuments);
+    } catch (error) {
+      logger.error('Azure AI Search request failed', error);
+      return 'There was an error with Azure AI Search.';
+    }
+  }
+}
+
+module.exports = AzureAISearch;

api/app/clients/tools/structured/AzureCognitiveSearch.js

@@ -1,116 +0,0 @@
-const { StructuredTool } = require('langchain/tools');
-const { z } = require('zod');
-const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
-
-class AzureCognitiveSearch extends StructuredTool {
-  constructor(fields = {}) {
-    super();
-    this.serviceEndpoint =
-      fields.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || this.getServiceEndpoint();
-    this.indexName = fields.AZURE_COGNITIVE_SEARCH_INDEX_NAME || this.getIndexName();
-    this.apiKey = fields.AZURE_COGNITIVE_SEARCH_API_KEY || this.getApiKey();
-
-    this.apiVersion = fields.AZURE_COGNITIVE_SEARCH_API_VERSION || this.getApiVersion();
-
-    this.queryType = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || this.getQueryType();
-    this.top = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP || this.getTop();
-    this.select = fields.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT || this.getSelect();
-
-    this.client = new SearchClient(
-      this.serviceEndpoint,
-      this.indexName,
-      new AzureKeyCredential(this.apiKey),
-      {
-        apiVersion: this.apiVersion,
-      },
-    );
-    this.schema = z.object({
-      query: z.string().describe('Search word or phrase to Azure Cognitive Search'),
-    });
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'azure-cognitive-search';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'azure-cognitive-search\' tool to retrieve search results relevant to your input';
-
-  getServiceEndpoint() {
-    const serviceEndpoint = process.env.AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT || '';
-    if (!serviceEndpoint) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_SERVICE_ENDPOINT environment variable.');
-    }
-    return serviceEndpoint;
-  }
-
-  getIndexName() {
-    const indexName = process.env.AZURE_COGNITIVE_SEARCH_INDEX_NAME || '';
-    if (!indexName) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_INDEX_NAME environment variable.');
-    }
-    return indexName;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.AZURE_COGNITIVE_SEARCH_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing AZURE_COGNITIVE_SEARCH_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getApiVersion() {
-    return process.env.AZURE_COGNITIVE_SEARCH_API_VERSION || '2020-06-30';
-  }
-
-  getQueryType() {
-    return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_QUERY_TYPE || 'simple';
-  }
-
-  getTop() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP) {
-      return Number(process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_TOP);
-    } else {
-      return 5;
-    }
-  }
-
-  getSelect() {
-    if (process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT) {
-      return process.env.AZURE_COGNITIVE_SEARCH_SEARCH_OPTION_SELECT.split(',');
-    } else {
-      return null;
-    }
-  }
-
-  async _call(data) {
-    const { query } = data;
-    try {
-      const searchOption = {
-        queryType: this.queryType,
-        top: this.top,
-      };
-      if (this.select) {
-        searchOption.select = this.select;
-      }
-      const searchResults = await this.client.search(query, searchOption);
-      const resultDocuments = [];
-      for await (const result of searchResults.results) {
-        resultDocuments.push(result.document);
-      }
-      return JSON.stringify(resultDocuments);
-    } catch (error) {
-      console.error(`Azure Cognitive Search request failed: ${error}`);
-      return 'There was an error with Azure Cognitive Search.';
-    }
-  }
-}
-
-module.exports = AzureCognitiveSearch;

api/app/clients/tools/structured/CodeSherpa.js

@@ -28,14 +28,14 @@ class RunCode extends StructuredTool {
   }
 
   async _call({ code, language = 'python' }) {
-    // console.log('<--------------- Running Code --------------->', { code, language });
+    // logger.debug('<--------------- Running Code --------------->', { code, language });
     const response = await axios({
       url: `${this.url}/repl`,
       method: 'post',
       headers: this.headers,
       data: { code, language },
     });
-    // console.log('<--------------- Sucessfully ran Code --------------->', response.data);
+    // logger.debug('<--------------- Sucessfully ran Code --------------->', response.data);
     return response.data.result;
   }
 }

api/app/clients/tools/structured/CodeSherpaTools.js

@@ -42,14 +42,14 @@ class RunCode extends StructuredTool {
   }
 
   async _call({ code, language = 'python' }) {
-    // console.log('<--------------- Running Code --------------->', { code, language });
+    // logger.debug('<--------------- Running Code --------------->', { code, language });
     const response = await axios({
       url: `${this.url}/repl`,
       method: 'post',
       headers: this.headers,
       data: { code, language },
     });
-    // console.log('<--------------- Sucessfully ran Code --------------->', response.data);
+    // logger.debug('<--------------- Sucessfully ran Code --------------->', response.data);
     return response.data.result;
   }
 }

api/app/clients/tools/structured/DALLE3.js

@@ -1,28 +1,50 @@
-// From https://platform.openai.com/docs/guides/images/usage?context=node
-// To use this tool, you must pass in a configured OpenAIApi object.
-const fs = require('fs');
-const path = require('path');
 const { z } = require('zod');
+const path = require('path');
 const OpenAI = require('openai');
+const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
-const saveImageFromUrl = require('../saveImageFromUrl');
-const extractBaseURL = require('../../../../utils/extractBaseURL');
-const { DALLE3_SYSTEM_PROMPT, DALLE_REVERSE_PROXY, PROXY } = process.env;
+const { FileContext } = require('librechat-data-provider');
+const { getImageBasename } = require('~/server/services/Files/images');
+const extractBaseURL = require('~/utils/extractBaseURL');
+const { logger } = require('~/config');
+
 class DALLE3 extends Tool {
   constructor(fields = {}) {
     super();
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
 
-    let apiKey = fields.DALLE_API_KEY || this.getApiKey();
+    this.userId = fields.userId;
+    this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      /** @type {processFileURL} Necessary for output to contain all image metadata. */
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
+
+    let apiKey = fields.DALLE3_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
     const config = { apiKey };
-    if (DALLE_REVERSE_PROXY) {
-      config.baseURL = extractBaseURL(DALLE_REVERSE_PROXY);
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
     }
 
-    if (PROXY) {
-      config.httpAgent = new HttpsProxyAgent(PROXY);
+    if (process.env.DALLE3_AZURE_API_VERSION && process.env.DALLE3_BASEURL) {
+      config.baseURL = process.env.DALLE3_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE3_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE3_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE3_API_KEY;
     }
 
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
+    }
+
+    /** @type {OpenAI} */
     this.openai = new OpenAI(config);
     this.name = 'dalle';
     this.description = `Use DALLE to create images from text descriptions.
@@ -30,7 +52,7 @@ class DALLE3 extends Tool {
     - Create only one image, without repeating or listing descriptions outside the "prompts" field.
     - Maintains the original intent of the description, with parameters for image style, quality, and size to tailor the output.`;
     this.description_for_model =
-      DALLE3_SYSTEM_PROMPT ??
+      process.env.DALLE3_SYSTEM_PROMPT ??
       `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
     // 1. Prompts must be in English. Translate to English if needed.
     // 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
@@ -42,7 +64,8 @@ class DALLE3 extends Tool {
     // - Use "various" or "diverse" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
     // - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
     // The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
-    // All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.`;
+    // All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.
+    // - The "vivid" style is HIGHLY preferred, but "natural" is also supported.`;
     this.schema = z.object({
       prompt: z
         .string()
@@ -67,8 +90,8 @@ class DALLE3 extends Tool {
   }
 
   getApiKey() {
-    const apiKey = process.env.DALLE_API_KEY || '';
-    if (!apiKey) {
+    const apiKey = process.env.DALLE3_API_KEY ?? process.env.DALLE_API_KEY ?? '';
+    if (!apiKey && !this.override) {
       throw new Error('Missing DALLE_API_KEY environment variable.');
     }
     return apiKey;
@@ -81,12 +104,8 @@ class DALLE3 extends Tool {
       .trim();
   }
 
-  getMarkdownImageUrl(imageName) {
-    const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
-      .replace(/\\/g, '/')
-      .replace('public/', '');
-    return `![generated image](/${imageUrl})`;
+  wrapInMarkdown(imageUrl) {
+    return `![generated image](${imageUrl})`;
   }
 
   async _call(data) {
@@ -106,12 +125,13 @@ class DALLE3 extends Tool {
         n: 1,
       });
     } catch (error) {
-      return `Something went wrong when trying to generate the image. The DALL-E API may unavailable:
+      logger.error('[DALL-E-3] Problem generating the image:', error);
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
 Error Message: ${error.message}`;
     }
 
     if (!resp) {
-      return 'Something went wrong when trying to generate the image. The DALL-E API may unavailable';
+      return 'Something went wrong when trying to generate the image. The DALL-E API may be unavailable';
     }
 
     const theImageUrl = resp.data[0].url;
@@ -120,42 +140,39 @@ Error Message: ${error.message}`;
       return 'No image URL returned from OpenAI API. There may be a problem with the API or your configuration.';
     }
 
-    const regex = /img-[\w\d]+.png/;
-    const match = theImageUrl.match(regex);
-    let imageName = '1.png';
+    const imageBasename = getImageBasename(theImageUrl);
+    const imageExt = path.extname(imageBasename);
 
-    if (match) {
-      imageName = match[0];
-      console.log(imageName); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
-    } else {
-      console.log('No image name found in the string.');
-    }
+    const extension = imageExt.startsWith('.') ? imageExt.slice(1) : imageExt;
+    const imageName = `img-${uuidv4()}.${extension}`;
 
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
-
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
-    }
+    logger.debug('[DALL-E-3]', {
+      imageName,
+      imageBasename,
+      imageExt,
+      extension,
+      theImageUrl,
+      data: resp.data[0],
+    });
 
     try {
-      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
-      this.result = this.getMarkdownImageUrl(imageName);
+      const result = await this.processFileURL({
+        fileStrategy: this.fileStrategy,
+        userId: this.userId,
+        URL: theImageUrl,
+        fileName: imageName,
+        basePath: 'images',
+        context: FileContext.image_generation,
+      });
+
+      if (this.returnMetadata) {
+        this.result = result;
+      } else {
+        this.result = this.wrapInMarkdown(result.filepath);
+      }
     } catch (error) {
-      console.error('Error while saving the image:', error);
-      this.result = theImageUrl;
+      logger.error('Error while saving the image:', error);
+      this.result = `Failed to save the image locally. ${error.message}`;
     }
 
     return this.result;

api/app/clients/tools/structured/E2BTools.js

@@ -1,9 +1,10 @@
+const { z } = require('zod');
+const axios = require('axios');
 const { StructuredTool } = require('langchain/tools');
 const { PromptTemplate } = require('langchain/prompts');
-const { createExtractionChainFromZod } = require('./extractionChain');
 // const { ChatOpenAI } = require('langchain/chat_models/openai');
-const axios = require('axios');
-const { z } = require('zod');
+const { createExtractionChainFromZod } = require('./extractionChain');
+const { logger } = require('~/config');
 
 const envs = ['Nodejs', 'Go', 'Bash', 'Rust', 'Python3', 'PHP', 'Java', 'Perl', 'DotNET'];
 const env = z.enum(envs);
@@ -34,8 +35,8 @@ async function extractEnvFromCode(code, model) {
   // const chatModel = new ChatOpenAI({ openAIApiKey, modelName: 'gpt-4-0613', temperature: 0 });
   const chain = createExtractionChainFromZod(zodSchema, model, { prompt, verbose: true });
   const result = await chain.run(code);
-  console.log('<--------------- extractEnvFromCode --------------->');
-  console.log(result);
+  logger.debug('<--------------- extractEnvFromCode --------------->');
+  logger.debug(result);
   return result.env;
 }
 
@@ -69,7 +70,7 @@ class RunCommand extends StructuredTool {
   }
 
   async _call(data) {
-    console.log(`<--------------- Running ${data} --------------->`);
+    logger.debug(`<--------------- Running ${data} --------------->`);
     const response = await axios({
       url: `${this.url}/commands`,
       method: 'post',
@@ -96,7 +97,7 @@ class ReadFile extends StructuredTool {
   }
 
   async _call(data) {
-    console.log(`<--------------- Reading ${data} --------------->`);
+    logger.debug(`<--------------- Reading ${data} --------------->`);
     const response = await axios.get(`${this.url}/files`, { params: data, headers: this.headers });
     return response.data;
   }
@@ -121,12 +122,12 @@ class WriteFile extends StructuredTool {
 
   async _call(data) {
     let { env, path, content } = data;
-    console.log(`<--------------- environment ${env} typeof ${typeof env}--------------->`);
+    logger.debug(`<--------------- environment ${env} typeof ${typeof env}--------------->`);
     if (env && !envs.includes(env)) {
-      console.log(`<--------------- Invalid environment ${env} --------------->`);
+      logger.debug(`<--------------- Invalid environment ${env} --------------->`);
       env = await extractEnvFromCode(content, this.model);
     } else if (!env) {
-      console.log('<--------------- Undefined environment --------------->');
+      logger.debug('<--------------- Undefined environment --------------->');
       env = await extractEnvFromCode(content, this.model);
     }
 
@@ -139,7 +140,7 @@ class WriteFile extends StructuredTool {
         content,
       },
     };
-    console.log('Writing to file', JSON.stringify(payload));
+    logger.debug('Writing to file', JSON.stringify(payload));
 
     await axios({
       url: `${this.url}/files`,

api/app/clients/tools/structured/GoogleSearch.js

@@ -0,0 +1,65 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class GoogleSearchResults extends Tool {
+  static lc_name() {
+    return 'GoogleSearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVarApiKey = 'GOOGLE_API_KEY';
+    this.envVarSearchEngineId = 'GOOGLE_CSE_ID';
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? getEnvironmentVariable(this.envVarApiKey);
+    this.searchEngineId =
+      fields.searchEngineId ?? getEnvironmentVariable(this.envVarSearchEngineId);
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'google';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 10.'),
+      // Note: Google API has its own parameters for search customization, adjust as needed.
+    });
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, max_results = 5 } = validationResult.data;
+
+    const response = await fetch(
+      `https://www.googleapis.com/customsearch/v1?key=${this.apiKey}&cx=${
+        this.searchEngineId
+      }&q=${encodeURIComponent(query)}&num=${max_results}`,
+      {
+        method: 'GET',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error.message}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = GoogleSearchResults;

api/app/clients/tools/structured/StableDiffusion.js

@@ -1,14 +1,31 @@
 // Generates image using stable diffusion webui's api (automatic1111)
 const fs = require('fs');
-const { StructuredTool } = require('langchain/tools');
 const { z } = require('zod');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { v4: uuidv4 } = require('uuid');
+const { StructuredTool } = require('langchain/tools');
+const { FileContext } = require('librechat-data-provider');
+const paths = require('~/config/paths');
+const { logger } = require('~/config');
 
 class StableDiffusionAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /** @type {string} User ID */
+    this.userId = fields.userId;
+    /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
+    this.req = fields.req;
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
+    if (fields.uploadImageBuffer) {
+      /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */
+      this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);
+    }
+
     this.name = 'stable-diffusion';
     this.url = fields.SD_WEBUI_URL || this.getServerURL();
     this.description_for_model = `// Generate images and visuals using text.
@@ -43,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getMarkdownImageUrl(imageName) {
     const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
+      .join(this.relativePath, this.userId, imageName)
       .replace(/\\/g, '/')
       .replace('public/', '');
     return `![generated image](/${imageUrl})`;
@@ -51,7 +68,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getServerURL() {
     const url = process.env.SD_WEBUI_URL || '';
-    if (!url) {
+    if (!url && !this.override) {
       throw new Error('Missing SD_WEBUI_URL environment variable.');
     }
     return url;
@@ -69,46 +86,67 @@ class StableDiffusionAPI extends StructuredTool {
       width: 1024,
       height: 1024,
     };
-    const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
-    const image = response.data.images[0];
-    const pngPayload = { image: `data:image/png;base64,${image}` };
-    const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);
-    const info = response2.data.info;
+    const generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
+    const image = generationResponse.data.images[0];
 
-    // Generate unique name
-    const imageName = `${Date.now()}.png`;
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+    /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */
+    let info = {};
+    try {
+      info = JSON.parse(generationResponse.data.info);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while getting image metadata:', error);
+    }
 
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
+    const file_id = uuidv4();
+    const imageName = `${file_id}.png`;
+    const { imageOutput: imageOutputPath, clientPath } = paths;
+    const filepath = path.join(imageOutputPath, this.userId, imageName);
+    this.relativePath = path.relative(clientPath, imageOutputPath);
+
+    if (!fs.existsSync(path.join(imageOutputPath, this.userId))) {
+      fs.mkdirSync(path.join(imageOutputPath, this.userId), { recursive: true });
     }
 
     try {
       const buffer = Buffer.from(image.split(',', 1)[0], 'base64');
+      if (this.returnMetadata && this.uploadImageBuffer && this.req) {
+        const file = await this.uploadImageBuffer({
+          req: this.req,
+          context: FileContext.image_generation,
+          resize: false,
+          metadata: {
+            buffer,
+            height: info.height,
+            width: info.width,
+            bytes: Buffer.byteLength(buffer),
+            filename: imageName,
+            type: 'image/png',
+            file_id,
+          },
+        });
+
+        const generationInfo = info.infotexts[0].split('\n').pop();
+        return {
+          ...file,
+          prompt,
+          metadata: {
+            negative_prompt,
+            seed: info.seed,
+            info: generationInfo,
+          },
+        };
+      }
+
       await sharp(buffer)
         .withMetadata({
           iptcpng: {
-            parameters: info,
+            parameters: info.infotexts[0],
           },
         })
-        .toFile(this.outputPath + '/' + imageName);
+        .toFile(filepath);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
-      console.error('Error while saving the image:', error);
-      // this.result = theImageUrl;
+      logger.error('[StableDiffusion] Error while saving the image:', error);
     }
 
     return this.result;

api/app/clients/tools/structured/TavilySearchResults.js

@@ -0,0 +1,92 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class TavilySearchResults extends Tool {
+  static lc_name() {
+    return 'TavilySearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVar = 'TAVILY_API_KEY';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? this.getApiKey();
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'tavily_search_results_json';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 5.'),
+      search_depth: z
+        .enum(['basic', 'advanced'])
+        .optional()
+        .describe(
+          'The depth of the search, affecting result quality and response time (`basic` or `advanced`). Default is basic for quick results and advanced for indepth high quality results but longer response time. Advanced calls equals 2 requests.',
+        ),
+      include_images: z
+        .boolean()
+        .optional()
+        .describe(
+          'Whether to include a list of query-related images in the response. Default is False.',
+        ),
+      include_answer: z
+        .boolean()
+        .optional()
+        .describe('Whether to include answers in the search results. Default is False.'),
+      // include_raw_content: z.boolean().optional().describe('Whether to include raw content in the search results. Default is False.'),
+      // include_domains: z.array(z.string()).optional().describe('A list of domains to specifically include in the search results.'),
+      // exclude_domains: z.array(z.string()).optional().describe('A list of domains to specifically exclude from the search results.'),
+    });
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable(this.envVar);
+    if (!apiKey && !this.override) {
+      throw new Error(`Missing ${this.envVar} environment variable.`);
+    }
+    return apiKey;
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, ...rest } = validationResult.data;
+
+    const requestBody = {
+      api_key: this.apiKey,
+      query,
+      ...rest,
+      ...this.kwargs,
+    };
+
+    const response = await fetch('https://api.tavily.com/search', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(requestBody),
+    });
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = TavilySearchResults;

api/app/clients/tools/structured/TraversaalSearch.js

@@ -0,0 +1,89 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+const { logger } = require('~/config');
+
+/**
+ * Tool for the Traversaal AI search API, Ares.
+ */
+class TraversaalSearch extends Tool {
+  static lc_name() {
+    return 'TraversaalSearch';
+  }
+  constructor(fields) {
+    super(fields);
+    this.name = 'traversaal_search';
+    this.description = `An AI search engine optimized for comprehensive, accurate, and trusted results.
+    Useful for when you need to answer questions about current events. Input should be a search query.`;
+    this.description_for_model =
+      '\'Please create a specific sentence for the AI to understand and use as a query to search the web based on the user\'s request. For example, "Find information about the highest mountains in the world." or "Show me the latest news articles about climate change and its impact on polar ice caps."\'';
+    this.schema = z.object({
+      query: z
+        .string()
+        .describe(
+          'A properly written sentence to be interpreted by an AI to search the web according to the user\'s request.',
+        ),
+    });
+
+    this.apiKey = fields?.TRAVERSAAL_API_KEY ?? this.getApiKey();
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable('TRAVERSAAL_API_KEY');
+    if (!apiKey && this.override) {
+      throw new Error(
+        'No Traversaal API key found. Either set an environment variable named "TRAVERSAAL_API_KEY" or pass an API key as "apiKey".',
+      );
+    }
+    return apiKey;
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  async _call({ query }, _runManager) {
+    const body = {
+      query: [query],
+    };
+    try {
+      const response = await fetch('https://api-ares.traversaal.ai/live/predict', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-api-key': this.apiKey,
+        },
+        body: JSON.stringify({ ...body }),
+      });
+      const json = await response.json();
+      if (!response.ok) {
+        throw new Error(
+          `Request failed with status code ${response.status}: ${json.error ?? json.message}`,
+        );
+      }
+      if (!json.data) {
+        throw new Error('Could not parse Traversaal API results. Please try again.');
+      }
+
+      const baseText = json.data?.response_text ?? '';
+      const sources = json.data?.web_url;
+      const noResponse = 'No response found in Traversaal API results';
+
+      if (!baseText && !sources) {
+        return noResponse;
+      }
+
+      const sourcesText = sources?.length ? '\n\nSources:\n - ' + sources.join('\n - ') : '';
+
+      const result = baseText + sourcesText;
+
+      if (!result) {
+        return noResponse;
+      }
+
+      return result;
+    } catch (error) {
+      logger.error('Traversaal API request failed', error);
+      return `Traversaal API request failed: ${error.message}`;
+    }
+  }
+}
+
+module.exports = TraversaalSearch;

api/app/clients/tools/structured/Wolfram.js

@@ -1,11 +1,15 @@
 /* eslint-disable no-useless-escape */
 const axios = require('axios');
-const { StructuredTool } = require('langchain/tools');
 const { z } = require('zod');
+const { StructuredTool } = require('langchain/tools');
+const { logger } = require('~/config');
 
 class WolframAlphaAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
     this.name = 'wolfram';
     this.apiKey = fields.WOLFRAM_APP_ID || this.getAppId();
     this.description_for_model = `// Access dynamic computation and curated data from WolframAlpha and Wolfram Cloud.
@@ -47,14 +51,14 @@ class WolframAlphaAPI extends StructuredTool {
       const response = await axios.get(url, { responseType: 'text' });
       return response.data;
     } catch (error) {
-      console.error(`Error fetching raw text: ${error}`);
+      logger.error('[WolframAlphaAPI] Error fetching raw text:', error);
       throw error;
     }
   }
 
   getAppId() {
     const appId = process.env.WOLFRAM_APP_ID || '';
-    if (!appId) {
+    if (!appId && !this.override) {
       throw new Error('Missing WOLFRAM_APP_ID environment variable.');
     }
     return appId;
@@ -78,11 +82,10 @@ class WolframAlphaAPI extends StructuredTool {
       return response;
     } catch (error) {
       if (error.response && error.response.data) {
-        console.log('Error data:', error.response.data);
+        logger.error('[WolframAlphaAPI] Error data:', error);
         return error.response.data;
       } else {
-        console.log('Error querying Wolfram Alpha', error.message);
-        // throw error;
+        logger.error('[WolframAlphaAPI] Error querying Wolfram Alpha', error);
         return 'There was an error querying Wolfram Alpha.';
       }
     }

api/app/clients/tools/structured/specs/DALLE3.spec.js

@@ -1,11 +1,31 @@
-const fs = require('fs');
-const path = require('path');
 const OpenAI = require('openai');
 const DALLE3 = require('../DALLE3');
-const saveImageFromUrl = require('../../saveImageFromUrl');
+
+const { logger } = require('~/config');
 
 jest.mock('openai');
 
+const processFileURL = jest.fn();
+
+jest.mock('~/server/services/Files/images', () => ({
+  getImageBasename: jest.fn().mockImplementation((url) => {
+    // Split the URL by '/'
+    const parts = url.split('/');
+
+    // Get the last part of the URL
+    const lastPart = parts.pop();
+
+    // Check if the last part of the URL matches the image extension regex
+    const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
+    if (imageExtensionRegex.test(lastPart)) {
+      return lastPart;
+    }
+
+    // If the regex test fails, return an empty string
+    return '';
+  }),
+}));
+
 const generate = jest.fn();
 OpenAI.mockImplementation(() => ({
   images: {
@@ -20,15 +40,14 @@ jest.mock('fs', () => {
   };
 });
 
-jest.mock('../../saveImageFromUrl', () => {
-  return jest.fn();
-});
-
 jest.mock('path', () => {
   return {
     resolve: jest.fn(),
     join: jest.fn(),
     relative: jest.fn(),
+    extname: jest.fn().mockImplementation((filename) => {
+      return filename.slice(filename.lastIndexOf('.'));
+    }),
   };
 });
 
@@ -47,7 +66,7 @@ describe('DALLE3', () => {
     jest.resetModules();
     process.env = { ...originalEnv, DALLE_API_KEY: mockApiKey };
     // Instantiate DALLE3 for tests that do not depend on DALLE3_SYSTEM_PROMPT
-    dalle = new DALLE3();
+    dalle = new DALLE3({ processFileURL });
   });
 
   afterEach(() => {
@@ -56,7 +75,8 @@ describe('DALLE3', () => {
     process.env = originalEnv;
   });
 
-  it('should throw an error if DALLE_API_KEY is missing', () => {
+  it('should throw an error if all potential API keys are missing', () => {
+    delete process.env.DALLE3_API_KEY;
     delete process.env.DALLE_API_KEY;
     expect(() => new DALLE3()).toThrow('Missing DALLE_API_KEY environment variable.');
   });
@@ -69,10 +89,8 @@ describe('DALLE3', () => {
 
   it('should generate markdown image URL correctly', () => {
     const imageName = 'test.png';
-    path.join.mockReturnValue('images/test.png');
-    path.relative.mockReturnValue('images/test.png');
-    const markdownImage = dalle.getMarkdownImageUrl(imageName);
-    expect(markdownImage).toBe('![generated image](/images/test.png)');
+    const markdownImage = dalle.wrapInMarkdown(imageName);
+    expect(markdownImage).toBe('![generated image](test.png)');
   });
 
   it('should call OpenAI API with correct parameters', async () => {
@@ -92,11 +110,9 @@ describe('DALLE3', () => {
     };
 
     generate.mockResolvedValue(mockResponse);
-    saveImageFromUrl.mockResolvedValue(true);
-    fs.existsSync.mockReturnValue(true);
-    path.resolve.mockReturnValue('/fakepath/images');
-    path.join.mockReturnValue('/fakepath/images/img-test.png');
-    path.relative.mockReturnValue('images/img-test.png');
+    processFileURL.mockResolvedValue({
+      filepath: 'http://example.com/img-test.png',
+    });
 
     const result = await dalle._call(mockData);
 
@@ -108,6 +124,7 @@ describe('DALLE3', () => {
       prompt: mockData.prompt,
       n: 1,
     });
+
     expect(result).toContain('![generated image]');
   });
 
@@ -134,7 +151,7 @@ describe('DALLE3', () => {
     await expect(dalle._call(mockData)).rejects.toThrow('Missing required field: prompt');
   });
 
-  it('should log to console if no image name is found in the URL', async () => {
+  it('should log appropriate debug values', async () => {
     const mockData = {
       prompt: 'A test prompt',
     };
@@ -145,27 +162,17 @@ describe('DALLE3', () => {
         },
       ],
     };
-    console.log = jest.fn(); // Mock console.log
-    generate.mockResolvedValue(mockResponse);
-    await dalle._call(mockData);
-    expect(console.log).toHaveBeenCalledWith('No image name found in the string.');
-  });
 
-  it('should create the directory if it does not exist', async () => {
-    const mockData = {
-      prompt: 'A test prompt',
-    };
-    const mockResponse = {
-      data: [
-        {
-          url: 'http://example.com/img-test.png',
-        },
-      ],
-    };
     generate.mockResolvedValue(mockResponse);
-    fs.existsSync.mockReturnValue(false); // Simulate directory does not exist
     await dalle._call(mockData);
-    expect(fs.mkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true });
+    expect(logger.debug).toHaveBeenCalledWith('[DALL-E-3]', {
+      data: { url: 'http://example.com/invalid-url' },
+      theImageUrl: 'http://example.com/invalid-url',
+      extension: expect.any(String),
+      imageBasename: expect.any(String),
+      imageExt: expect.any(String),
+      imageName: expect.any(String),
+    });
   });
 
   it('should log an error and return the image URL if there is an error saving the image', async () => {
@@ -181,10 +188,25 @@ describe('DALLE3', () => {
     };
     const error = new Error('Error while saving the image');
     generate.mockResolvedValue(mockResponse);
-    saveImageFromUrl.mockRejectedValue(error);
-    console.error = jest.fn(); // Mock console.error
+    processFileURL.mockRejectedValue(error);
     const result = await dalle._call(mockData);
-    expect(console.error).toHaveBeenCalledWith('Error while saving the image:', error);
-    expect(result).toBe(mockResponse.data[0].url);
+    expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
+    expect(result).toBe('Failed to save the image locally. Error while saving the image');
+  });
+
+  it('should handle error when saving image to Firebase Storage fails', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+    };
+    const mockImageUrl = 'http://example.com/img-test.png';
+    const mockResponse = { data: [{ url: mockImageUrl }] };
+    const error = new Error('Error while saving to Firebase');
+    generate.mockResolvedValue(mockResponse);
+    processFileURL.mockRejectedValue(error);
+
+    const result = await dalle._call(mockData);
+
+    expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
+    expect(result).toContain('Failed to save the image');
   });
 });

api/app/clients/tools/util/handleOpenAIErrors.js

@@ -1,4 +1,5 @@
 const OpenAI = require('openai');
+const { logger } = require('~/config');
 
 /**
  * Handles errors that may occur when making requests to OpenAI's API.
@@ -12,14 +13,14 @@ const OpenAI = require('openai');
  */
 async function handleOpenAIErrors(err, errorCallback, context = 'stream') {
   if (err instanceof OpenAI.APIError && err?.message?.includes('abort')) {
-    console.warn(`[OpenAIClient.chatCompletion][${context}] Aborted Message`);
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] Aborted Message`);
   }
   if (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason')) {
-    console.warn(`[OpenAIClient.chatCompletion][${context}] Missing finish_reason`);
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] Missing finish_reason`);
   } else if (err instanceof OpenAI.APIError) {
-    console.warn(`[OpenAIClient.chatCompletion][${context}] API Error`);
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] API error`);
   } else {
-    console.warn(`[OpenAIClient.chatCompletion][${context}] Unhandled error type`);
+    logger.warn(`[OpenAIClient.chatCompletion][${context}] Unhandled error type`);
   }
 
   if (errorCallback) {

api/app/clients/tools/util/handleTools.js

@@ -1,31 +1,32 @@
-const { getUserPluginAuthValue } = require('../../../../server/services/PluginService');
-const { OpenAIEmbeddings } = require('langchain/embeddings/openai');
 const { ZapierToolKit } = require('langchain/agents');
-const { SerpAPI, ZapierNLAWrapper } = require('langchain/tools');
-const { ChatOpenAI } = require('langchain/chat_models/openai');
 const { Calculator } = require('langchain/tools/calculator');
 const { WebBrowser } = require('langchain/tools/webbrowser');
+const { SerpAPI, ZapierNLAWrapper } = require('langchain/tools');
+const { OpenAIEmbeddings } = require('langchain/embeddings/openai');
+const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const {
   availableTools,
-  CodeInterpreter,
-  AIPluginTool,
+  // Basic Tools
+  CodeBrew,
+  AzureAISearch,
   GoogleSearchAPI,
   WolframAlphaAPI,
-  StructuredWolfram,
-  HttpRequestTool,
   OpenAICreateImage,
   StableDiffusionAPI,
+  // Structured Tools
   DALLE3,
-  StructuredSD,
-  AzureCognitiveSearch,
-  StructuredACS,
   E2BTools,
   CodeSherpa,
+  StructuredSD,
+  StructuredACS,
   CodeSherpaTools,
-  CodeBrew,
+  TraversaalSearch,
+  StructuredWolfram,
+  TavilySearchResults,
 } = require('../');
-const { loadSpecs } = require('./loadSpecs');
 const { loadToolSuite } = require('./loadToolSuite');
+const { loadSpecs } = require('./loadSpecs');
+const { logger } = require('~/config');
 
 const getOpenAIKey = async (options, user) => {
   let openAIApiKey = options.openAIApiKey ?? process.env.OPENAI_API_KEY;
@@ -33,6 +34,14 @@ const getOpenAIKey = async (options, user) => {
   return openAIApiKey || (await getUserPluginAuthValue(user, 'OPENAI_API_KEY'));
 };
 
+/**
+ * Validates the availability and authentication of tools for a user based on environment variables or user-specific plugin authentication values.
+ * Tools without required authentication or with valid authentication are considered valid.
+ *
+ * @param {Object} user The user object for whom to validate tool access.
+ * @param {Array<string>} tools An array of tool identifiers to validate. Defaults to an empty array.
+ * @returns {Promise<Array<string>>} A promise that resolves to an array of valid tool identifiers.
+ */
 const validateTools = async (user, tools = []) => {
   try {
     const validToolsSet = new Set(tools);
@@ -40,16 +49,34 @@ const validateTools = async (user, tools = []) => {
       validToolsSet.has(tool.pluginKey),
     );
 
+    /**
+     * Validates the credentials for a given auth field or set of alternate auth fields for a tool.
+     * If valid admin or user authentication is found, the function returns early. Otherwise, it removes the tool from the set of valid tools.
+     *
+     * @param {string} authField The authentication field or fields (separated by "||" for alternates) to validate.
+     * @param {string} toolName The identifier of the tool being validated.
+     */
     const validateCredentials = async (authField, toolName) => {
-      const adminAuth = process.env[authField];
-      if (adminAuth && adminAuth.length > 0) {
-        return;
+      const fields = authField.split('||');
+      for (const field of fields) {
+        const adminAuth = process.env[field];
+        if (adminAuth && adminAuth.length > 0) {
+          return;
+        }
+
+        let userAuth = null;
+        try {
+          userAuth = await getUserPluginAuthValue(user, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !userAuth) {
+            throw err;
+          }
+        }
+        if (userAuth && userAuth.length > 0) {
+          return;
+        }
       }
 
-      const userAuth = await getUserPluginAuthValue(user, authField);
-      if (userAuth && userAuth.length > 0) {
-        return;
-      }
       validToolsSet.delete(toolName);
     };
 
@@ -65,24 +92,59 @@ const validateTools = async (user, tools = []) => {
 
     return Array.from(validToolsSet.values());
   } catch (err) {
-    console.log('There was a problem validating tools', err);
-    throw new Error(err);
+    logger.error('[validateTools] There was a problem validating tools', err);
+    throw new Error('There was a problem validating tools');
   }
 };
 
-const loadToolWithAuth = async (user, authFields, ToolConstructor, options = {}) => {
+/**
+ * Initializes a tool with authentication values for the given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {string} userId The user ID for which the tool is being loaded.
+ * @param {Array<string>} authFields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+ * @param {typeof import('langchain/tools').Tool} ToolConstructor The constructor function for the tool to be initialized.
+ * @param {Object} options Optional parameters to be passed to the tool constructor alongside authentication values.
+ * @returns {Function} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication.
+ */
+const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => {
   return async function () {
     let authValues = {};
 
-    for (const authField of authFields) {
-      let authValue = process.env[authField];
-      if (!authValue) {
-        authValue = await getUserPluginAuthValue(user, authField);
+    /**
+     * Finds the first non-empty value for the given authentication field, supporting alternate fields.
+     * @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+     * @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found.
+     */
+    const findAuthValue = async (fields) => {
+      for (const field of fields) {
+        let value = process.env[field];
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+        try {
+          value = await getUserPluginAuthValue(userId, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !value) {
+            throw err;
+          }
+        }
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+      }
+      return null;
+    };
+
+    for (let authField of authFields) {
+      const fields = authField.split('||');
+      const result = await findAuthValue(fields);
+      if (result) {
+        authValues[result.authField] = result.authValue;
       }
-      authValues[authField] = authValue;
     }
 
-    return new ToolConstructor({ ...options, ...authValues });
+    return new ToolConstructor({ ...options, ...authValues, userId });
   };
 };
 
@@ -93,16 +155,18 @@ const loadTools = async ({
   returnMap = false,
   tools = [],
   options = {},
+  skipSpecs = false,
 }) => {
   const toolConstructors = {
+    tavily_search_results_json: TavilySearchResults,
     calculator: Calculator,
-    codeinterpreter: CodeInterpreter,
     google: GoogleSearchAPI,
     wolfram: functions ? StructuredWolfram : WolframAlphaAPI,
     'dall-e': OpenAICreateImage,
     'stable-diffusion': functions ? StructuredSD : StableDiffusionAPI,
-    'azure-cognitive-search': functions ? StructuredACS : AzureCognitiveSearch,
+    'azure-ai-search': functions ? StructuredACS : AzureAISearch,
     CodeBrew: CodeBrew,
+    traversaal_search: TraversaalSearch,
   };
 
   const openAIApiKey = await getOpenAIKey(options, user);
@@ -163,15 +227,6 @@ const loadTools = async ({
       const zapier = new ZapierNLAWrapper({ apiKey });
       return ZapierToolKit.fromZapierNLAWrapper(zapier);
     },
-    plugins: async () => {
-      return [
-        new HttpRequestTool(),
-        await AIPluginTool.fromPluginUrl(
-          'https://www.klarna.com/.well-known/ai-plugin.json',
-          new ChatOpenAI({ openAIApiKey: options.openAIApiKey, temperature: 0 }),
-        ),
-      ];
-    },
   };
 
   const requestedTools = {};
@@ -181,8 +236,19 @@ const loadTools = async ({
     toolConstructors.codesherpa = CodeSherpa;
   }
 
+  const imageGenOptions = {
+    req: options.req,
+    fileStrategy: options.fileStrategy,
+    processFileURL: options.processFileURL,
+    returnMetadata: options.returnMetadata,
+    uploadImageBuffer: options.uploadImageBuffer,
+  };
+
   const toolOptions = {
     serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' },
+    dalle: imageGenOptions,
+    'dall-e': imageGenOptions,
+    'stable-diffusion': imageGenOptions,
   };
 
   const toolAuthFields = {};
@@ -205,7 +271,7 @@ const loadTools = async ({
 
     if (toolConstructors[tool]) {
       const options = toolOptions[tool] || {};
-      const toolInstance = await loadToolWithAuth(
+      const toolInstance = loadToolWithAuth(
         user,
         toolAuthFields[tool],
         toolConstructors[tool],
@@ -221,7 +287,7 @@ const loadTools = async ({
   }
 
   let specs = null;
-  if (functions && remainingTools.length > 0) {
+  if (functions && remainingTools.length > 0 && skipSpecs !== true) {
     specs = await loadSpecs({
       llm: model,
       user,
@@ -248,6 +314,9 @@ const loadTools = async ({
   let result = [];
   for (const tool of tools) {
     const validTool = requestedTools[tool];
+    if (!validTool) {
+      continue;
+    }
     const plugin = await validTool();
 
     if (Array.isArray(plugin)) {
@@ -261,6 +330,7 @@ const loadTools = async ({
 };
 
 module.exports = {
+  loadToolWithAuth,
   validateTools,
   loadTools,
 };

api/app/clients/tools/util/handleTools.test.js

@@ -4,26 +4,33 @@ const mockUser = {
   findByIdAndDelete: jest.fn(),
 };
 
-var mockPluginService = {
+const mockPluginService = {
   updateUserPluginAuth: jest.fn(),
   deleteUserPluginAuth: jest.fn(),
   getUserPluginAuthValue: jest.fn(),
 };
 
-jest.mock('../../../../models/User', () => {
+jest.mock('~/models/User', () => {
   return function () {
     return mockUser;
   };
 });
 
-jest.mock('../../../../server/services/PluginService', () => mockPluginService);
+jest.mock('~/server/services/PluginService', () => mockPluginService);
 
-const User = require('../../../../models/User');
-const { validateTools, loadTools } = require('./');
-const PluginService = require('../../../../server/services/PluginService');
-const { BaseChatModel } = require('langchain/chat_models/openai');
 const { Calculator } = require('langchain/tools/calculator');
-const { availableTools, OpenAICreateImage, GoogleSearchAPI, StructuredSD } = require('../');
+const { BaseChatModel } = require('langchain/chat_models/openai');
+
+const User = require('~/models/User');
+const PluginService = require('~/server/services/PluginService');
+const { validateTools, loadTools, loadToolWithAuth } = require('./handleTools');
+const {
+  availableTools,
+  OpenAICreateImage,
+  GoogleSearchAPI,
+  StructuredSD,
+  WolframAlphaAPI,
+} = require('../');
 
 describe('Tool Handlers', () => {
   let fakeUser;
@@ -44,7 +51,10 @@ describe('Tool Handlers', () => {
     });
     mockPluginService.updateUserPluginAuth.mockImplementation(
       (userId, authField, _pluginKey, credential) => {
-        userAuthValues[`${userId}-${authField}`] = credential;
+        const fields = authField.split('||');
+        fields.forEach((field) => {
+          userAuthValues[`${userId}-${field}`] = credential;
+        });
       },
     );
 
@@ -53,6 +63,7 @@ describe('Tool Handlers', () => {
       username: 'fakeuser',
       email: 'fakeuser@example.com',
       emailVerified: false,
+      // file deepcode ignore NoHardcodedPasswords/test: fake value
       password: 'fakepassword123',
       avatar: '',
       provider: 'local',
@@ -133,6 +144,18 @@ describe('Tool Handlers', () => {
       loadTool2 = toolFunctions[sampleTools[1]];
       loadTool3 = toolFunctions[sampleTools[2]];
     });
+
+    let originalEnv;
+
+    beforeEach(() => {
+      originalEnv = process.env;
+      process.env = { ...originalEnv };
+    });
+
+    afterEach(() => {
+      process.env = originalEnv;
+    });
+
     it('returns the expected load functions for requested tools', async () => {
       expect(loadTool1).toBeDefined();
       expect(loadTool2).toBeDefined();
@@ -149,6 +172,86 @@ describe('Tool Handlers', () => {
       expect(authTool).toBeInstanceOf(ToolClass);
       expect(tool).toBeInstanceOf(ToolClass2);
     });
+
+    it('should initialize an authenticated tool with primary auth field', async () => {
+      process.env.DALLE2_API_KEY = 'mocked_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool with alternate auth field when primary is missing', async () => {
+      delete process.env.DALLE2_API_KEY; // Ensure the primary key is not set
+      process.env.DALLE_API_KEY = 'mocked_alternate_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'DALLE2_API_KEY',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when env vars are missing', async () => {
+      mockPluginService.updateUserPluginAuth('userId', 'DALLE_API_KEY', 'dalle', 'mocked_api_key');
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(2);
+    });
+
+    it('should initialize an authenticated tool with singular auth field', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool when env var is set', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when singular env var is missing', async () => {
+      delete process.env.WOLFRAM_APP_ID; // Ensure the environment variable is not set
+      mockPluginService.getUserPluginAuthValue.mockResolvedValue('mocked_user_auth_value');
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
     it('should throw an error for an unauthenticated tool', async () => {
       try {
         await loadTool2();

api/app/clients/tools/util/loadSpecs.js

@@ -1,7 +1,8 @@
 const fs = require('fs');
 const path = require('path');
 const { z } = require('zod');
-const { createOpenAPIPlugin } = require('../dynamic/OpenAPIPlugin');
+const { logger } = require('~/config');
+const { createOpenAPIPlugin } = require('~/app/clients/tools/dynamic/OpenAPIPlugin');
 
 // The minimum Manifest definition
 const ManifestDefinition = z.object({
@@ -26,28 +27,17 @@ const ManifestDefinition = z.object({
   legal_info_url: z.string().optional(),
 });
 
-function validateJson(json, verbose = true) {
+function validateJson(json) {
   try {
     return ManifestDefinition.parse(json);
   } catch (error) {
-    if (verbose) {
-      console.debug('validateJson error', error);
-    }
+    logger.debug('[validateJson] manifest parsing error', error);
     return false;
   }
 }
 
 // omit the LLM to return the well known jsons as objects
-async function loadSpecs({
-  llm,
-  user,
-  message,
-  tools = [],
-  map = false,
-  memory,
-  signal,
-  verbose = false,
-}) {
+async function loadSpecs({ llm, user, message, tools = [], map = false, memory, signal }) {
   const directoryPath = path.join(__dirname, '..', '.well-known');
   let files = [];
 
@@ -60,7 +50,7 @@ async function loadSpecs({
       await fs.promises.access(filePath, fs.constants.F_OK);
       files.push(tools[i] + '.json');
     } catch (err) {
-      console.error(`File ${tools[i] + '.json'} does not exist`);
+      logger.error(`[loadSpecs] File ${tools[i] + '.json'} does not exist`, err);
     }
   }
 
@@ -73,9 +63,7 @@ async function loadSpecs({
   const validJsons = [];
   const constructorMap = {};
 
-  if (verbose) {
-    console.debug('files', files);
-  }
+  logger.debug('[validateJson] files', files);
 
   for (const file of files) {
     if (path.extname(file) === '.json') {
@@ -84,7 +72,7 @@ async function loadSpecs({
       const json = JSON.parse(fileContent);
 
       if (!validateJson(json)) {
-        verbose && console.debug('Invalid json', json);
+        logger.debug('[validateJson] Invalid json', json);
         continue;
       }
 
@@ -97,13 +85,12 @@ async function loadSpecs({
             memory,
             signal,
             user,
-            verbose,
           });
         continue;
       }
 
       if (llm) {
-        validJsons.push(createOpenAPIPlugin({ data: json, llm, verbose }));
+        validJsons.push(createOpenAPIPlugin({ data: json, llm }));
         continue;
       }
 
@@ -117,10 +104,8 @@ async function loadSpecs({
 
   const plugins = (await Promise.all(validJsons)).filter((plugin) => plugin);
 
-  // if (verbose) {
-  //   console.debug('plugins', plugins);
-  //   console.debug(plugins[0].name);
-  // }
+  //   logger.debug('[validateJson] plugins', plugins);
+  //   logger.debug(plugins[0].name);
 
   return plugins;
 }

api/app/clients/tools/util/loadToolSuite.js

@@ -1,17 +1,49 @@
-const { getUserPluginAuthValue } = require('../../../../server/services/PluginService');
+const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const { availableTools } = require('../');
+const { logger } = require('~/config');
 
-const loadToolSuite = async ({ pluginKey, tools, user, options }) => {
+/**
+ * Loads a suite of tools with authentication values for a given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {Object} params Parameters for loading the tool suite.
+ * @param {string} params.pluginKey Key identifying the plugin whose tools are to be loaded.
+ * @param {Array<Function>} params.tools Array of tool constructor functions.
+ * @param {Object} params.user User object for whom the tools are being loaded.
+ * @param {Object} [params.options={}] Optional parameters to be passed to each tool constructor.
+ * @returns {Promise<Array>} A promise that resolves to an array of instantiated tools.
+ */
+const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
   const authConfig = availableTools.find((tool) => tool.pluginKey === pluginKey).authConfig;
   const suite = [];
   const authValues = {};
 
-  for (const auth of authConfig) {
-    let authValue = process.env[auth.authField];
-    if (!authValue) {
-      authValue = await getUserPluginAuthValue(user, auth.authField);
+  const findAuthValue = async (authField) => {
+    const fields = authField.split('||');
+    for (const field of fields) {
+      let value = process.env[field];
+      if (value) {
+        return value;
+      }
+      try {
+        value = await getUserPluginAuthValue(user, field);
+        if (value) {
+          return value;
+        }
+      } catch (err) {
+        logger.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
+      }
+    }
+    return null;
+  };
+
+  for (const auth of authConfig) {
+    const authValue = await findAuthValue(auth.authField);
+    if (authValue !== null) {
+      authValues[auth.authField] = authValue;
+    } else {
+      logger.warn(`[loadToolSuite] No auth value found for ${auth.authField}`);
     }
-    authValues[auth.authField] = authValue;
   }
 
   for (const tool of tools) {

api/app/titleConvoBing.js

@@ -1,5 +1,6 @@
-const { isEnabled } = require('../server/utils');
 const throttle = require('lodash/throttle');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
 
 const titleConvo = async ({ text, response }) => {
   let title = 'New Chat';
@@ -30,11 +31,10 @@ const titleConvo = async ({ text, response }) => {
     const res = await titleGenerator.sendMessage(titlePrompt, options);
     title = res.response.replace(/Title: /, '').replace(/[".]/g, '');
   } catch (e) {
-    console.error(e);
-    console.log('There was an issue generating title, see error above');
+    logger.error('There was an issue generating title with BingAI', e);
   }
 
-  console.log('CONVERSATION TITLE', title);
+  logger.debug('[/ask/bingAI] CONVERSATION TITLE: ' + title);
   return title;
 };
 

api/cache/banViolation.js

@@ -1,6 +1,8 @@
-const Session = require('../models/Session');
+const Session = require('~/models/Session');
 const getLogStores = require('./getLogStores');
-const { isEnabled, math, removePorts } = require('../server/utils');
+const { isEnabled, math, removePorts } = require('~/server/utils');
+const { logger } = require('~/config');
+
 const { BAN_VIOLATIONS, BAN_INTERVAL } = process.env ?? {};
 const interval = math(BAN_INTERVAL, 20);
 
@@ -54,7 +56,7 @@ const banViolation = async (req, res, errorMessage) => {
   }
 
   req.ip = removePorts(req);
-  console.log(
+  logger.info(
     `[BAN] Banning user ${user_id} ${req.ip ? `@ ${req.ip} ` : ''}for ${
       duration / 1000 / 60
     } minutes`,

api/cache/getLogStores.js

@@ -1,8 +1,10 @@
 const Keyv = require('keyv');
-const keyvMongo = require('./keyvMongo');
-const keyvRedis = require('./keyvRedis');
-const { math, isEnabled } = require('../server/utils');
+const { CacheKeys, ViolationTypes } = require('librechat-data-provider');
 const { logFile, violationFile } = require('./keyvFiles');
+const { math, isEnabled } = require('~/server/utils');
+const keyvRedis = require('./keyvRedis');
+const keyvMongo = require('./keyvMongo');
+
 const { BAN_DURATION, USE_REDIS } = process.env ?? {};
 
 const duration = math(BAN_DURATION, 7200000);
@@ -17,35 +19,60 @@ const pending_req = isEnabled(USE_REDIS)
   ? new Keyv({ store: keyvRedis })
   : new Keyv({ namespace: 'pending_req' });
 
+const config = isEnabled(USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.CONFIG_STORE });
+
+const tokenConfig = isEnabled(USE_REDIS) // ttl: 30 minutes
+  ? new Keyv({ store: keyvRedis, ttl: 1800000 })
+  : new Keyv({ namespace: CacheKeys.TOKEN_CONFIG, ttl: 1800000 });
+
+const genTitle = isEnabled(USE_REDIS) // ttl: 2 minutes
+  ? new Keyv({ store: keyvRedis, ttl: 120000 })
+  : new Keyv({ namespace: CacheKeys.GEN_TITLE, ttl: 120000 });
+
+const modelQueries = isEnabled(process.env.USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.MODEL_QUERIES });
+
+const abortKeys = isEnabled(USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.ABORT_KEYS, ttl: 600000 });
+
 const namespaces = {
+  [CacheKeys.CONFIG_STORE]: config,
   pending_req,
   ban: new Keyv({ store: keyvMongo, namespace: 'bans', ttl: duration }),
   general: new Keyv({ store: logFile, namespace: 'violations' }),
   concurrent: createViolationInstance('concurrent'),
   non_browser: createViolationInstance('non_browser'),
   message_limit: createViolationInstance('message_limit'),
-  token_balance: createViolationInstance('token_balance'),
+  token_balance: createViolationInstance(ViolationTypes.TOKEN_BALANCE),
   registrations: createViolationInstance('registrations'),
+  [ViolationTypes.FILE_UPLOAD_LIMIT]: createViolationInstance(ViolationTypes.FILE_UPLOAD_LIMIT),
+  [ViolationTypes.ILLEGAL_MODEL_REQUEST]: createViolationInstance(
+    ViolationTypes.ILLEGAL_MODEL_REQUEST,
+  ),
   logins: createViolationInstance('logins'),
+  [CacheKeys.ABORT_KEYS]: abortKeys,
+  [CacheKeys.TOKEN_CONFIG]: tokenConfig,
+  [CacheKeys.GEN_TITLE]: genTitle,
+  [CacheKeys.MODEL_QUERIES]: modelQueries,
 };
 
 /**
  * Returns the keyv cache specified by type.
  * If an invalid type is passed, an error will be thrown.
  *
- * @module getLogStores
- * @requires keyv - a simple key-value storage that allows you to easily switch out storage adapters.
- * @requires keyvFiles - a module that includes the logFile and violationFile.
- *
- * @param {string} type - The type of violation, which can be 'concurrent', 'message_limit', 'registrations' or 'logins'.
- * @returns {Keyv} - If a valid type is passed, returns an object containing the logs for violations of the specified type.
- * @throws Will throw an error if an invalid violation type is passed.
+ * @param {string} key - The key for the namespace to access
+ * @returns {Keyv} - If a valid key is passed, returns an object containing the cache store of the specified key.
+ * @throws Will throw an error if an invalid key is passed.
  */
-const getLogStores = (type) => {
-  if (!type || !namespaces[type]) {
-    throw new Error(`Invalid store type: ${type}`);
+const getLogStores = (key) => {
+  if (!key || !namespaces[key]) {
+    throw new Error(`Invalid store key: ${key}`);
   }
-  return namespaces[type];
+  return namespaces[key];
 };
 
 module.exports = getLogStores;

api/cache/keyvMongo.js

@@ -1,7 +1,9 @@
 const KeyvMongo = require('@keyv/mongo');
+const { logger } = require('~/config');
+
 const { MONGO_URI } = process.env ?? {};
 
 const keyvMongo = new KeyvMongo(MONGO_URI, { collection: 'logs' });
-keyvMongo.on('error', (err) => console.error('KeyvMongo connection error:', err));
+keyvMongo.on('error', (err) => logger.error('KeyvMongo connection error:', err));
 
 module.exports = keyvMongo;

api/cache/keyvRedis.js

@@ -1,14 +1,20 @@
 const KeyvRedis = require('@keyv/redis');
+const { logger } = require('~/config');
+const { isEnabled } = require('~/server/utils');
 
-const { REDIS_URI } = process.env;
+const { REDIS_URI, USE_REDIS } = process.env;
 
 let keyvRedis;
 
-if (REDIS_URI) {
+if (REDIS_URI && isEnabled(USE_REDIS)) {
   keyvRedis = new KeyvRedis(REDIS_URI, { useRedisSets: false });
-  keyvRedis.on('error', (err) => console.error('KeyvRedis connection error:', err));
+  keyvRedis.on('error', (err) => logger.error('KeyvRedis connection error:', err));
+  keyvRedis.setMaxListeners(20);
+  logger.info(
+    '[Optional] Redis initialized. Note: Redis support is experimental. If you have issues, disable it. Cache needs to be flushed for values to refresh.',
+  );
 } else {
-  // console.log('REDIS_URI not provided. Redis module will not be initialized.');
+  logger.info('[Optional] Redis not initialized. Note: Redis support is experimental.');
 }
 
 module.exports = keyvRedis;

api/config/index.js

@@ -0,0 +1,5 @@
+const logger = require('./winston');
+
+module.exports = {
+  logger,
+};

api/config/meiliLogger.js

@@ -0,0 +1,78 @@
+const path = require('path');
+const winston = require('winston');
+require('winston-daily-rotate-file');
+
+const logDir = path.join(__dirname, '..', 'logs');
+
+const { NODE_ENV } = process.env;
+
+const levels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  http: 3,
+  verbose: 4,
+  debug: 5,
+  activity: 6,
+  silly: 7,
+};
+
+winston.addColors({
+  info: 'green', // fontStyle color
+  warn: 'italic yellow',
+  error: 'red',
+  debug: 'blue',
+});
+
+const level = () => {
+  const env = NODE_ENV || 'development';
+  const isDevelopment = env === 'development';
+  return isDevelopment ? 'debug' : 'warn';
+};
+
+const fileFormat = winston.format.combine(
+  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.errors({ stack: true }),
+  winston.format.splat(),
+);
+
+const transports = [
+  new winston.transports.DailyRotateFile({
+    level: 'debug',
+    filename: `${logDir}/meiliSync-%DATE%.log`,
+    datePattern: 'YYYY-MM-DD',
+    zippedArchive: true,
+    maxSize: '20m',
+    maxFiles: '14d',
+    format: fileFormat,
+  }),
+];
+
+// if (NODE_ENV !== 'production') {
+//   transports.push(
+//     new winston.transports.Console({
+//       format: winston.format.combine(winston.format.colorize(), winston.format.simple()),
+//     }),
+//   );
+// }
+
+const consoleFormat = winston.format.combine(
+  winston.format.colorize({ all: true }),
+  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`),
+);
+
+transports.push(
+  new winston.transports.Console({
+    level: 'info',
+    format: consoleFormat,
+  }),
+);
+
+const logger = winston.createLogger({
+  level: level(),
+  levels,
+  transports,
+});
+
+module.exports = logger;

api/config/parsers.js

@@ -0,0 +1,186 @@
+const { klona } = require('klona');
+const winston = require('winston');
+const traverse = require('traverse');
+
+const SPLAT_SYMBOL = Symbol.for('splat');
+const MESSAGE_SYMBOL = Symbol.for('message');
+
+const sensitiveKeys = [
+  /^(sk-)[^\s]+/, // OpenAI API key pattern
+  /(Bearer )[^\s]+/, // Header: Bearer token pattern
+  /(api-key:? )[^\s]+/, // Header: API key pattern
+  /(key=)[^\s]+/, // URL query param: sensitive key pattern (Google)
+];
+
+/**
+ * Determines if a given value string is sensitive and returns matching regex patterns.
+ *
+ * @param {string} valueStr - The value string to check.
+ * @returns {Array<RegExp>} An array of regex patterns that match the value string.
+ */
+function getMatchingSensitivePatterns(valueStr) {
+  if (valueStr) {
+    // Filter and return all regex patterns that match the value string
+    return sensitiveKeys.filter((regex) => regex.test(valueStr));
+  }
+  return [];
+}
+
+/**
+ * Redacts sensitive information from a console message.
+ *
+ * @param {string} str - The console message to be redacted.
+ * @returns {string} - The redacted console message.
+ */
+function redactMessage(str) {
+  if (!str) {
+    return '';
+  }
+
+  const patterns = getMatchingSensitivePatterns(str);
+
+  if (patterns.length === 0) {
+    return str;
+  }
+
+  patterns.forEach((pattern) => {
+    str = str.replace(pattern, '$1[REDACTED]');
+  });
+
+  return str;
+}
+
+/**
+ * Redacts sensitive information from log messages if the log level is 'error'.
+ * Note: Intentionally mutates the object.
+ * @param {Object} info - The log information object.
+ * @returns {Object} - The modified log information object.
+ */
+const redactFormat = winston.format((info) => {
+  if (info.level === 'error') {
+    info.message = redactMessage(info.message);
+    if (info[MESSAGE_SYMBOL]) {
+      info[MESSAGE_SYMBOL] = redactMessage(info[MESSAGE_SYMBOL]);
+    }
+  }
+  return info;
+});
+
+/**
+ * Truncates long strings, especially base64 image data, within log messages.
+ *
+ * @param {any} value - The value to be inspected and potentially truncated.
+ * @param {number} [length] - The length at which to truncate the value. Default: 100.
+ * @returns {any} - The truncated or original value.
+ */
+const truncateLongStrings = (value, length = 100) => {
+  if (typeof value === 'string') {
+    return value.length > length ? value.substring(0, length) + '... [truncated]' : value;
+  }
+
+  return value;
+};
+
+/**
+ * An array mapping function that truncates long strings (objects converted to JSON strings).
+ * @param {any} item - The item to be condensed.
+ * @returns {any} - The condensed item.
+ */
+const condenseArray = (item) => {
+  if (typeof item === 'string') {
+    return truncateLongStrings(JSON.stringify(item));
+  } else if (typeof item === 'object') {
+    return truncateLongStrings(JSON.stringify(item));
+  }
+  return item;
+};
+
+/**
+ * Formats log messages for debugging purposes.
+ * - Truncates long strings within log messages.
+ * - Condenses arrays by truncating long strings and objects as strings within array items.
+ * - Redacts sensitive information from log messages if the log level is 'error'.
+ * - Converts log information object to a formatted string.
+ *
+ * @param {Object} options - The options for formatting log messages.
+ * @param {string} options.level - The log level.
+ * @param {string} options.message - The log message.
+ * @param {string} options.timestamp - The timestamp of the log message.
+ * @param {Object} options.metadata - Additional metadata associated with the log message.
+ * @returns {string} - The formatted log message.
+ */
+const debugTraverse = winston.format.printf(({ level, message, timestamp, ...metadata }) => {
+  let msg = `${timestamp} ${level}: ${truncateLongStrings(message?.trim(), 150)}`;
+  try {
+    if (level !== 'debug') {
+      return msg;
+    }
+
+    if (!metadata) {
+      return msg;
+    }
+
+    const debugValue = metadata[SPLAT_SYMBOL]?.[0];
+
+    if (!debugValue) {
+      return msg;
+    }
+
+    if (debugValue && Array.isArray(debugValue)) {
+      msg += `\n${JSON.stringify(debugValue.map(condenseArray))}`;
+      return msg;
+    }
+
+    if (typeof debugValue !== 'object') {
+      return (msg += ` ${debugValue}`);
+    }
+
+    msg += '\n{';
+
+    const copy = klona(metadata);
+    traverse(copy).forEach(function (value) {
+      if (typeof this?.key === 'symbol') {
+        return;
+      }
+
+      let _parentKey = '';
+      const parent = this.parent;
+
+      if (typeof parent?.key !== 'symbol' && parent?.key) {
+        _parentKey = parent.key;
+      }
+
+      const parentKey = `${parent && parent.notRoot ? _parentKey + '.' : ''}`;
+
+      const tabs = `${parent && parent.notRoot ? '    ' : '  '}`;
+
+      const currentKey = this?.key ?? 'unknown';
+
+      if (this.isLeaf && typeof value === 'string') {
+        const truncatedText = truncateLongStrings(value);
+        msg += `\n${tabs}${parentKey}${currentKey}: ${JSON.stringify(truncatedText)},`;
+      } else if (this.notLeaf && Array.isArray(value) && value.length > 0) {
+        const currentMessage = `\n${tabs}// ${value.length} ${currentKey.replace(/s$/, '')}(s)`;
+        this.update(currentMessage, true);
+        msg += currentMessage;
+        const stringifiedArray = value.map(condenseArray);
+        msg += `\n${tabs}${parentKey}${currentKey}: [${stringifiedArray}],`;
+      } else if (this.isLeaf && typeof value === 'function') {
+        msg += `\n${tabs}${parentKey}${currentKey}: function,`;
+      } else if (this.isLeaf) {
+        msg += `\n${tabs}${parentKey}${currentKey}: ${value},`;
+      }
+    });
+
+    msg += '\n}';
+    return msg;
+  } catch (e) {
+    return (msg += `\n[LOGGER PARSING ERROR] ${e.message}`);
+  }
+});
+
+module.exports = {
+  redactFormat,
+  redactMessage,
+  debugTraverse,
+};

api/config/paths.js

@@ -0,0 +1,12 @@
+const path = require('path');
+
+module.exports = {
+  root: path.resolve(__dirname, '..', '..'),
+  uploads: path.resolve(__dirname, '..', '..', 'uploads'),
+  clientPath: path.resolve(__dirname, '..', '..', 'client'),
+  dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),
+  publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),
+  imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),
+  structuredTools: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'structured'),
+  pluginManifest: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'manifest.json'),
+};