Merge branch 'main' into re-add-download-audio

* style: bookmarks update; style(Files): minor update

* style: update conversation bookmarks

* style: fix w TableCell

.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM node:18-bullseye
+
+RUN useradd -m -s /bin/bash vscode
+RUN mkdir -p /workspaces && chown -R vscode:vscode /workspaces
+WORKDIR /workspaces

.devcontainer/devcontainer.json

@@ -13,5 +13,6 @@
     }
   },
   "postCreateCommand": "",
-  "features": { "ghcr.io/devcontainers/features/git:1": {} }
+  "features": { "ghcr.io/devcontainers/features/git:1": {} },
+  "remoteUser": "vscode"
 }

.devcontainer/docker-compose.yml

@@ -2,7 +2,9 @@ version: "3.8"
 
 services:
   app:
-    image: node:19-bullseye
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
     # restart: always
     links:
       - mongodb
@@ -30,8 +32,8 @@ services:
     # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. 
     # (Adding the "ports" property to this file will not forward from a Codespace.)
 
-    # Uncomment the next line to use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
-    # user: vscode
+    # Use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
+    user: vscode
 
     # Overrides default command so things don't shut down after the process ends.
     command: /bin/sh -c "while sleep 1000; do :; done"  

.env.example

@@ -1,21 +1,16 @@
-#=============================================================#
-#                   LibreChat Configuration                   #
-#=============================================================#
-# Please refer to the reference documentation for assistance  #
-# with configuring your LibreChat environment. The guide is   #
-# available both online and within your local LibreChat       #
-# directory:                                                  #
-# Online: https://docs.librechat.ai/install/dotenv.html       #
-# Locally: ./docs/install/dotenv.md                           #
-#=============================================================#
+#=====================================================================#
+#                       LibreChat Configuration                       #
+#=====================================================================#
+# Please refer to the reference documentation for assistance          #
+# with configuring your LibreChat environment.                        #
+#                                                                     #
+# https://www.librechat.ai/docs/configuration/dotenv                  #
+#=====================================================================#
 
 #==================================================#
 #               Server Configuration               #
 #==================================================#
 
-APP_TITLE=LibreChat
-# CUSTOM_FOOTER="My custom footer"
-
 HOST=localhost
 PORT=3080
 
@@ -26,6 +21,13 @@ DOMAIN_SERVER=http://localhost:3080
 
 NO_INDEX=true
 
+#===============#
+# JSON Logging  #
+#===============#
+
+# Use when process console logs in cloud deployment like GCP/AWS
+CONSOLE_JSON=false
+
 #===============#
 # Debug Logging #
 #===============#
@@ -40,38 +42,66 @@ DEBUG_CONSOLE=false
 # UID=1000
 # GID=1000
 
+#===============#
+# Configuration #
+#===============#
+# Use an absolute path, a relative path, or a URL
+
+# CONFIG_PATH="/alternative/path/to/librechat.yaml"
+
 #===================================================#
 #                     Endpoints                     #
 #===================================================#
 
-# ENDPOINTS=openAI,azureOpenAI,bingAI,chatGPTBrowser,google,gptPlugins,anthropic
+# ENDPOINTS=openAI,assistants,azureOpenAI,bingAI,google,gptPlugins,anthropic
 
 PROXY=
 
+#===================================#
+# Known Endpoints - librechat.yaml  #
+#===================================#
+# https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints
+
+# ANYSCALE_API_KEY=
+# APIPIE_API_KEY=
+# COHERE_API_KEY=
+# DATABRICKS_API_KEY=
+# FIREWORKS_API_KEY=
+# GROQ_API_KEY=
+# HUGGINGFACE_TOKEN=
+# MISTRAL_API_KEY=
+# OPENROUTER_KEY=
+# PERPLEXITY_API_KEY=
+# SHUTTLEAI_API_KEY=
+# TOGETHERAI_API_KEY=
+
 #============#
 # Anthropic  #
 #============#
 
 ANTHROPIC_API_KEY=user_provided
-ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2
+# ANTHROPIC_MODELS=claude-3-5-sonnet-20240620,claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
 # ANTHROPIC_REVERSE_PROXY=
 
 #============#
 # Azure      #
 #============#
 
-# AZURE_API_KEY=
-AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4
-# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo
-# PLUGINS_USE_AZURE="true"
 
-AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
+# Note: these variables are DEPRECATED
+# Use the `librechat.yaml` configuration for `azureOpenAI` instead
+# You may also continue to use them if you opt out of using the `librechat.yaml` configuration
 
-# AZURE_OPENAI_API_INSTANCE_NAME=
-# AZURE_OPENAI_API_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_VERSION=
-# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=
+# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated
+# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 # Deprecated
+# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE # Deprecated
+# AZURE_API_KEY= # Deprecated
+# AZURE_OPENAI_API_INSTANCE_NAME= # Deprecated
+# AZURE_OPENAI_API_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_VERSION= # Deprecated
+# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated
+# PLUGINS_USE_AZURE="true" # Deprecated
 
 #============#
 # BingAI     #
@@ -80,28 +110,41 @@ AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
 BINGAI_TOKEN=user_provided
 # BINGAI_HOST=https://cn.bing.com
 
-#============#
-# ChatGPT    #
-#============#
-
-CHATGPT_TOKEN=
-CHATGPT_MODELS=text-davinci-002-render-sha
-# CHATGPT_REVERSE_PROXY=<YOUR REVERSE PROXY>
-
 #============#
 # Google     #
 #============#
 
 GOOGLE_KEY=user_provided
-# GOOGLE_MODELS=gemini-pro,gemini-pro-vision,chat-bison,chat-bison-32k,codechat-bison,codechat-bison-32k,text-bison,text-bison-32k,text-unicorn,code-gecko,code-bison,code-bison-32k
 # GOOGLE_REVERSE_PROXY=
 
+# Gemini API
+# GOOGLE_MODELS=gemini-1.5-flash-latest,gemini-1.0-pro,gemini-1.0-pro-001,gemini-1.0-pro-latest,gemini-1.0-pro-vision-latest,gemini-1.5-pro-latest,gemini-pro,gemini-pro-vision
+
+# Vertex AI
+# GOOGLE_MODELS=gemini-1.5-flash-preview-0514,gemini-1.5-pro-preview-0514,gemini-1.0-pro-vision-001,gemini-1.0-pro-002,gemini-1.0-pro-001,gemini-pro-vision,gemini-1.0-pro
+
+# GOOGLE_TITLE_MODEL=gemini-pro
+
+# Google Gemini Safety Settings
+# NOTE (Vertex AI): You do not have access to the BLOCK_NONE setting by default.
+# To use this restricted HarmBlockThreshold setting, you will need to either:
+#
+# (a) Get access through an allowlist via your Google account team
+# (b) Switch your account type to monthly invoiced billing following this instruction:
+#     https://cloud.google.com/billing/docs/how-to/invoiced-billing
+#
+# GOOGLE_SAFETY_SEXUALLY_EXPLICIT=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_HATE_SPEECH=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_HARASSMENT=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_DANGEROUS_CONTENT=BLOCK_ONLY_HIGH
+
+
 #============#
 # OpenAI     #
 #============#
 
 OPENAI_API_KEY=user_provided
-# OPENAI_MODELS=gpt-3.5-turbo-1106,gpt-4-1106-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,gpt-4,gpt-4-0314,gpt-4-0613
+# OPENAI_MODELS=gpt-4o,gpt-4o-mini,gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k
 
 DEBUG_OPENAI=false
 
@@ -115,19 +158,37 @@ DEBUG_OPENAI=false
 
 # OPENAI_REVERSE_PROXY=
 
-# OPENAI_ORGANIZATION= 
+# OPENAI_ORGANIZATION=
+
+#====================#
+#   Assistants API   #
+#====================#
+
+ASSISTANTS_API_KEY=user_provided
+# ASSISTANTS_BASE_URL=
+# ASSISTANTS_MODELS=gpt-4o,gpt-4o-mini,gpt-3.5-turbo-0125,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-16k,gpt-3.5-turbo,gpt-4,gpt-4-0314,gpt-4-32k-0314,gpt-4-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-1106,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview
+
+#==========================#
+#   Azure Assistants API   #
+#==========================#
+
+# Note: You should map your credentials with custom variables according to your Azure OpenAI Configuration
+# The models for Azure Assistants are also determined by your Azure OpenAI configuration.
+
+# More info, including how to enable use of Assistants with Azure here:
+# https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints/azure#using-assistants-with-azure
 
 #============#
 # OpenRouter #
 #============#
-
+# !!!Warning: Use the variable above instead of this one. Using this one will override the OpenAI endpoint
 # OPENROUTER_API_KEY=
 
 #============#
 # Plugins    #
 #============#
 
-# PLUGIN_MODELS=gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,gpt-4,gpt-4-0314,gpt-4-0613
+# PLUGIN_MODELS=gpt-4o,gpt-4o-mini,gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613
 
 DEBUG_PLUGINS=true
 
@@ -147,24 +208,24 @@ AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
 
 # DALL·E
 #----------------
-# DALLE_API_KEY= # Key for both DALL-E-2 and DALL-E-3
-# DALLE3_API_KEY= # Key for DALL-E-3 only
-# DALLE2_API_KEY= # Key for DALL-E-2 only
-# DALLE3_SYSTEM_PROMPT="Your DALL-E-3 System Prompt here"
-# DALLE2_SYSTEM_PROMPT="Your DALL-E-2 System Prompt here"
-# DALLE_REVERSE_PROXY= # Reverse proxy for DALL-E-2 and DALL-E-3
-# DALLE3_BASEURL= # Base URL for DALL-E-3
-# DALLE2_BASEURL= # Base URL for DALL-E-2
+# DALLE_API_KEY=
+# DALLE3_API_KEY=
+# DALLE2_API_KEY=
+# DALLE3_SYSTEM_PROMPT=
+# DALLE2_SYSTEM_PROMPT=
+# DALLE_REVERSE_PROXY=
+# DALLE3_BASEURL=
+# DALLE2_BASEURL=
 
 # DALL·E (via Azure OpenAI)
 # Note: requires some of the variables above to be set
 #----------------
-# DALLE3_AZURE_API_VERSION= # Azure OpenAI API version for DALL-E-3
-# DALLE2_AZURE_API_VERSION= # Azure OpenAI API versiion for DALL-E-2
+# DALLE3_AZURE_API_VERSION=
+# DALLE2_AZURE_API_VERSION=
 
 # Google
 #-----------------
-GOOGLE_API_KEY=
+GOOGLE_SEARCH_API_KEY=
 GOOGLE_CSE_ID=
 
 # SerpAPI
@@ -175,6 +236,14 @@ SERPAPI_API_KEY=
 #-----------------
 SD_WEBUI_URL=http://host.docker.internal:7860
 
+# Tavily
+#-----------------
+TAVILY_API_KEY=
+
+# Traversaal
+#-----------------
+TRAVERSAAL_API_KEY=
+
 # WolframAlpha
 #-----------------
 WOLFRAM_APP_ID=
@@ -192,6 +261,14 @@ MEILI_NO_ANALYTICS=true
 MEILI_HOST=http://0.0.0.0:7700
 MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
 
+
+#==================================================#
+#          Speech to Text & Text to Speech         #
+#==================================================#
+
+STT_API_KEY=
+TTS_API_KEY=
+
 #===================================================#
 #                    User System                    #
 #===================================================#
@@ -202,7 +279,7 @@ MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
 
 OPENAI_MODERATION=false
 OPENAI_MODERATION_API_KEY=
-# OPENAI_MODERATION_REVERSE_PROXY=not working with some reverse proxys
+# OPENAI_MODERATION_REVERSE_PROXY=
 
 BAN_VIOLATIONS=true
 BAN_DURATION=1000 * 60 * 60 * 2
@@ -230,6 +307,8 @@ LIMIT_MESSAGE_USER=false
 MESSAGE_USER_MAX=40
 MESSAGE_USER_WINDOW=1
 
+ILLEGAL_MODEL_REQ_SCORE=5
+
 #========================#
 # Balance                #
 #========================#
@@ -244,6 +323,9 @@ ALLOW_EMAIL_LOGIN=true
 ALLOW_REGISTRATION=true
 ALLOW_SOCIAL_LOGIN=false
 ALLOW_SOCIAL_REGISTRATION=false
+ALLOW_PASSWORD_RESET=false
+# ALLOW_ACCOUNT_DELETION=true # note: enabled by default if omitted/commented out
+ALLOW_UNVERIFIED_EMAIL_LOGIN=true
 
 SESSION_EXPIRY=1000 * 60 * 15
 REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7
@@ -278,23 +360,39 @@ OPENID_ISSUER=
 OPENID_SESSION_SECRET=
 OPENID_SCOPE="openid profile email"
 OPENID_CALLBACK_URL=/oauth/openid/callback
+OPENID_REQUIRED_ROLE=
+OPENID_REQUIRED_ROLE_TOKEN_KIND=
+OPENID_REQUIRED_ROLE_PARAMETER_PATH=
 
 OPENID_BUTTON_LABEL=
 OPENID_IMAGE_URL=
 
+# LDAP
+LDAP_URL=
+LDAP_BIND_DN=
+LDAP_BIND_CREDENTIALS=
+LDAP_USER_SEARCH_BASE=
+LDAP_SEARCH_FILTER=mail={{username}}
+LDAP_CA_CERT_PATH=
+# LDAP_TLS_REJECT_UNAUTHORIZED=
+# LDAP_LOGIN_USES_USERNAME=true
+# LDAP_ID=
+# LDAP_USERNAME=
+# LDAP_FULL_NAME=
+
 #========================#
 # Email Password Reset   #
 #========================#
 
-EMAIL_SERVICE=                  
-EMAIL_HOST=                     
-EMAIL_PORT=25                   
-EMAIL_ENCRYPTION=               
-EMAIL_ENCRYPTION_HOSTNAME=      
-EMAIL_ALLOW_SELFSIGNED=         
-EMAIL_USERNAME=                 
-EMAIL_PASSWORD=                 
-EMAIL_FROM_NAME=                
+EMAIL_SERVICE=
+EMAIL_HOST=
+EMAIL_PORT=25
+EMAIL_ENCRYPTION=
+EMAIL_ENCRYPTION_HOSTNAME=
+EMAIL_ALLOW_SELFSIGNED=
+EMAIL_USERNAME=
+EMAIL_PASSWORD=
+EMAIL_FROM_NAME=
 EMAIL_FROM=noreply@librechat.ai
 
 #========================#
@@ -308,6 +406,26 @@ FIREBASE_STORAGE_BUCKET=
 FIREBASE_MESSAGING_SENDER_ID=
 FIREBASE_APP_ID=
 
+#========================#
+# Shared Links           #
+#========================#
+
+ALLOW_SHARED_LINKS=true
+ALLOW_SHARED_LINKS_PUBLIC=true
+
+#===================================================#
+#                        UI                         #
+#===================================================#
+
+APP_TITLE=LibreChat
+# CUSTOM_FOOTER="My custom footer"
+HELP_AND_FAQ_URL=https://librechat.ai
+
+# SHOW_BIRTHDAY_ICON=true
+
+# Google tag manager id
+#ANALYTICS_GTM_ID=user provided google tag manager id
+
 #==================================================#
 #                      Others                      #
 #==================================================#

.eslintrc.js

@@ -19,6 +19,7 @@ module.exports = {
     'e2e/playwright-report/**/*',
     'packages/data-provider/types/**/*',
     'packages/data-provider/dist/**/*',
+    'packages/data-provider/test_bundle/**/*',
     'data-node/**/*',
     'meili_data/**/*',
     'node_modules/**/*',
@@ -131,6 +132,19 @@ module.exports = {
         },
       ],
     },
+    {
+      files: './config/translations/**/*.ts',
+      parser: '@typescript-eslint/parser',
+      parserOptions: {
+        project: './config/translations/tsconfig.json',
+      },
+    },
+    {
+      files: ['./packages/data-provider/specs/**/*.ts'],
+      parserOptions: {
+        project: './packages/data-provider/tsconfig.spec.json',
+      },
+    },
   ],
   settings: {
     react: {

.github/CODE_OF_CONDUCT.md

@@ -60,7 +60,7 @@ representative at an online or offline event.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported to the community leaders responsible for enforcement here on GitHub or
-on the official [Discord Server](https://discord.gg/uDyZ5Tzhct). 
+on the official [Discord Server](https://discord.librechat.ai). 
 All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the

.github/CONTRIBUTING.md

@@ -8,7 +8,7 @@ If the feature you would like to contribute has not already received prior appro
 
 Please note that a pull request involving a feature that has not been reviewed and approved by the project maintainers may be rejected. We appreciate your understanding and cooperation.
 
-If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.gg/uDyZ5Tzhct), where you can engage with other contributors and seek guidance from the community.
+If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.librechat.ai), where you can engage with other contributors and seek guidance from the community.
 
 ## Our Standards
 
@@ -126,6 +126,18 @@ Apply the following naming conventions to branches, labels, and other Git-relate
       
    - **Current Stance**: At present, this backend transition is of lower priority and might not be pursued.
 
+## 7. Module Import Conventions
+
+- `npm` packages first, 
+     - from shortest line (top) to longest (bottom)
+
+- Followed by typescript types (pertains to data-provider and client workspaces)
+     - longest line (top) to shortest (bottom)
+     - types from package come first
+
+- Lastly, local imports
+     - longest line (top) to shortest (bottom)
+     - imports with alias `~` treated the same as relative import with respect to line length
 
 ---
 

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -50,7 +50,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -43,7 +43,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/ISSUE_TEMPLATE/QUESTION.yml

@@ -44,7 +44,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/SECURITY.md

@@ -12,7 +12,7 @@ When reporting a security vulnerability, you have the following options to reach
 
 - **Option 2: GitHub Issues**: You can initiate first contact via GitHub Issues. However, please note that initial contact through GitHub Issues should not include any sensitive details.
 
-- **Option 3: Discord Server**: You can join our [Discord community](https://discord.gg/5rbRxn4uME) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
+- **Option 3: Discord Server**: You can join our [Discord community](https://discord.librechat.ai) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
 
 _After the initial contact, we will establish a private communication channel for further discussion._
 
@@ -39,11 +39,11 @@ Please note that as a security-conscious community, we may not always disclose d
 
 This security policy applies to the following GitHub repository:
 
-- Repository: [LibreChat](https://github.com/danny-avila/LibreChat)
+- Repository: [LibreChat](https://github.librechat.ai)
 
 ## Contact
 
-If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.gg/NGaa9RPCft) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
+If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.librechat.ai) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
 
 ## Acknowledgments
 

.github/pull_request_template.md

@@ -1,7 +1,10 @@
 # Pull Request Template
 
+⚠️ Before Submitting a PR, Please Review:
+- Please ensure that you have thoroughly read and understood the [Contributing Docs](https://github.com/danny-avila/LibreChat/blob/main/.github/CONTRIBUTING.md) before submitting your Pull Request.
 
-### ⚠️ Before Submitting a PR, read the [Contributing Docs](https://github.com/danny-avila/LibreChat/blob/main/.github/CONTRIBUTING.md) in full!
+⚠️ Documentation Updates Notice:
+- Kindly note that documentation updates are managed in this repository: [librechat.ai](https://github.com/LibreChat-AI/librechat.ai)
 
 ## Summary
 
@@ -15,7 +18,6 @@ Please delete any irrelevant options.
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] This change requires a documentation update
-- [ ] Documentation update
 - [ ] Translation update
 
 ## Testing
@@ -26,6 +28,8 @@ Please describe your test process and include instructions so that we can reprod
 
 ## Checklist
 
+Please delete any irrelevant options.
+
 - [ ] My code adheres to this project's style guidelines
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented in any complex areas of my code
@@ -34,3 +38,4 @@ Please describe your test process and include instructions so that we can reprod
 - [ ] I have written tests demonstrating that my changes are effective or that my feature works
 - [ ] Local unit tests pass with my changes
 - [ ] Any changes dependent on mine have been merged and published in downstream modules.
+- [ ] A pull request for updating the documentation has been submitted.

.github/workflows/backend-review.yml

@@ -35,11 +35,32 @@ jobs:
 
       - name: Install Data Provider
         run: npm run build:data-provider
+      
+      - name: Create empty auth.json file
+        run: |
+          mkdir -p api/data
+          echo '{}' > api/data/auth.json
+
+      - name: Check for Circular dependency in rollup
+        working-directory: ./packages/data-provider
+        run: |
+          output=$(npm run rollup:api)
+          echo "$output"
+          if echo "$output" | grep -q "Circular dependency"; then
+            echo "Error: Circular dependency detected!"
+            exit 1
+          fi
+
+      - name: Prepare .env.test file
+        run: cp api/test/.env.test.example api/test/.env.test
 
       - name: Run unit tests
         run: cd api && npm run test:ci
 
+      - name: Run librechat-data-provider unit tests
+        run: cd packages/data-provider && npm run test:ci
+
       - name: Run linters
         uses: wearerequired/lint-action@v2
         with:
-          eslint: true
+          eslint: true

.github/workflows/container.yml

@@ -1,52 +0,0 @@
-name: Docker Compose Build on Tag
-
-# The workflow is triggered when a tag is pushed
-on:
-  push:
-    tags:
-      - "*"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Run docker-compose build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-          docker-compose build
-          docker build -f Dockerfile.multi --target api-build -t librechat-api .
-
-      # Get Tag Name
-      - name: Get Tag Name
-        id: tag_name
-        run: echo "TAG_NAME=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
-
-      # Tag it properly before push to github
-      - name: tag image and push
-        run: |
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:latest

.github/workflows/dev-images.yml

@@ -8,18 +8,32 @@ on:
     paths:
       - 'api/**'
       - 'client/**'
+      - 'packages/**'
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-dev-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat-dev
 
     steps:
       # Check out the repository
       - name: Checkout
         uses: actions/checkout@v4
 
-      # Set up Docker
-      - name: Set up Docker
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       # Log in to GitHub Container Registry
@@ -30,22 +44,29 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Build Docker images
-      - name: Build Docker images
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
         run: |
           cp .env.example .env
-          docker build -f Dockerfile.multi --target api-build -t librechat-dev-api .
-          docker build -f Dockerfile -t librechat-dev .
 
-      # Tag and push the images to GitHub Container Registry
-      - name: Tag and push images
-        run: |
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.sha }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.sha }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/frontend-review.yml

@@ -1,11 +1,6 @@
-#github action to run unit tests for frontend with jest
 name: Frontend Unit Tests
+
 on:
-  # push:
-  #   branches: 
-  #     - main
-  #     - dev
-  #     - release/*
   pull_request:
     branches: 
       - main
@@ -14,9 +9,10 @@ on:
     paths:
       - 'client/**'
       - 'packages/**'
+
 jobs:
-  tests_frontend:
-    name: Run frontend unit tests
+  tests_frontend_ubuntu:
+    name: Run frontend unit tests on Ubuntu
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
@@ -35,4 +31,26 @@ jobs:
 
       - name: Run unit tests
         run: npm run test:ci --verbose
-        working-directory: client
+        working-directory: client
+
+  tests_frontend_windows:
+    name: Run frontend unit tests on Windows
+    timeout-minutes: 60
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build Client
+        run: npm run frontend:ci
+
+      - name: Run unit tests
+        run: npm run test:ci --verbose
+        working-directory: client

.github/workflows/generate_embeddings.yml

@@ -0,0 +1,20 @@
+name: 'generate_embeddings'
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: supabase/embeddings-generator@v0.0.5
+        with:
+          supabase-url: ${{ secrets.SUPABASE_URL }}
+          supabase-service-role-key: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
+          openai-key: ${{ secrets.OPENAI_DOC_EMBEDDINGS_KEY }}
+          docs-root-path: 'docs'

.github/workflows/latest-images-main.yml

@@ -1,40 +0,0 @@
-name: Docker Compose Build on Main Branch
-
-on:
-  workflow_dispatch:  # This line allows manual triggering
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Run docker-compose build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-          docker-compose build
-          docker build -f Dockerfile.multi --target api-build -t librechat-api .
-
-      # Tag and push the images with the 'latest' tag
-      - name: Tag image and push
-        run: |
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:latest

.github/workflows/main-image-workflow.yml

@@ -0,0 +1,69 @@
+name: Docker Compose Build Latest Main Image Tag (Manual Dispatch)
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Fetch tags and set the latest tag
+        run: |
+          git fetch --tags
+          echo "LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)" >> $GITHUB_ENV
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/mkdocs.yaml

@@ -1,27 +0,0 @@
-name: mkdocs
-on:
-  push:
-    branches:
-      - main
-permissions:
-  contents: write
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.x
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
-      - uses: actions/cache@v3
-        with:
-          key: mkdocs-material-${{ env.cache_id }}
-          path: .cache
-          restore-keys: |
-            mkdocs-material-
-      - run: pip install mkdocs-material 
-      - run: pip install mkdocs-nav-weight
-      - run: pip install mkdocs-publisher
-      - run: pip install mkdocs-exclude
-      - run: mkdocs gh-deploy --force

.github/workflows/tag-images.yml

@@ -0,0 +1,67 @@
+name: Docker Images Build on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      # Check out the repository
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.gitignore

@@ -11,6 +11,7 @@ logs
 pids
 *.pid
 *.seed
+.git
 
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
@@ -21,6 +22,10 @@ coverage
 # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
 .grunt
 
+# translation services
+config/translations/stores/*
+client/src/localization/languages/*_missing_keys.json
+
 # Compiled Dirs (http://nodejs.org/api/addons.html)
 build/
 dist/
@@ -41,6 +46,7 @@ api/node_modules/
 client/node_modules/
 bower_components/
 *.d.ts
+!vite-env.d.ts
 
 # Floobits
 .floo
@@ -50,6 +56,7 @@ bower_components/
 
 #config file
 librechat.yaml
+librechat.yml
 
 # Environment
 .npmrc
@@ -68,12 +75,16 @@ src/style - official.css
 /playwright/.cache/
 .DS_Store
 *.code-workspace
+.idx
+monospace.json
 .idea
 *.iml
 *.pem
 config.local.ts
 **/storageState.json
 junit.xml
+**/.venv/
+**/venv/
 
 # docker override file
 docker-compose.override.yaml
@@ -88,4 +99,10 @@ auth.json
 /packages/ux-shared/
 /images
 
-!client/src/components/Nav/SettingsTabs/Data/
+!client/src/components/Nav/SettingsTabs/Data/
+
+# User uploads
+uploads/
+
+# owner
+release/

.husky/pre-commit

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env sh 
 set -e
 . "$(dirname -- "$0")/_/husky.sh"
 [ -n "$CI" ] && exit 0

Dockerfile

@@ -1,19 +1,32 @@
-# Base node image
-FROM node:18-alpine AS node
+# v0.7.3
 
-COPY . /app
+# Base node image
+FROM node:20-alpine AS node
+
+RUN apk --no-cache add curl
+
+RUN mkdir -p /app && chown node:node /app
 WORKDIR /app
 
-# Allow mounting of these files, which have no default
-# values.
-RUN touch .env
-# Install call deps - Install curl for health check
-RUN apk --no-cache add curl && \
-    npm ci
+USER node
 
-# React client build
-ENV NODE_OPTIONS="--max-old-space-size=2048"
-RUN npm run frontend
+COPY --chown=node:node . .
+
+RUN \
+    # Allow mounting of these files, which have no default
+    touch .env ; \
+    # Create directories for the volumes to inherit the correct permissions
+    mkdir -p /app/client/public/images /app/api/logs ; \
+    npm config set fetch-retry-maxtimeout 600000 ; \
+    npm config set fetch-retries 5 ; \
+    npm config set fetch-retry-mintimeout 15000 ; \
+    npm install --no-audit; \
+    # React client build
+    NODE_OPTIONS="--max-old-space-size=2048" npm run frontend; \
+    npm prune --production; \
+    npm cache clean --force
+
+RUN mkdir -p /app/client/public/images /app/api/logs
 
 # Node API setup
 EXPOSE 3080

Dockerfile.multi

@@ -1,3 +1,5 @@
+# v0.7.3
+
 # Build API, Client and Data Provider
 FROM node:20-alpine AS base
 
@@ -5,29 +7,31 @@ FROM node:20-alpine AS base
 FROM base AS data-provider-build
 WORKDIR /app/packages/data-provider
 COPY ./packages/data-provider ./
-RUN npm install
+RUN npm install; npm cache clean --force
 RUN npm run build
+RUN npm prune --production
 
 # React client build
-FROM data-provider-build AS client-build
+FROM base AS client-build
 WORKDIR /app/client
-COPY ./client/ ./
+COPY ./client/package*.json ./
 # Copy data-provider to client's node_modules
-RUN mkdir -p /app/client/node_modules/librechat-data-provider/
-RUN cp -R /app/packages/data-provider/* /app/client/node_modules/librechat-data-provider/
-RUN npm install
+COPY --from=data-provider-build /app/packages/data-provider/ /app/client/node_modules/librechat-data-provider/
+RUN npm install; npm cache clean --force
+COPY ./client/ ./
 ENV NODE_OPTIONS="--max-old-space-size=2048"
 RUN npm run build
 
 # Node API setup
-FROM data-provider-build AS api-build
+FROM base AS api-build
 WORKDIR /app/api
 COPY api/package*.json ./
 COPY api/ ./
+# Copy helper scripts
+COPY config/ ./
 # Copy data-provider to API's node_modules
-RUN mkdir -p /app/api/node_modules/librechat-data-provider/
-RUN cp -R /app/packages/data-provider/* /app/api/node_modules/librechat-data-provider/
-RUN npm install
+COPY --from=data-provider-build /app/packages/data-provider/ /app/api/node_modules/librechat-data-provider/
+RUN npm install --include prod; npm cache clean --force
 COPY --from=client-build /app/client/dist /app/client/dist
 EXPOSE 3080
 ENV HOST=0.0.0.0

README.md

@@ -1,10 +1,10 @@
 <p align="center">
-  <a href="https://docs.librechat.ai">
-    <img src="docs/assets/LibreChat.svg" height="256">
-  </a>
-  <a href="https://docs.librechat.ai">
-    <h1 align="center">LibreChat</h1>
+  <a href="https://librechat.ai">
+    <img src="client/public/assets/logo.svg" height="256">
   </a>
+  <h1 align="center">
+    <a href="https://librechat.ai">LibreChat</a>
+  </h1>
 </p>
 
 <p align="center">
@@ -27,83 +27,116 @@
 </p>
 
 <p align="center">
-  <a href="https://railway.app/template/b5k2mn?referralCode=HI9hWz">
-    <img src="https://railway.app/button.svg" alt="Deploy on Railway">
-  </a>
-</p>
-
-<p align="center">
-  <a href="https://template.cloud.sealos.io/deploy?templateName=librechat">
-    <img src="https://raw.githubusercontent.com/labring-actions/templates/main/Deploy-on-Sealos.svg" alt="Deploy on Sealos">
-  </a>
+<a href="https://railway.app/template/b5k2mn?referralCode=HI9hWz">
+  <img src="https://railway.app/button.svg" alt="Deploy on Railway" height="30">
+</a>
+<a href="https://zeabur.com/templates/0X2ZY8">
+  <img src="https://zeabur.com/button.svg" alt="Deploy on Zeabur" height="30"/>
+</a>
+<a href="https://template.cloud.sealos.io/deploy?templateName=librechat">
+  <img src="https://raw.githubusercontent.com/labring-actions/templates/main/Deploy-on-Sealos.svg" alt="Deploy on Sealos" height="30">
+</a>
 </p>
 
 # 📃 Features
- - 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and 11-2023 updates
- - 💬 Multimodal Chat:
-     - Upload and analyze images with GPT-4 and Gemini Vision 📸
-     - More filetypes and Assistants API integration in Active Development 🚧 
- - 🌎 Multilingual UI:
-     - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
-     - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands
- - 🤖 AI model selection: OpenAI API, Azure, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins
- - 💾 Create, Save, & Share Custom Presets
- - 🔄 Edit, Resubmit, and Continue messages with conversation branching
- - 📤 Export conversations as screenshots, markdown, text, json.
- - 🔍 Search all messages/conversations
- - 🔌 Plugins, including web access, image generation with DALL-E-3 and more
- - 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
- - ⚙️ Configure Proxy, Reverse Proxy, Docker, many Deployment options, and completely Open-Source
 
-[For a thorough review of our features, see our docs here](https://docs.librechat.ai/features/plugins/introduction.html) 📚
+- 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and latest updates
+- 🤖 AI model selection:
+  - OpenAI, Azure OpenAI, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins, Assistants API (including Azure Assistants)
+- ✅ Compatible across both **[Remote & Local AI services](https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints):**
+  - groq, Ollama, Cohere, Mistral AI, Apple MLX, koboldcpp, OpenRouter, together.ai, Perplexity, ShuttleAI, and more
+- 💾 Create, Save, & Share Custom Presets
+- 🔀 Switch between AI Endpoints and Presets, mid-chat
+- 🔄 Edit, Resubmit, and Continue Messages with Conversation branching
+- 🌿 Fork Messages & Conversations for Advanced Context control
+- 💬 Multimodal Chat:
+    - Upload and analyze images with Claude 3, GPT-4 (including `gpt-4o` and `gpt-4o-mini`), and Gemini Vision 📸
+    - Chat with Files using Custom Endpoints, OpenAI, Azure, Anthropic, & Google. 🗃️
+    - Advanced Agents with Files, Code Interpreter, Tools, and API Actions 🔦
+      - Available through the [OpenAI Assistants API](https://platform.openai.com/docs/assistants/overview) 🌤️
+      - Non-OpenAI Agents in Active Development 🚧
+- 🌎 Multilingual UI:
+  - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
+  - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands, עברית
+- 🎨 Customizable Dropdown & Interface: Adapts to both power users and newcomers
+- 📧 Verify your email to ensure secure access
+- 🗣️ Chat hands-free with Speech-to-Text and Text-to-Speech magic
+  - Automatically send and play Audio
+  - Supports OpenAI, Azure OpenAI, and Elevenlabs
+- 📥 Import Conversations from LibreChat, ChatGPT, Chatbot UI
+- 📤 Export conversations as screenshots, markdown, text, json
+- 🔍 Search all messages/conversations
+- 🔌 Plugins, including web access, image generation with DALL-E-3 and more
+- 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
+- ⚙️ Configure Proxy, Reverse Proxy, Docker, & many Deployment options:
+  - Use completely local or deploy on the cloud
+- 📖 Completely Open-Source & Built in Public
+- 🧑‍🤝‍🧑 Community-driven development, support, and feedback
 
+[For a thorough review of our features, see our docs here](https://docs.librechat.ai/) 📚
 
 ## 🪶 All-In-One AI Conversations with LibreChat
+
 LibreChat brings together the future of assistant AIs with the revolutionary technology of OpenAI's ChatGPT. Celebrating the original styling, LibreChat gives you the ability to integrate multiple AI models. It also integrates and enhances original client features such as conversation and message search, prompt templates and plugins.
 
 With LibreChat, you no longer need to opt for ChatGPT Plus and can instead use free or pay-per-call APIs. We welcome contributions, cloning, and forking to enhance the capabilities of this advanced chatbot platform.
-  
-<!-- https://github.com/danny-avila/LibreChat/assets/110412045/c1eb0c0f-41f6-4335-b982-84b278b53d59 -->
 
-[![Watch the video](https://img.youtube.com/vi/pNIOs1ovsXw/maxresdefault.jpg)](https://youtu.be/pNIOs1ovsXw)
+[![Watch the video](https://img.youtube.com/vi/bSVHEbVPNl4/maxresdefault.jpg)](https://www.youtube.com/watch?v=bSVHEbVPNl4)
 Click on the thumbnail to open the video☝️
 
 ---
 
-## 📚 Documentation
-For more information on how to use our advanced features, install and configure our software, and access our guidelines and tutorials, please check out our documentation at [docs.librechat.ai](https://docs.librechat.ai)
+## 🌐 Resources
+
+**GitHub Repo:**
+  - **RAG API:** [github.com/danny-avila/rag_api](https://github.com/danny-avila/rag_api)
+  - **Website:** [github.com/LibreChat-AI/librechat.ai](https://github.com/LibreChat-AI/librechat.ai)
+
+**Other:**
+  - **Website:** [librechat.ai](https://librechat.ai)
+  - **Documentation:** [docs.librechat.ai](https://docs.librechat.ai)
+  - **Blog:** [blog.librechat.ai](https://docs.librechat.ai)
 
 ---
 
-## 📝 Changelog 
-Keep up with the latest updates by visiting the releases page - [Releases](https://github.com/danny-avila/LibreChat/releases)
+## 📝 Changelog
 
-**⚠️ [Breaking Changes](docs/general_info/breaking_changes.md)**
-Please consult the breaking changes before updating.
+Keep up with the latest updates by visiting the releases page and notes:
+- [Releases](https://github.com/danny-avila/LibreChat/releases)
+- [Changelog](https://www.librechat.ai/changelog) 
+
+**⚠️ Please consult the [changelog](https://www.librechat.ai/changelog) for breaking changes before updating.**
 
 ---
 
 ## ⭐ Star History
 
 <p align="center">
-<a href="https://trendshift.io/repositories/4685" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4685" alt="danny-avila%2FLibreChat | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+  <a href="https://star-history.com/#danny-avila/LibreChat&Date">
+    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
+  </a>
+</p>
+<p align="center">
+  <a href="https://trendshift.io/repositories/4685" target="_blank" style="padding: 10px;">
+    <img src="https://trendshift.io/api/badge/repositories/4685" alt="danny-avila%2FLibreChat | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
+  </a>
+  <a href="https://runacap.com/ross-index/q1-24/" target="_blank" rel="noopener" style="margin-left: 20px;">
+    <img style="width: 260px; height: 56px" src="https://runacap.com/wp-content/uploads/2024/04/ROSS_badge_white_Q1_2024.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q1 2024 | Runa Capital" width="260" height="56"/>
+  </a>
 </p>
-
-<a href="https://star-history.com/#danny-avila/LibreChat&Date">
-  <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
-</a>
 
 ---
 
 ## ✨ Contributions
+
 Contributions, suggestions, bug reports and fixes are welcome!
 
-For new features, components, or extensions, please open an issue and discuss before sending a PR. 
+For new features, components, or extensions, please open an issue and discuss before sending a PR.
 
 ---
 
-💖 This project exists in its current state thanks to all the people who contribute
----
+## 💖 This project exists in its current state thanks to all the people who contribute
+
 <a href="https://github.com/danny-avila/LibreChat/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=danny-avila/LibreChat" />
 </a>

api/app/bingai.js

@@ -1,5 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
+const { EModelEndpoint } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
 const { logger } = require('~/config');
 
@@ -23,10 +24,7 @@ const askBing = async ({
 
   let key = null;
   if (expiresAt && isUserProvided) {
-    checkUserKeyExpiry(
-      expiresAt,
-      'Your BingAI Cookies have expired. Please provide your cookies again.',
-    );
+    checkUserKeyExpiry(expiresAt, EModelEndpoint.bingAI);
     key = await getUserKey({ userId, name: 'bingAI' });
   }
 

api/app/chatgpt-browser.js

@@ -1,5 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
+const { Constants, EModelEndpoint } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('../server/services/UserService');
 
 const browserClient = async ({
@@ -17,10 +18,7 @@ const browserClient = async ({
 
   let key = null;
   if (expiresAt && isUserProvided) {
-    checkUserKeyExpiry(
-      expiresAt,
-      'Your ChatGPT Access Token has expired. Please provide your token again.',
-    );
+    checkUserKeyExpiry(expiresAt, EModelEndpoint.chatGPTBrowser);
     key = await getUserKey({ userId, name: 'chatGPTBrowser' });
   }
 
@@ -48,7 +46,7 @@ const browserClient = async ({
     options = { ...options, parentMessageId, conversationId };
   }
 
-  if (parentMessageId === '00000000-0000-0000-0000-000000000000') {
+  if (parentMessageId === Constants.NO_PARENT) {
     delete options.conversationId;
   }
 

api/app/clients/AnthropicClient.js

@@ -1,7 +1,24 @@
 const Anthropic = require('@anthropic-ai/sdk');
+const { HttpsProxyAgent } = require('https-proxy-agent');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
-const { getResponseSender, EModelEndpoint } = require('librechat-data-provider');
+const {
+  Constants,
+  EModelEndpoint,
+  anthropicSettings,
+  getResponseSender,
+  validateVisionModel,
+} = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images/encode');
+const {
+  truncateText,
+  formatMessage,
+  titleFunctionPrompt,
+  parseParamFromPrompt,
+  createContextHandlers,
+} = require('./prompts');
+const spendTokens = require('~/models/spendTokens');
 const { getModelMaxTokens } = require('~/utils');
+const { sleep } = require('~/server/utils');
 const BaseClient = require('./BaseClient');
 const { logger } = require('~/config');
 
@@ -10,12 +27,22 @@ const AI_PROMPT = '\n\nAssistant:';
 
 const tokenizersCache = {};
 
+/** Helper function to introduce a delay before retrying */
+function delayBeforeRetry(attempts, baseDelay = 1000) {
+  return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts));
+}
+
+const { legacy } = anthropicSettings;
+
 class AnthropicClient extends BaseClient {
   constructor(apiKey, options = {}) {
     super(apiKey, options);
     this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY;
     this.userLabel = HUMAN_PROMPT;
     this.assistantLabel = AI_PROMPT;
+    this.contextStrategy = options.contextStrategy
+      ? options.contextStrategy.toLowerCase()
+      : 'discard';
     this.setOptions(options);
   }
 
@@ -39,16 +66,29 @@ class AnthropicClient extends BaseClient {
     const modelOptions = this.options.modelOptions || {};
     this.modelOptions = {
       ...modelOptions,
-      // set some good defaults (check for undefined in some cases because they may be 0)
-      model: modelOptions.model || 'claude-1',
-      temperature: typeof modelOptions.temperature === 'undefined' ? 1 : modelOptions.temperature, // 0 - 1, 1 is default
-      topP: typeof modelOptions.topP === 'undefined' ? 0.7 : modelOptions.topP, // 0 - 1, default: 0.7
-      topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40
-      stop: modelOptions.stop, // no stop method for now
+      model: modelOptions.model || anthropicSettings.model.default,
     };
 
+    this.isClaude3 = this.modelOptions.model.includes('claude-3');
+    this.isLegacyOutput = !this.modelOptions.model.includes('claude-3-5-sonnet');
+
+    if (
+      this.isLegacyOutput &&
+      this.modelOptions.maxOutputTokens &&
+      this.modelOptions.maxOutputTokens > legacy.maxOutputTokens.default
+    ) {
+      this.modelOptions.maxOutputTokens = legacy.maxOutputTokens.default;
+    }
+
+    this.useMessages = this.isClaude3 || !!this.options.attachments;
+
+    this.defaultVisionModel = this.options.visionModel ?? 'claude-3-sonnet-20240229';
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
+
     this.maxContextTokens =
-      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ?? 100000;
+      this.options.maxContextTokens ??
+      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ??
+      100000;
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500;
     this.maxPromptTokens =
       this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens;
@@ -87,18 +127,104 @@ class AnthropicClient extends BaseClient {
     return this;
   }
 
-  getClient() {
+  /**
+   * Get the initialized Anthropic client.
+   * @param {Partial<Anthropic.ClientOptions>} requestOptions - The options for the client.
+   * @returns {Anthropic} The Anthropic client instance.
+   */
+  getClient(requestOptions) {
+    /** @type {Anthropic.ClientOptions} */
     const options = {
+      fetch: this.fetch,
       apiKey: this.apiKey,
     };
 
+    if (this.options.proxy) {
+      options.httpAgent = new HttpsProxyAgent(this.options.proxy);
+    }
+
     if (this.options.reverseProxyUrl) {
       options.baseURL = this.options.reverseProxyUrl;
     }
 
+    if (requestOptions?.model && requestOptions.model.includes('claude-3-5-sonnet')) {
+      options.defaultHeaders = {
+        'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15',
+      };
+    }
+
     return new Anthropic(options);
   }
 
+  getTokenCountForResponse(response) {
+    return this.getTokenCountForMessage({
+      role: 'assistant',
+      content: response.text,
+    });
+  }
+
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
+   * - Sets `this.isVisionModel` to `true` if vision request.
+   * - Deletes `this.modelOptions.stop` if vision request.
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.anthropic];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    const visionModelAvailable = availableModels?.includes(this.defaultVisionModel);
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      visionModelAvailable &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+  }
+
+  /**
+   * Calculate the token cost in tokens for an image based on its dimensions and detail level.
+   *
+   * For reference, see: https://docs.anthropic.com/claude/docs/vision#image-costs
+   *
+   * @param {Object} image - The image object.
+   * @param {number} image.width - The width of the image.
+   * @param {number} image.height - The height of the image.
+   * @returns {number} The calculated token cost measured by tokens.
+   *
+   */
+  calculateImageTokenCost({ width, height }) {
+    return Math.ceil((width * height) / 750);
+  }
+
+  async addImageURLs(message, attachments) {
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      EModelEndpoint.anthropic,
+    );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
+  async recordTokenUsage({ promptTokens, completionTokens, model, context = 'message' }) {
+    await spendTokens(
+      {
+        context,
+        user: this.user,
+        conversationId: this.conversationId,
+        model: model ?? this.modelOptions.model,
+        endpointTokenConfig: this.options.endpointTokenConfig,
+      },
+      { promptTokens, completionTokens },
+    );
+  }
+
   async buildMessages(messages, parentMessageId) {
     const orderedMessages = this.constructor.getMessagesForConversation({
       messages,
@@ -107,28 +233,145 @@ class AnthropicClient extends BaseClient {
 
     logger.debug('[AnthropicClient] orderedMessages', { orderedMessages, parentMessageId });
 
-    const formattedMessages = orderedMessages.map((message) => ({
-      author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
-      content: message?.content ?? message.text,
-    }));
+    if (this.options.attachments) {
+      const attachments = await this.options.attachments;
+      const images = attachments.filter((file) => file.type.includes('image'));
+
+      if (images.length && !this.isVisionModel) {
+        throw new Error('Images are only supported with the Claude 3 family of models');
+      }
+
+      const latestMessage = orderedMessages[orderedMessages.length - 1];
+
+      if (this.message_file_map) {
+        this.message_file_map[latestMessage.messageId] = attachments;
+      } else {
+        this.message_file_map = {
+          [latestMessage.messageId]: attachments,
+        };
+      }
+
+      const files = await this.addImageURLs(latestMessage, attachments);
+
+      this.options.attachments = files;
+    }
+
+    if (this.message_file_map) {
+      this.contextHandlers = createContextHandlers(
+        this.options.req,
+        orderedMessages[orderedMessages.length - 1].text,
+      );
+    }
+
+    const formattedMessages = orderedMessages.map((message, i) => {
+      const formattedMessage = this.useMessages
+        ? formatMessage({
+          message,
+          endpoint: EModelEndpoint.anthropic,
+        })
+        : {
+          author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
+          content: message?.content ?? message.text,
+        };
+
+      const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+      /* If tokens were never counted, or, is a Vision request and the message has files, count again */
+      if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {
+        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
+      }
+
+      /* If message has files, calculate image token cost */
+      if (this.message_file_map && this.message_file_map[message.messageId]) {
+        const attachments = this.message_file_map[message.messageId];
+        for (const file of attachments) {
+          if (file.embedded) {
+            this.contextHandlers?.processFile(file);
+            continue;
+          }
+
+          orderedMessages[i].tokenCount += this.calculateImageTokenCost({
+            width: file.width,
+            height: file.height,
+          });
+        }
+      }
+
+      formattedMessage.tokenCount = orderedMessages[i].tokenCount;
+      return formattedMessage;
+    });
+
+    if (this.contextHandlers) {
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + (this.options.promptPrefix ?? '');
+    }
+
+    let { context: messagesInWindow, remainingContextTokens } =
+      await this.getMessagesWithinTokenLimit(formattedMessages);
+
+    const tokenCountMap = orderedMessages
+      .slice(orderedMessages.length - messagesInWindow.length)
+      .reduce((map, message, index) => {
+        const { messageId } = message;
+        if (!messageId) {
+          return map;
+        }
+
+        map[messageId] = orderedMessages[index].tokenCount;
+        return map;
+      }, {});
+
+    logger.debug('[AnthropicClient]', {
+      messagesInWindow: messagesInWindow.length,
+      remainingContextTokens,
+    });
 
     let lastAuthor = '';
     let groupedMessages = [];
 
-    for (let message of formattedMessages) {
+    for (let i = 0; i < messagesInWindow.length; i++) {
+      const message = messagesInWindow[i];
+      const author = message.role ?? message.author;
       // If last author is not same as current author, add to new group
-      if (lastAuthor !== message.author) {
-        groupedMessages.push({
-          author: message.author,
+      if (lastAuthor !== author) {
+        const newMessage = {
           content: [message.content],
-        });
-        lastAuthor = message.author;
+        };
+
+        if (message.role) {
+          newMessage.role = message.role;
+        } else {
+          newMessage.author = message.author;
+        }
+
+        groupedMessages.push(newMessage);
+        lastAuthor = author;
         // If same author, append content to the last group
       } else {
         groupedMessages[groupedMessages.length - 1].content.push(message.content);
       }
     }
 
+    groupedMessages = groupedMessages.map((msg, i) => {
+      const isLast = i === groupedMessages.length - 1;
+      if (msg.content.length === 1) {
+        const content = msg.content[0];
+        return {
+          ...msg,
+          // reason: final assistant content cannot end with trailing whitespace
+          content:
+            isLast && this.useMessages && msg.role === 'assistant' && typeof content === 'string'
+              ? content?.trim()
+              : content,
+        };
+      }
+
+      if (!this.useMessages && msg.tokenCount) {
+        delete msg.tokenCount;
+      }
+
+      return msg;
+    });
+
     let identityPrefix = '';
     if (this.options.userLabel) {
       identityPrefix = `\nHuman's name: ${this.options.userLabel}`;
@@ -154,9 +397,10 @@ class AnthropicClient extends BaseClient {
     // Prompt AI to respond, empty if last message was from AI
     let isEdited = lastAuthor === this.assistantLabel;
     const promptSuffix = isEdited ? '' : `${promptPrefix}${this.assistantLabel}\n`;
-    let currentTokenCount = isEdited
-      ? this.getTokenCount(promptPrefix)
-      : this.getTokenCount(promptSuffix);
+    let currentTokenCount =
+      isEdited || this.useMessages
+        ? this.getTokenCount(promptPrefix)
+        : this.getTokenCount(promptSuffix);
 
     let promptBody = '';
     const maxTokenCount = this.maxPromptTokens;
@@ -224,7 +468,69 @@ class AnthropicClient extends BaseClient {
       return true;
     };
 
-    await buildPromptBody();
+    const messagesPayload = [];
+    const buildMessagesPayload = async () => {
+      let canContinue = true;
+
+      if (promptPrefix) {
+        this.systemMessage = promptPrefix;
+      }
+
+      while (currentTokenCount < maxTokenCount && groupedMessages.length > 0 && canContinue) {
+        const message = groupedMessages.pop();
+
+        let tokenCountForMessage = message.tokenCount ?? this.getTokenCountForMessage(message);
+
+        const newTokenCount = currentTokenCount + tokenCountForMessage;
+        const exceededMaxCount = newTokenCount > maxTokenCount;
+
+        if (exceededMaxCount && messagesPayload.length === 0) {
+          throw new Error(
+            `Prompt is too long. Max token count is ${maxTokenCount}, but prompt is ${newTokenCount} tokens long.`,
+          );
+        } else if (exceededMaxCount) {
+          canContinue = false;
+          break;
+        }
+
+        delete message.tokenCount;
+        messagesPayload.unshift(message);
+        currentTokenCount = newTokenCount;
+
+        // Switch off isEdited after using it once
+        if (isEdited && message.role === 'assistant') {
+          isEdited = false;
+        }
+
+        // Wait for next tick to avoid blocking the event loop
+        await new Promise((resolve) => setImmediate(resolve));
+      }
+    };
+
+    const processTokens = () => {
+      // Add 2 tokens for metadata after all messages have been counted.
+      currentTokenCount += 2;
+
+      // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
+      this.modelOptions.maxOutputTokens = Math.min(
+        this.maxContextTokens - currentTokenCount,
+        this.maxResponseTokens,
+      );
+    };
+
+    if (this.modelOptions.model.startsWith('claude-3')) {
+      await buildMessagesPayload();
+      processTokens();
+      return {
+        prompt: messagesPayload,
+        context: messagesInWindow,
+        promptTokens: currentTokenCount,
+        tokenCountMap,
+      };
+    } else {
+      await buildPromptBody();
+      processTokens();
+    }
 
     if (nextMessage.remove) {
       promptBody = promptBody.replace(nextMessage.messageString, '');
@@ -234,22 +540,26 @@ class AnthropicClient extends BaseClient {
 
     let prompt = `${promptBody}${promptSuffix}`;
 
-    // Add 2 tokens for metadata after all messages have been counted.
-    currentTokenCount += 2;
-
-    // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
-    this.modelOptions.maxOutputTokens = Math.min(
-      this.maxContextTokens - currentTokenCount,
-      this.maxResponseTokens,
-    );
-
-    return { prompt, context };
+    return { prompt, context, promptTokens: currentTokenCount, tokenCountMap };
   }
 
   getCompletion() {
     logger.debug('AnthropicClient doesn\'t use getCompletion (all handled in sendCompletion)');
   }
 
+  /**
+   * Creates a message or completion response using the Anthropic client.
+   * @param {Anthropic} client - The Anthropic client instance.
+   * @param {Anthropic.default.MessageCreateParams | Anthropic.default.CompletionCreateParams} options - The options for the message or completion.
+   * @param {boolean} useMessages - Whether to use messages or completions. Defaults to `this.useMessages`.
+   * @returns {Promise<Anthropic.default.Message | Anthropic.default.Completion>} The response from the Anthropic client.
+   */
+  async createResponse(client, options, useMessages) {
+    return useMessages ?? this.useMessages
+      ? await client.messages.create(options)
+      : await client.completions.create(options);
+  }
+
   async sendCompletion(payload, { onProgress, abortController }) {
     if (!abortController) {
       abortController = new AbortController();
@@ -263,8 +573,6 @@ class AnthropicClient extends BaseClient {
     }
 
     logger.debug('modelOptions', { modelOptions });
-
-    const client = this.getClient();
     const metadata = {
       user_id: this.user,
     };
@@ -279,44 +587,105 @@ class AnthropicClient extends BaseClient {
       topP: top_p,
       topK: top_k,
     } = this.modelOptions;
+
     const requestOptions = {
-      prompt: payload,
       model,
       stream: stream || true,
-      max_tokens_to_sample: maxOutputTokens || 1500,
       stop_sequences,
       temperature,
       metadata,
       top_p,
       top_k,
     };
-    logger.debug('[AnthropicClient]', { ...requestOptions });
-    const response = await client.completions.create(requestOptions);
 
-    signal.addEventListener('abort', () => {
-      logger.debug('[AnthropicClient] message aborted!');
-      response.controller.abort();
-    });
-
-    for await (const completion of response) {
-      // Uncomment to debug message stream
-      // logger.debug(completion);
-      text += completion.completion;
-      onProgress(completion.completion);
+    if (this.useMessages) {
+      requestOptions.messages = payload;
+      requestOptions.max_tokens = maxOutputTokens || legacy.maxOutputTokens.default;
+    } else {
+      requestOptions.prompt = payload;
+      requestOptions.max_tokens_to_sample = maxOutputTokens || 1500;
     }
 
-    signal.removeEventListener('abort', () => {
-      logger.debug('[AnthropicClient] message aborted!');
-      response.controller.abort();
-    });
+    if (this.systemMessage) {
+      requestOptions.system = this.systemMessage;
+    }
+
+    logger.debug('[AnthropicClient]', { ...requestOptions });
+
+    const handleChunk = (currentChunk) => {
+      if (currentChunk) {
+        text += currentChunk;
+        onProgress(currentChunk);
+      }
+    };
+
+    const maxRetries = 3;
+    const streamRate = this.options.streamRate ?? Constants.DEFAULT_STREAM_RATE;
+    async function processResponse() {
+      let attempts = 0;
+
+      while (attempts < maxRetries) {
+        let response;
+        try {
+          const client = this.getClient(requestOptions);
+          response = await this.createResponse(client, requestOptions);
+
+          signal.addEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+
+          for await (const completion of response) {
+            // Handle each completion as before
+            if (completion?.delta?.text) {
+              handleChunk(completion.delta.text);
+            } else if (completion.completion) {
+              handleChunk(completion.completion);
+            }
+
+            await sleep(streamRate);
+          }
+
+          // Successful processing, exit loop
+          break;
+        } catch (error) {
+          attempts += 1;
+          logger.warn(
+            `User: ${this.user} | Anthropic Request ${attempts} failed: ${error.message}`,
+          );
+
+          if (attempts < maxRetries) {
+            await delayBeforeRetry(attempts, 350);
+          } else {
+            throw new Error(`Operation failed after ${maxRetries} attempts: ${error.message}`);
+          }
+        } finally {
+          signal.removeEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+        }
+      }
+    }
+
+    await processResponse.bind(this)();
 
     return text.trim();
   }
 
   getSaveOptions() {
     return {
+      maxContextTokens: this.options.maxContextTokens,
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
+      resendFiles: this.options.resendFiles,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
@@ -342,6 +711,82 @@ class AnthropicClient extends BaseClient {
   getTokenCount(text) {
     return this.gptEncoder.encode(text, 'all').length;
   }
+
+  /**
+   * Generates a concise title for a conversation based on the user's input text and response.
+   * Involves sending a chat completion request with specific instructions for title generation.
+   *
+   * This function capitlizes on [Anthropic's function calling training](https://docs.anthropic.com/claude/docs/functions-external-tools).
+   *
+   * @param {Object} params - The parameters for the conversation title generation.
+   * @param {string} params.text - The user's input.
+   * @param {string} [params.responseText=''] - The AI's immediate response to the user.
+   *
+   * @returns {Promise<string | 'New Chat'>} A promise that resolves to the generated conversation title.
+   *                            In case of failure, it will return the default title, "New Chat".
+   */
+  async titleConvo({ text, responseText = '' }) {
+    let title = 'New Chat';
+    const convo = `<initial_message>
+  ${truncateText(text)}
+  </initial_message>
+  <response>
+  ${JSON.stringify(truncateText(responseText))}
+  </response>`;
+
+    const { ANTHROPIC_TITLE_MODEL } = process.env ?? {};
+    const model = this.options.titleModel ?? ANTHROPIC_TITLE_MODEL ?? 'claude-3-haiku-20240307';
+    const system = titleFunctionPrompt;
+
+    const titleChatCompletion = async () => {
+      const content = `<conversation_context>
+  ${convo}
+  </conversation_context>
+  
+  Please generate a title for this conversation.`;
+
+      const titleMessage = { role: 'user', content };
+      const requestOptions = {
+        model,
+        temperature: 0.3,
+        max_tokens: 1024,
+        system,
+        stop_sequences: ['\n\nHuman:', '\n\nAssistant', '</function_calls>'],
+        messages: [titleMessage],
+      };
+
+      try {
+        const response = await this.createResponse(
+          this.getClient(requestOptions),
+          requestOptions,
+          true,
+        );
+        let promptTokens = response?.usage?.input_tokens;
+        let completionTokens = response?.usage?.output_tokens;
+        if (!promptTokens) {
+          promptTokens = this.getTokenCountForMessage(titleMessage);
+          promptTokens += this.getTokenCountForMessage({ role: 'system', content: system });
+        }
+        if (!completionTokens) {
+          completionTokens = this.getTokenCountForMessage(response.content[0]);
+        }
+        await this.recordTokenUsage({
+          model,
+          promptTokens,
+          completionTokens,
+          context: 'title',
+        });
+        const text = response.content[0].text;
+        title = parseParamFromPrompt(text, 'title');
+      } catch (e) {
+        logger.error('[AnthropicClient] There was an issue generating the title', e);
+      }
+    };
+
+    await titleChatCompletion();
+    logger.debug('[AnthropicClient] Convo Title: ' + title);
+    return title;
+  }
 }
 
 module.exports = AnthropicClient;

api/app/clients/BaseClient.js

@@ -1,8 +1,11 @@
 const crypto = require('crypto');
-const { supportsBalanceCheck } = require('librechat-data-provider');
-const { getConvo, getMessages, saveMessage, updateMessage, saveConvo } = require('~/models');
+const fetch = require('node-fetch');
+const { supportsBalanceCheck, Constants, CacheKeys, Time } = require('librechat-data-provider');
+const { getMessages, saveMessage, updateMessage, saveConvo } = require('~/models');
 const { addSpaceIfNeeded, isEnabled } = require('~/server/utils');
 const checkBalance = require('~/models/checkBalance');
+const { getFiles } = require('~/models/File');
+const { getLogStores } = require('~/cache');
 const TextStream = require('./TextStream');
 const { logger } = require('~/config');
 
@@ -16,13 +19,22 @@ class BaseClient {
       month: 'long',
       day: 'numeric',
     });
+    this.fetch = this.fetch.bind(this);
+    /** @type {boolean} */
+    this.skipSaveConvo = false;
+    /** @type {boolean} */
+    this.skipSaveUserMessage = false;
+    /** @type {ClientDatabaseSavePromise} */
+    this.userMessagePromise;
+    /** @type {ClientDatabaseSavePromise} */
+    this.responsePromise;
   }
 
   setOptions() {
     throw new Error('Method \'setOptions\' must be implemented.');
   }
 
-  getCompletion() {
+  async getCompletion() {
     throw new Error('Method \'getCompletion\' must be implemented.');
   }
 
@@ -46,10 +58,6 @@ class BaseClient {
     logger.debug('`[BaseClient] recordTokenUsage` not implemented.', response);
   }
 
-  async addPreviousAttachments(messages) {
-    return messages;
-  }
-
   async recordTokenUsage({ promptTokens, completionTokens }) {
     logger.debug('`[BaseClient] recordTokenUsage` not implemented.', {
       promptTokens,
@@ -57,6 +65,25 @@ class BaseClient {
     });
   }
 
+  /**
+   * Makes an HTTP request and logs the process.
+   *
+   * @param {RequestInfo} url - The URL to make the request to. Can be a string or a Request object.
+   * @param {RequestInit} [init] - Optional init options for the request.
+   * @returns {Promise<Response>} - A promise that resolves to the response of the fetch request.
+   */
+  async fetch(_url, init) {
+    let url = _url;
+    if (this.options.directEndpoint) {
+      url = this.options.reverseProxyUrl;
+    }
+    logger.debug(`Making request to ${url}`);
+    if (typeof Bun !== 'undefined') {
+      return await fetch(url, init);
+    }
+    return await fetch(url, init);
+  }
+
   getBuildMessagesOptions() {
     throw new Error('Subclasses must implement getBuildMessagesOptions');
   }
@@ -66,19 +93,45 @@ class BaseClient {
     await stream.processTextStream(onProgress);
   }
 
+  /**
+   * @returns {[string|undefined, string|undefined]}
+   */
+  processOverideIds() {
+    /** @type {Record<string, string | undefined>} */
+    let { overrideConvoId, overrideUserMessageId } = this.options?.req?.body ?? {};
+    if (overrideConvoId) {
+      const [conversationId, index] = overrideConvoId.split(Constants.COMMON_DIVIDER);
+      overrideConvoId = conversationId;
+      if (index !== '0') {
+        this.skipSaveConvo = true;
+      }
+    }
+    if (overrideUserMessageId) {
+      const [userMessageId, index] = overrideUserMessageId.split(Constants.COMMON_DIVIDER);
+      overrideUserMessageId = userMessageId;
+      if (index !== '0') {
+        this.skipSaveUserMessage = true;
+      }
+    }
+
+    return [overrideConvoId, overrideUserMessageId];
+  }
+
   async setMessageOptions(opts = {}) {
     if (opts && opts.replaceOptions) {
       this.setOptions(opts);
     }
 
+    const [overrideConvoId, overrideUserMessageId] = this.processOverideIds();
     const { isEdited, isContinued } = opts;
     const user = opts.user ?? null;
     this.user = user;
     const saveOptions = this.getSaveOptions();
     this.abortController = opts.abortController ?? new AbortController();
-    const conversationId = opts.conversationId ?? crypto.randomUUID();
-    const parentMessageId = opts.parentMessageId ?? '00000000-0000-0000-0000-000000000000';
-    const userMessageId = opts.overrideParentMessageId ?? crypto.randomUUID();
+    const conversationId = overrideConvoId ?? opts.conversationId ?? crypto.randomUUID();
+    const parentMessageId = opts.parentMessageId ?? Constants.NO_PARENT;
+    const userMessageId =
+      overrideUserMessageId ?? opts.overrideParentMessageId ?? crypto.randomUUID();
     let responseMessageId = opts.responseMessageId ?? crypto.randomUUID();
     let head = isEdited ? responseMessageId : parentMessageId;
     this.currentMessages = (await this.loadHistory(conversationId, head)) ?? [];
@@ -142,7 +195,7 @@ class BaseClient {
     }
 
     if (typeof opts?.onStart === 'function') {
-      opts.onStart(userMessage);
+      opts.onStart(userMessage, responseMessageId);
     }
 
     return {
@@ -376,6 +429,14 @@ class BaseClient {
     const { user, head, isEdited, conversationId, responseMessageId, saveOptions, userMessage } =
       await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     const { generation = '' } = opts;
 
     // It's not necessary to push to currentMessages
@@ -424,11 +485,19 @@ class BaseClient {
       this.handleTokenCountMap(tokenCountMap);
     }
 
-    if (!isEdited) {
-      await this.saveMessageToDatabase(userMessage, saveOptions, user);
+    if (!isEdited && !this.skipSaveUserMessage) {
+      this.userMessagePromise = this.saveMessageToDatabase(userMessage, saveOptions, user);
+      if (typeof opts?.getReqData === 'function') {
+        opts.getReqData({
+          userMessagePromise: this.userMessagePromise,
+        });
+      }
     }
 
-    if (isEnabled(process.env.CHECK_BALANCE) && supportsBalanceCheck[this.options.endpoint]) {
+    if (
+      isEnabled(process.env.CHECK_BALANCE) &&
+      supportsBalanceCheck[this.options.endpointType ?? this.options.endpoint]
+    ) {
       await checkBalance({
         req: this.options.req,
         res: this.options.res,
@@ -438,11 +507,14 @@ class BaseClient {
           amount: promptTokens,
           model: this.modelOptions.model,
           endpoint: this.options.endpoint,
+          endpointTokenConfig: this.options.endpointTokenConfig,
         },
       });
     }
 
     const completion = await this.sendCompletion(payload, opts);
+    this.abortController.requestCompleted = true;
+
     const responseMessage = {
       messageId: responseMessageId,
       conversationId,
@@ -453,6 +525,9 @@ class BaseClient {
       sender: this.sender,
       text: addSpaceIfNeeded(generation) + completion,
       promptTokens,
+      iconURL: this.options.iconURL,
+      endpoint: this.options.endpoint,
+      ...(this.metadata ?? {}),
     };
 
     if (
@@ -465,15 +540,23 @@ class BaseClient {
       const completionTokens = this.getTokenCount(completion);
       await this.recordTokenUsage({ promptTokens, completionTokens });
     }
-    await this.saveMessageToDatabase(responseMessage, saveOptions, user);
+    if (this.userMessagePromise) {
+      await this.userMessagePromise;
+    }
+    this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user);
+    const messageCache = getLogStores(CacheKeys.MESSAGES);
+    messageCache.set(
+      responseMessageId,
+      {
+        text: responseMessage.text,
+        complete: true,
+      },
+      Time.FIVE_MINUTES,
+    );
     delete responseMessage.tokenCount;
     return responseMessage;
   }
 
-  async getConversation(conversationId, user = null) {
-    return await getConvo(user, conversationId);
-  }
-
   async loadHistory(conversationId, parentMessageId = null) {
     logger.debug('[BaseClient] Loading history:', { conversationId, parentMessageId });
 
@@ -521,18 +604,48 @@ class BaseClient {
     return _messages;
   }
 
+  /**
+   * Save a message to the database.
+   * @param {TMessage} message
+   * @param {Partial<TConversation>} endpointOptions
+   * @param {string | null} user
+   */
   async saveMessageToDatabase(message, endpointOptions, user = null) {
-    await saveMessage({ ...message, endpoint: this.options.endpoint, user, unfinished: false });
-    await saveConvo(user, {
-      conversationId: message.conversationId,
-      endpoint: this.options.endpoint,
-      endpointType: this.options.endpointType,
-      ...endpointOptions,
-    });
+    if (this.user && user !== this.user) {
+      throw new Error('User mismatch.');
+    }
+
+    const savedMessage = await saveMessage(
+      this.options.req,
+      {
+        ...message,
+        endpoint: this.options.endpoint,
+        unfinished: false,
+        user,
+      },
+      { context: 'api/app/clients/BaseClient.js - saveMessageToDatabase #saveMessage' },
+    );
+
+    if (this.skipSaveConvo) {
+      return { message: savedMessage };
+    }
+
+    const conversation = await saveConvo(
+      this.options.req,
+      {
+        conversationId: message.conversationId,
+        endpoint: this.options.endpoint,
+        endpointType: this.options.endpointType,
+        ...endpointOptions,
+      },
+      { context: 'api/app/clients/BaseClient.js - saveMessageToDatabase #saveConvo' },
+    );
+
+    return { message: savedMessage, conversation };
   }
 
   async updateMessageInDatabase(message) {
-    await updateMessage(message);
+    await updateMessage(this.options.req, message);
   }
 
   /**
@@ -548,15 +661,15 @@ class BaseClient {
    *
    * Each message object should have an 'id' or 'messageId' property and may have a 'parentMessageId' property.
    * The 'parentMessageId' is the ID of the message that the current message is a reply to.
-   * If 'parentMessageId' is not present, null, or is '00000000-0000-0000-0000-000000000000',
+   * If 'parentMessageId' is not present, null, or is Constants.NO_PARENT,
    * the message is considered a root message.
    *
    * @param {Object} options - The options for the function.
-   * @param {Array} options.messages - An array of message objects. Each object should have either an 'id' or 'messageId' property, and may have a 'parentMessageId' property.
+   * @param {TMessage[]} options.messages - An array of message objects. Each object should have either an 'id' or 'messageId' property, and may have a 'parentMessageId' property.
    * @param {string} options.parentMessageId - The ID of the parent message to start the traversal from.
    * @param {Function} [options.mapMethod] - An optional function to map over the ordered messages. If provided, it will be applied to each message in the resulting array.
    * @param {boolean} [options.summary=false] - If set to true, the traversal modifies messages with 'summary' and 'summaryTokenCount' properties and stops at the message with a 'summary' property.
-   * @returns {Array} An array containing the messages in the order they should be displayed, starting with the most recent message with a 'summary' property if the 'summary' option is true, and ending with the message identified by 'parentMessageId'.
+   * @returns {TMessage[]} An array containing the messages in the order they should be displayed, starting with the most recent message with a 'summary' property if the 'summary' option is true, and ending with the message identified by 'parentMessageId'.
    */
   static getMessagesForConversation({
     messages,
@@ -603,9 +716,7 @@ class BaseClient {
       }
 
       currentMessageId =
-        message.parentMessageId === '00000000-0000-0000-0000-000000000000'
-          ? null
-          : message.parentMessageId;
+        message.parentMessageId === Constants.NO_PARENT ? null : message.parentMessageId;
     }
 
     orderedMessages.reverse();
@@ -679,6 +790,54 @@ class BaseClient {
 
     return await this.sendCompletion(payload, opts);
   }
+
+  /**
+   *
+   * @param {TMessage[]} _messages
+   * @returns {Promise<TMessage[]>}
+   */
+  async addPreviousAttachments(_messages) {
+    if (!this.options.resendFiles) {
+      return _messages;
+    }
+
+    /**
+     *
+     * @param {TMessage} message
+     */
+    const processMessage = async (message) => {
+      if (!this.message_file_map) {
+        /** @type {Record<string, MongoFile[]> */
+        this.message_file_map = {};
+      }
+
+      const fileIds = message.files.map((file) => file.file_id);
+      const files = await getFiles({
+        file_id: { $in: fileIds },
+      });
+
+      await this.addImageURLs(message, files);
+
+      this.message_file_map[message.messageId] = files;
+      return message;
+    };
+
+    const promises = [];
+
+    for (const message of _messages) {
+      if (!message.files) {
+        promises.push(message);
+        continue;
+      }
+
+      promises.push(processMessage(message));
+    }
+
+    const messages = await Promise.all(promises);
+
+    this.checkVisionRequest(Object.values(this.message_file_map ?? {}).flat());
+    return messages;
+  }
 }
 
 module.exports = BaseClient;

api/app/clients/ChatGPTClient.js

@@ -1,9 +1,19 @@
-const crypto = require('crypto');
 const Keyv = require('keyv');
+const crypto = require('crypto');
+const {
+  EModelEndpoint,
+  resolveHeaders,
+  CohereConstants,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
+const { CohereClient } = require('cohere-ai');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
+const { createCoherePayload } = require('./llm');
 const { Agent, ProxyAgent } = require('undici');
 const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
+const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils');
 
 const CHATGPT_MODEL = 'gpt-3.5-turbo';
 const tokenizersCache = {};
@@ -140,11 +150,13 @@ class ChatGPTClient extends BaseClient {
     return tokenizer;
   }
 
-  async getCompletion(input, onProgress, abortController = null) {
+  /** @type {getCompletion} */
+  async getCompletion(input, onProgress, onTokenProgress, abortController = null) {
     if (!abortController) {
       abortController = new AbortController();
     }
-    const modelOptions = { ...this.modelOptions };
+
+    let modelOptions = { ...this.modelOptions };
     if (typeof onProgress === 'function') {
       modelOptions.stream = true;
     }
@@ -159,56 +171,176 @@ class ChatGPTClient extends BaseClient {
     }
 
     const { debug } = this.options;
-    const url = this.completionsUrl;
+    let baseURL = this.completionsUrl;
     if (debug) {
       console.debug();
-      console.debug(url);
+      console.debug(baseURL);
       console.debug(modelOptions);
       console.debug();
     }
 
-    if (this.azure || this.options.azure) {
-      // Azure does not accept `model` in the body, so we need to remove it.
-      delete modelOptions.model;
-    }
-
     const opts = {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify(modelOptions),
       dispatcher: new Agent({
         bodyTimeout: 0,
         headersTimeout: 0,
       }),
     };
 
-    if (this.apiKey && this.options.azure) {
-      opts.headers['api-key'] = this.apiKey;
+    if (this.isVisionModel) {
+      modelOptions.max_tokens = 4000;
+    }
+
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const isAzure = this.azure || this.options.azure;
+    if (
+      (isAzure && this.isVisionModel && azureConfig) ||
+      (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    ) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+      opts.headers = resolveHeaders(headers);
+      this.langchainProxy = extractBaseURL(baseURL);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+      this.azure = !serverless && azureOptions;
+      this.azureEndpoint =
+        !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+    }
+
+    if (this.options.headers) {
+      opts.headers = { ...opts.headers, ...this.options.headers };
+    }
+
+    if (isAzure) {
+      // Azure does not accept `model` in the body, so we need to remove it.
+      delete modelOptions.model;
+
+      baseURL = this.langchainProxy
+        ? constructAzureURL({
+          baseURL: this.langchainProxy,
+          azureOptions: this.azure,
+        })
+        : this.azureEndpoint.split(/(?<!\/)\/(chat|completion)\//)[0];
+
+      if (this.options.forcePrompt) {
+        baseURL += '/completions';
+      } else {
+        baseURL += '/chat/completions';
+      }
+
+      opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
+      opts.headers = { ...opts.headers, 'api-key': this.apiKey };
     } else if (this.apiKey) {
       opts.headers.Authorization = `Bearer ${this.apiKey}`;
     }
 
+    if (process.env.OPENAI_ORGANIZATION) {
+      opts.headers['OpenAI-Organization'] = process.env.OPENAI_ORGANIZATION;
+    }
+
     if (this.useOpenRouter) {
       opts.headers['HTTP-Referer'] = 'https://librechat.ai';
       opts.headers['X-Title'] = 'LibreChat';
     }
 
-    if (this.options.headers) {
-      opts.headers = { ...opts.headers, ...this.options.headers };
-    }
-
     if (this.options.proxy) {
       opts.dispatcher = new ProxyAgent(this.options.proxy);
     }
 
+    /* hacky fixes for Mistral AI API:
+      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
+      - If there is only one message and it's a system message, change the role to user
+      */
+    if (baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+      const { messages } = modelOptions;
+
+      const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+      if (systemMessageIndex > 0) {
+        const [systemMessage] = messages.splice(systemMessageIndex, 1);
+        messages.unshift(systemMessage);
+      }
+
+      modelOptions.messages = messages;
+
+      if (messages.length === 1 && messages[0].role === 'system') {
+        modelOptions.messages[0].role = 'user';
+      }
+    }
+
+    if (this.options.addParams && typeof this.options.addParams === 'object') {
+      modelOptions = {
+        ...modelOptions,
+        ...this.options.addParams,
+      };
+      logger.debug('[ChatGPTClient] chatCompletion: added params', {
+        addParams: this.options.addParams,
+        modelOptions,
+      });
+    }
+
+    if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
+      this.options.dropParams.forEach((param) => {
+        delete modelOptions[param];
+      });
+      logger.debug('[ChatGPTClient] chatCompletion: dropped params', {
+        dropParams: this.options.dropParams,
+        modelOptions,
+      });
+    }
+
+    if (baseURL.startsWith(CohereConstants.API_URL)) {
+      const payload = createCoherePayload({ modelOptions });
+      return await this.cohereChatCompletion({ payload, onTokenProgress });
+    }
+
+    if (baseURL.includes('v1') && !baseURL.includes('/completions') && !this.isChatCompletion) {
+      baseURL = baseURL.split('v1')[0] + 'v1/completions';
+    } else if (
+      baseURL.includes('v1') &&
+      !baseURL.includes('/chat/completions') &&
+      this.isChatCompletion
+    ) {
+      baseURL = baseURL.split('v1')[0] + 'v1/chat/completions';
+    }
+
+    const BASE_URL = new URL(baseURL);
+    if (opts.defaultQuery) {
+      Object.entries(opts.defaultQuery).forEach(([key, value]) => {
+        BASE_URL.searchParams.append(key, value);
+      });
+      delete opts.defaultQuery;
+    }
+
+    const completionsURL = BASE_URL.toString();
+    opts.body = JSON.stringify(modelOptions);
+
     if (modelOptions.stream) {
       // eslint-disable-next-line no-async-promise-executor
       return new Promise(async (resolve, reject) => {
         try {
           let done = false;
-          await fetchEventSource(url, {
+          await fetchEventSource(completionsURL, {
             ...opts,
             signal: abortController.signal,
             async onopen(response) {
@@ -236,7 +368,6 @@ class ChatGPTClient extends BaseClient {
               // workaround for private API not sending [DONE] event
               if (!done) {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
               }
             },
@@ -249,14 +380,13 @@ class ChatGPTClient extends BaseClient {
             },
             onmessage(message) {
               if (debug) {
-                // console.debug(message);
+                console.debug(message);
               }
               if (!message.data || message.event === 'ping') {
                 return;
               }
               if (message.data === '[DONE]') {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
                 done = true;
                 return;
@@ -269,7 +399,7 @@ class ChatGPTClient extends BaseClient {
         }
       });
     }
-    const response = await fetch(url, {
+    const response = await fetch(completionsURL, {
       ...opts,
       signal: abortController.signal,
     });
@@ -287,6 +417,43 @@ class ChatGPTClient extends BaseClient {
     return response.json();
   }
 
+  /** @type {cohereChatCompletion} */
+  async cohereChatCompletion({ payload, onTokenProgress }) {
+    const cohere = new CohereClient({
+      token: this.apiKey,
+      environment: this.completionsUrl,
+    });
+
+    if (!payload.stream) {
+      const chatResponse = await cohere.chat(payload);
+      return chatResponse.text;
+    }
+
+    const chatStream = await cohere.chatStream(payload);
+    let reply = '';
+    for await (const message of chatStream) {
+      if (!message) {
+        continue;
+      }
+
+      if (message.eventType === 'text-generation' && message.text) {
+        onTokenProgress(message.text);
+        reply += message.text;
+      }
+      /*
+      Cohere API Chinese Unicode character replacement hotfix.
+      Should be un-commented when the following issue is resolved:
+      https://github.com/cohere-ai/cohere-typescript/issues/151
+
+      else if (message.eventType === 'stream-end' && message.response) {
+        reply = message.response.text;
+      }
+      */
+    }
+
+    return reply;
+  }
+
   async generateTitle(userMessage, botMessage) {
     const instructionsPayload = {
       role: 'system',

api/app/clients/GoogleClient.js

@@ -1,21 +1,32 @@
 const { google } = require('googleapis');
 const { Agent, ProxyAgent } = require('undici');
-const { GoogleVertexAI } = require('langchain/llms/googlevertexai');
+const { ChatVertexAI } = require('@langchain/google-vertexai');
 const { ChatGoogleGenerativeAI } = require('@langchain/google-genai');
+const { GoogleGenerativeAI: GenAI } = require('@google/generative-ai');
+const { GoogleVertexAI } = require('@langchain/community/llms/googlevertexai');
 const { ChatGoogleVertexAI } = require('langchain/chat_models/googlevertexai');
 const { AIMessage, HumanMessage, SystemMessage } = require('langchain/schema');
-const { encodeAndFormat, validateVisionModel } = require('~/server/services/Files/images');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const {
+  validateVisionModel,
   getResponseSender,
-  EModelEndpoint,
   endpointSettings,
+  EModelEndpoint,
+  VisionModes,
+  Constants,
   AuthKeys,
 } = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images');
 const { getModelMaxTokens } = require('~/utils');
-const { formatMessage } = require('./prompts');
-const BaseClient = require('./BaseClient');
+const { sleep } = require('~/server/utils');
 const { logger } = require('~/config');
+const {
+  formatMessage,
+  createContextHandlers,
+  titleInstruction,
+  truncateText,
+} = require('./prompts');
+const BaseClient = require('./BaseClient');
 
 const loc = 'us-central1';
 const publisher = 'google';
@@ -123,25 +134,21 @@ class GoogleClient extends BaseClient {
       // stop: modelOptions.stop // no stop method for now
     };
 
-    if (this.options.attachments) {
-      this.modelOptions.model = 'gemini-pro-vision';
-    }
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
 
-    // TODO: as of 12/14/23, only gemini models are "Generative AI" models provided by Google
+    /** @type {boolean} Whether using a "GenerativeAI" Model */
     this.isGenerativeModel = this.modelOptions.model.includes('gemini');
-    this.isVisionModel = validateVisionModel(this.modelOptions.model);
     const { isGenerativeModel } = this;
-    if (this.isVisionModel && !this.options.attachments) {
-      this.modelOptions.model = 'gemini-pro';
-      this.isVisionModel = false;
-    }
     this.isChatModel = !isGenerativeModel && this.modelOptions.model.includes('chat');
     const { isChatModel } = this;
     this.isTextModel =
       !isGenerativeModel && !isChatModel && /code|text/.test(this.modelOptions.model);
     const { isTextModel } = this;
 
-    this.maxContextTokens = getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google);
+    this.maxContextTokens =
+      this.options.maxContextTokens ??
+      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google);
+
     // The max prompt tokens is determined by the max context tokens minus the max response tokens.
     // Earlier messages will be dropped until the prompt is within the limit.
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || settings.maxOutputTokens.default;
@@ -219,6 +226,33 @@ class GoogleClient extends BaseClient {
     return this;
   }
 
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    /* Validation vision request */
+    this.defaultVisionModel = this.options.visionModel ?? 'gemini-pro-vision';
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.google];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      availableModels?.includes(this.defaultVisionModel) &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+
+    if (this.isVisionModel && !attachments && this.modelOptions.model.includes('gemini-pro')) {
+      this.modelOptions.model = 'gemini-pro';
+      this.isVisionModel = false;
+    }
+  }
+
   formatMessages() {
     return ((message) => ({
       author: message?.author ?? (message.isCreatedByUser ? this.userLabel : this.modelLabel),
@@ -226,18 +260,91 @@ class GoogleClient extends BaseClient {
     })).bind(this);
   }
 
-  async buildVisionMessages(messages = [], parentMessageId) {
-    const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId);
+  /**
+   * Formats messages for generative AI
+   * @param {TMessage[]} messages
+   * @returns
+   */
+  async formatGenerativeMessages(messages) {
+    const formattedMessages = [];
     const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    const files = await this.addImageURLs(latestMessage, attachments, VisionModes.generative);
+    this.options.attachments = files;
+    messages[messages.length - 1] = latestMessage;
+
+    for (const _message of messages) {
+      const role = _message.isCreatedByUser ? this.userLabel : this.modelLabel;
+      const parts = [];
+      parts.push({ text: _message.text });
+      if (!_message.image_urls?.length) {
+        formattedMessages.push({ role, parts });
+        continue;
+      }
+
+      for (const images of _message.image_urls) {
+        if (images.inlineData) {
+          parts.push({ inlineData: images.inlineData });
+        }
+      }
+
+      formattedMessages.push({ role, parts });
+    }
+
+    return formattedMessages;
+  }
+
+  /**
+   *
+   * Adds image URLs to the message object and returns the files
+   *
+   * @param {TMessage[]} messages
+   * @param {MongoFile[]} files
+   * @returns {Promise<MongoFile[]>}
+   */
+  async addImageURLs(message, attachments, mode = '') {
     const { files, image_urls } = await encodeAndFormat(
       this.options.req,
-      attachments.filter((file) => file.type.includes('image')),
+      attachments,
       EModelEndpoint.google,
+      mode,
     );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
 
+  /**
+   * Builds the augmented prompt for attachments
+   * TODO: Add File API Support
+   * @param {TMessage[]} messages
+   */
+  async buildAugmentedPrompt(messages = []) {
+    const attachments = await this.options.attachments;
     const latestMessage = { ...messages[messages.length - 1] };
+    this.contextHandlers = createContextHandlers(this.options.req, latestMessage.text);
+
+    if (this.contextHandlers) {
+      for (const file of attachments) {
+        if (file.embedded) {
+          this.contextHandlers?.processFile(file);
+          continue;
+        }
+      }
+
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + this.options.promptPrefix;
+    }
+  }
+
+  async buildVisionMessages(messages = [], parentMessageId) {
+    const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    await this.buildAugmentedPrompt(messages);
+
+    const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId);
+
+    const files = await this.addImageURLs(latestMessage, attachments);
 
-    latestMessage.image_urls = image_urls;
     this.options.attachments = files;
 
     latestMessage.text = prompt;
@@ -253,18 +360,29 @@ class GoogleClient extends BaseClient {
     return { prompt: payload };
   }
 
+  /** @param {TMessage[]} [messages=[]]  */
+  async buildGenerativeMessages(messages = []) {
+    this.userLabel = 'user';
+    this.modelLabel = 'model';
+    const promises = [];
+    promises.push(await this.formatGenerativeMessages(messages));
+    promises.push(this.buildAugmentedPrompt(messages));
+    const [formattedMessages] = await Promise.all(promises);
+    return { prompt: formattedMessages };
+  }
+
   async buildMessages(messages = [], parentMessageId) {
     if (!this.isGenerativeModel && !this.project_id) {
       throw new Error(
         '[GoogleClient] a Service Account JSON Key is required for PaLM 2 and Codey models (Vertex AI)',
       );
-    } else if (this.isGenerativeModel && (!this.apiKey || this.apiKey === 'user_provided')) {
-      throw new Error(
-        '[GoogleClient] an API Key is required for Gemini models (Generative Language API)',
-      );
     }
 
-    if (this.options.attachments) {
+    if (!this.project_id && this.modelOptions.model.includes('1.5')) {
+      return await this.buildGenerativeMessages(messages);
+    }
+
+    if (this.options.attachments && this.isGenerativeModel) {
       return this.buildVisionMessages(messages, parentMessageId);
     }
 
@@ -478,25 +596,42 @@ class GoogleClient extends BaseClient {
   }
 
   createLLM(clientOptions) {
-    if (this.isGenerativeModel) {
-      return new ChatGoogleGenerativeAI({ ...clientOptions, apiKey: this.apiKey });
+    const model = clientOptions.modelName ?? clientOptions.model;
+    if (this.project_id && this.isTextModel) {
+      logger.debug('Creating Google VertexAI client');
+      return new GoogleVertexAI(clientOptions);
+    } else if (this.project_id && this.isChatModel) {
+      logger.debug('Creating Chat Google VertexAI client');
+      return new ChatGoogleVertexAI(clientOptions);
+    } else if (this.project_id) {
+      logger.debug('Creating VertexAI client');
+      return new ChatVertexAI(clientOptions);
+    } else if (model.includes('1.5')) {
+      logger.debug('Creating GenAI client');
+      return new GenAI(this.apiKey).getGenerativeModel(
+        {
+          ...clientOptions,
+          model,
+        },
+        { apiVersion: 'v1beta' },
+      );
     }
 
-    return this.isTextModel
-      ? new GoogleVertexAI(clientOptions)
-      : new ChatGoogleVertexAI(clientOptions);
+    logger.debug('Creating Chat Google Generative AI client');
+    return new ChatGoogleGenerativeAI({ ...clientOptions, apiKey: this.apiKey });
   }
 
   async getCompletion(_payload, options = {}) {
-    const { onProgress, abortController } = options;
     const { parameters, instances } = _payload;
+    const { onProgress, abortController } = options;
+    const streamRate = this.options.streamRate ?? Constants.DEFAULT_STREAM_RATE;
     const { messages: _messages, context, examples: _examples } = instances?.[0] ?? {};
 
     let examples;
 
     let clientOptions = { ...parameters, maxRetries: 2 };
 
-    if (!this.isGenerativeModel) {
+    if (this.project_id) {
       clientOptions['authOptions'] = {
         credentials: {
           ...this.serviceKey,
@@ -509,7 +644,7 @@ class GoogleClient extends BaseClient {
       clientOptions = { ...clientOptions, ...this.modelOptions };
     }
 
-    if (this.isGenerativeModel) {
+    if (this.isGenerativeModel && !this.project_id) {
       clientOptions.modelName = clientOptions.model;
       delete clientOptions.model;
     }
@@ -540,25 +675,193 @@ class GoogleClient extends BaseClient {
       messages.unshift(new SystemMessage(context));
     }
 
+    const modelName = clientOptions.modelName ?? clientOptions.model ?? '';
+    if (modelName?.includes('1.5') && !this.project_id) {
+      /** @type {GenerativeModel} */
+      const client = model;
+      const requestOptions = {
+        contents: _payload,
+      };
+
+      if (this.options?.promptPrefix?.length) {
+        requestOptions.systemInstruction = {
+          parts: [
+            {
+              text: this.options.promptPrefix,
+            },
+          ],
+        };
+      }
+
+      const safetySettings = _payload.safetySettings;
+      requestOptions.safetySettings = safetySettings;
+
+      const delay = modelName.includes('flash') ? 8 : 14;
+      const result = await client.generateContentStream(requestOptions);
+      for await (const chunk of result.stream) {
+        const chunkText = chunk.text();
+        await this.generateTextStream(chunkText, onProgress, {
+          delay,
+        });
+        reply += chunkText;
+        await sleep(streamRate);
+      }
+      return reply;
+    }
+
+    const safetySettings = _payload.safetySettings;
     const stream = await model.stream(messages, {
       signal: abortController.signal,
       timeout: 7000,
+      safetySettings: safetySettings,
     });
 
+    let delay = this.options.streamRate || 8;
+
+    if (!this.options.streamRate) {
+      if (this.isGenerativeModel) {
+        delay = 12;
+      }
+      if (modelName.includes('flash')) {
+        delay = 5;
+      }
+    }
+
     for await (const chunk of stream) {
-      await this.generateTextStream(chunk?.content ?? chunk, onProgress, {
-        delay: this.isGenerativeModel ? 12 : 8,
+      const chunkText = chunk?.content ?? chunk;
+      await this.generateTextStream(chunkText, onProgress, {
+        delay,
       });
-      reply += chunk?.content ?? chunk;
+      reply += chunkText;
     }
 
     return reply;
   }
 
+  /**
+   * Stripped-down logic for generating a title. This uses the non-streaming APIs, since the user does not see titles streaming
+   */
+  async titleChatCompletion(_payload, options = {}) {
+    const { abortController } = options;
+    const { parameters, instances } = _payload;
+    const { messages: _messages, examples: _examples } = instances?.[0] ?? {};
+
+    let clientOptions = { ...parameters, maxRetries: 2 };
+
+    logger.debug('Initialized title client options');
+
+    if (this.project_id) {
+      clientOptions['authOptions'] = {
+        credentials: {
+          ...this.serviceKey,
+        },
+        projectId: this.project_id,
+      };
+    }
+
+    if (!parameters) {
+      clientOptions = { ...clientOptions, ...this.modelOptions };
+    }
+
+    if (this.isGenerativeModel && !this.project_id) {
+      clientOptions.modelName = clientOptions.model;
+      delete clientOptions.model;
+    }
+
+    const model = this.createLLM(clientOptions);
+
+    let reply = '';
+    const messages = this.isTextModel ? _payload.trim() : _messages;
+
+    const modelName = clientOptions.modelName ?? clientOptions.model ?? '';
+    if (modelName?.includes('1.5') && !this.project_id) {
+      logger.debug('Identified titling model as 1.5 version');
+      /** @type {GenerativeModel} */
+      const client = model;
+      const requestOptions = {
+        contents: _payload,
+      };
+
+      if (this.options?.promptPrefix?.length) {
+        requestOptions.systemInstruction = {
+          parts: [
+            {
+              text: this.options.promptPrefix,
+            },
+          ],
+        };
+      }
+
+      const safetySettings = _payload.safetySettings;
+      requestOptions.safetySettings = safetySettings;
+
+      const result = await client.generateContent(requestOptions);
+
+      reply = result.response?.text();
+
+      return reply;
+    } else {
+      logger.debug('Beginning titling');
+      const safetySettings = _payload.safetySettings;
+
+      const titleResponse = await model.invoke(messages, {
+        signal: abortController.signal,
+        timeout: 7000,
+        safetySettings: safetySettings,
+      });
+
+      reply = titleResponse.content;
+
+      return reply;
+    }
+  }
+
+  async titleConvo({ text, responseText = '' }) {
+    let title = 'New Chat';
+    const convo = `||>User:
+"${truncateText(text)}"
+||>Response:
+"${JSON.stringify(truncateText(responseText))}"`;
+
+    let { prompt: payload } = await this.buildMessages([
+      {
+        text: `Please generate ${titleInstruction}
+
+    ${convo}
+    
+    ||>Title:`,
+        isCreatedByUser: true,
+        author: this.userLabel,
+      },
+    ]);
+
+    if (this.isVisionModel) {
+      logger.warn(
+        `Current vision model does not support titling without an attachment; falling back to default model ${settings.model.default}`,
+      );
+
+      payload.parameters = { ...payload.parameters, model: settings.model.default };
+    }
+
+    try {
+      title = await this.titleChatCompletion(payload, {
+        abortController: new AbortController(),
+        onProgress: () => {},
+      });
+    } catch (e) {
+      logger.error('[GoogleClient] There was an issue generating the title', e);
+    }
+    logger.debug(`Title response: ${title}`);
+    return title;
+  }
+
   getSaveOptions() {
     return {
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
@@ -568,6 +871,33 @@ class GoogleClient extends BaseClient {
   }
 
   async sendCompletion(payload, opts = {}) {
+    const modelName = payload.parameters?.model;
+
+    if (modelName && modelName.toLowerCase().includes('gemini')) {
+      const safetySettings = [
+        {
+          category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+          threshold:
+            process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_HATE_SPEECH',
+          threshold: process.env.GOOGLE_SAFETY_HATE_SPEECH || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_HARASSMENT',
+          threshold: process.env.GOOGLE_SAFETY_HARASSMENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+          threshold:
+            process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+      ];
+
+      payload.safetySettings = safetySettings;
+    }
+
     let reply = '';
     reply = await this.getCompletion(payload, opts);
     return reply.trim();

api/app/clients/OllamaClient.js

@@ -0,0 +1,159 @@
+const { z } = require('zod');
+const axios = require('axios');
+const { Ollama } = require('ollama');
+const { Constants } = require('librechat-data-provider');
+const { deriveBaseURL } = require('~/utils');
+const { sleep } = require('~/server/utils');
+const { logger } = require('~/config');
+
+const ollamaPayloadSchema = z.object({
+  mirostat: z.number().optional(),
+  mirostat_eta: z.number().optional(),
+  mirostat_tau: z.number().optional(),
+  num_ctx: z.number().optional(),
+  repeat_last_n: z.number().optional(),
+  repeat_penalty: z.number().optional(),
+  temperature: z.number().optional(),
+  seed: z.number().nullable().optional(),
+  stop: z.array(z.string()).optional(),
+  tfs_z: z.number().optional(),
+  num_predict: z.number().optional(),
+  top_k: z.number().optional(),
+  top_p: z.number().optional(),
+  stream: z.optional(z.boolean()),
+  model: z.string(),
+});
+
+/**
+ * @param {string} imageUrl
+ * @returns {string}
+ * @throws {Error}
+ */
+const getValidBase64 = (imageUrl) => {
+  const parts = imageUrl.split(';base64,');
+
+  if (parts.length === 2) {
+    return parts[1];
+  } else {
+    logger.error('Invalid or no Base64 string found in URL.');
+  }
+};
+
+class OllamaClient {
+  constructor(options = {}) {
+    const host = deriveBaseURL(options.baseURL ?? 'http://localhost:11434');
+    this.streamRate = options.streamRate ?? Constants.DEFAULT_STREAM_RATE;
+    /** @type {Ollama} */
+    this.client = new Ollama({ host });
+  }
+
+  /**
+   * Fetches Ollama models from the specified base API path.
+   * @param {string} baseURL
+   * @returns {Promise<string[]>} The Ollama models.
+   */
+  static async fetchModels(baseURL) {
+    let models = [];
+    if (!baseURL) {
+      return models;
+    }
+    try {
+      const ollamaEndpoint = deriveBaseURL(baseURL);
+      /** @type {Promise<AxiosResponse<OllamaListResponse>>} */
+      const response = await axios.get(`${ollamaEndpoint}/api/tags`);
+      models = response.data.models.map((tag) => tag.name);
+      return models;
+    } catch (error) {
+      const logMessage =
+        'Failed to fetch models from Ollama API. If you are not using Ollama directly, and instead, through some aggregator or reverse proxy that handles fetching via OpenAI spec, ensure the name of the endpoint doesn\'t start with `ollama` (case-insensitive).';
+      logger.error(logMessage, error);
+      return [];
+    }
+  }
+
+  /**
+   * @param {ChatCompletionMessage[]} messages
+   * @returns {OllamaMessage[]}
+   */
+  static formatOpenAIMessages(messages) {
+    const ollamaMessages = [];
+
+    for (const message of messages) {
+      if (typeof message.content === 'string') {
+        ollamaMessages.push({
+          role: message.role,
+          content: message.content,
+        });
+        continue;
+      }
+
+      let aggregatedText = '';
+      let imageUrls = [];
+
+      for (const content of message.content) {
+        if (content.type === 'text') {
+          aggregatedText += content.text + ' ';
+        } else if (content.type === 'image_url') {
+          imageUrls.push(getValidBase64(content.image_url.url));
+        }
+      }
+
+      const ollamaMessage = {
+        role: message.role,
+        content: aggregatedText.trim(),
+      };
+
+      if (imageUrls.length > 0) {
+        ollamaMessage.images = imageUrls;
+      }
+
+      ollamaMessages.push(ollamaMessage);
+    }
+
+    return ollamaMessages;
+  }
+
+  /***
+   * @param {Object} params
+   * @param {ChatCompletionPayload} params.payload
+   * @param {onTokenProgress} params.onProgress
+   * @param {AbortController} params.abortController
+   */
+  async chatCompletion({ payload, onProgress, abortController = null }) {
+    let intermediateReply = '';
+
+    const parameters = ollamaPayloadSchema.parse(payload);
+    const messages = OllamaClient.formatOpenAIMessages(payload.messages);
+
+    if (parameters.stream) {
+      const stream = await this.client.chat({
+        messages,
+        ...parameters,
+      });
+
+      for await (const chunk of stream) {
+        const token = chunk.message.content;
+        intermediateReply += token;
+        onProgress(token);
+        if (abortController.signal.aborted) {
+          stream.controller.abort();
+          break;
+        }
+
+        await sleep(this.streamRate);
+      }
+    }
+    // TODO: regular completion
+    else {
+      // const generation = await this.client.generate(payload);
+    }
+
+    return intermediateReply;
+  }
+  catch(err) {
+    logger.error('[OllamaClient.chatCompletion]', err);
+    throw err;
+  }
+}
+
+module.exports = { OllamaClient, ollamaPayloadSchema };

api/app/clients/OpenAIClient.js

@@ -1,21 +1,37 @@
 const OpenAI = require('openai');
+const { OllamaClient } = require('./OllamaClient');
 const { HttpsProxyAgent } = require('https-proxy-agent');
-const { getResponseSender, ImageDetailCost, ImageDetail } = require('librechat-data-provider');
+const {
+  Constants,
+  ImageDetail,
+  EModelEndpoint,
+  resolveHeaders,
+  ImageDetailCost,
+  CohereConstants,
+  getResponseSender,
+  validateVisionModel,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const {
-  getModelMaxTokens,
-  genAzureChatCompletion,
   extractBaseURL,
   constructAzureURL,
+  getModelMaxTokens,
+  genAzureChatCompletion,
 } = require('~/utils');
-const { encodeAndFormat, validateVisionModel } = require('~/server/services/Files/images');
-const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts');
+const {
+  truncateText,
+  formatMessage,
+  CUT_OFF_PROMPT,
+  titleInstruction,
+  createContextHandlers,
+} = require('./prompts');
+const { encodeAndFormat } = require('~/server/services/Files/images/encode');
+const { isEnabled, sleep } = require('~/server/utils');
 const { handleOpenAIErrors } = require('./tools/util');
 const spendTokens = require('~/models/spendTokens');
 const { createLLM, RunManager } = require('./llm');
 const ChatGPTClient = require('./ChatGPTClient');
-const { isEnabled } = require('~/server/utils');
-const { getFiles } = require('~/models/File');
 const { summaryBuffer } = require('./memory');
 const { runTitleChain } = require('./chains');
 const { tokenSplit } = require('./document');
@@ -32,7 +48,10 @@ class OpenAIClient extends BaseClient {
     super(apiKey, options);
     this.ChatGPTClient = new ChatGPTClient();
     this.buildPrompt = this.ChatGPTClient.buildPrompt.bind(this);
+    /** @type {getCompletion} */
     this.getCompletion = this.ChatGPTClient.getCompletion.bind(this);
+    /** @type {cohereChatCompletion} */
+    this.cohereChatCompletion = this.ChatGPTClient.cohereChatCompletion.bind(this);
     this.contextStrategy = options.contextStrategy
       ? options.contextStrategy.toLowerCase()
       : 'discard';
@@ -40,6 +59,10 @@ class OpenAIClient extends BaseClient {
     /** @type {AzureOptions} */
     this.azure = options.azure || false;
     this.setOptions(options);
+    this.metadata = {};
+
+    /** @type {string | undefined} - The API Completions URL */
+    this.completionsUrl;
   }
 
   // TODO: PluginsClient calls this 3x, unneeded
@@ -83,7 +106,12 @@ class OpenAIClient extends BaseClient {
       };
     }
 
-    this.checkVisionRequest(this.options.attachments);
+    this.defaultVisionModel = this.options.visionModel ?? 'gpt-4-vision-preview';
+    if (typeof this.options.attachments?.then === 'function') {
+      this.options.attachments.then((attachments) => this.checkVisionRequest(attachments));
+    } else {
+      this.checkVisionRequest(this.options.attachments);
+    }
 
     const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {};
     if (OPENROUTER_API_KEY && !this.azure) {
@@ -101,6 +129,10 @@ class OpenAIClient extends BaseClient {
       this.useOpenRouter = true;
     }
 
+    if (this.options.endpoint?.toLowerCase() === 'ollama') {
+      this.isOllama = true;
+    }
+
     this.FORCE_PROMPT =
       isEnabled(OPENAI_FORCE_PROMPT) ||
       (reverseProxy && reverseProxy.includes('completions') && !reverseProxy.includes('chat'));
@@ -131,7 +163,15 @@ class OpenAIClient extends BaseClient {
     const { isChatGptModel } = this;
     this.isUnofficialChatGptModel =
       model.startsWith('text-chat') || model.startsWith('text-davinci-002-render');
-    this.maxContextTokens = getModelMaxTokens(model) ?? 4095; // 1 less than maximum
+
+    this.maxContextTokens =
+      this.options.maxContextTokens ??
+      getModelMaxTokens(
+        model,
+        this.options.endpointType ?? this.options.endpoint,
+        this.options.endpointTokenConfig,
+      ) ??
+      4095; // 1 less than maximum
 
     if (this.shouldSummarize) {
       this.maxContextTokens = Math.floor(this.maxContextTokens / 2);
@@ -168,16 +208,6 @@ class OpenAIClient extends BaseClient {
 
     this.setupTokens();
 
-    if (!this.modelOptions.stop && !this.isVisionModel) {
-      const stopTokens = [this.startToken];
-      if (this.endToken && this.endToken !== this.startToken) {
-        stopTokens.push(this.endToken);
-      }
-      stopTokens.push(`\n${this.userLabel}:`);
-      stopTokens.push('<|diff_marker|>');
-      this.modelOptions.stop = stopTokens;
-    }
-
     if (reverseProxy) {
       this.completionsUrl = reverseProxy;
       this.langchainProxy = extractBaseURL(reverseProxy);
@@ -208,19 +238,55 @@ class OpenAIClient extends BaseClient {
    * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
    * - Sets `this.isVisionModel` to `true` if vision request.
    * - Deletes `this.modelOptions.stop` if vision request.
-   * @param {Array<Promise<MongoFile[]> | MongoFile[]> | Record<string, MongoFile[]>} attachments
+   * @param {MongoFile[]} attachments
    */
   checkVisionRequest(attachments) {
-    this.isVisionModel = validateVisionModel(this.modelOptions.model);
-
-    if (attachments && !this.isVisionModel) {
-      this.modelOptions.model = 'gpt-4-vision-preview';
-      this.isVisionModel = true;
+    if (!attachments) {
+      return;
     }
 
+    const availableModels = this.options.modelsConfig?.[this.options.endpoint];
+    if (!availableModels) {
+      return;
+    }
+
+    let visionRequestDetected = false;
+    for (const file of attachments) {
+      if (file?.type?.includes('image')) {
+        visionRequestDetected = true;
+        break;
+      }
+    }
+    if (!visionRequestDetected) {
+      return;
+    }
+
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
     if (this.isVisionModel) {
       delete this.modelOptions.stop;
+      return;
     }
+
+    for (const model of availableModels) {
+      if (!validateVisionModel({ model, availableModels })) {
+        continue;
+      }
+      this.modelOptions.model = model;
+      this.isVisionModel = true;
+      delete this.modelOptions.stop;
+      return;
+    }
+
+    if (!availableModels.includes(this.defaultVisionModel)) {
+      return;
+    }
+    if (!validateVisionModel({ model: this.defaultVisionModel, availableModels })) {
+      return;
+    }
+
+    this.modelOptions.model = this.defaultVisionModel;
+    this.isVisionModel = true;
+    delete this.modelOptions.stop;
   }
 
   setupTokens() {
@@ -242,7 +308,7 @@ class OpenAIClient extends BaseClient {
     let tokenizer;
     this.encoding = 'text-davinci-003';
     if (this.isChatCompletion) {
-      this.encoding = 'cl100k_base';
+      this.encoding = this.modelOptions.model.includes('gpt-4o') ? 'o200k_base' : 'cl100k_base';
       tokenizer = this.constructor.getTokenizer(this.encoding);
     } else if (this.isUnofficialChatGptModel) {
       const extendSpecialTokens = {
@@ -347,10 +413,14 @@ class OpenAIClient extends BaseClient {
 
   getSaveOptions() {
     return {
+      maxContextTokens: this.options.maxContextTokens,
       chatGptLabel: this.options.chatGptLabel,
       promptPrefix: this.options.promptPrefix,
-      resendImages: this.options.resendImages,
+      resendFiles: this.options.resendFiles,
       imageDetail: this.options.imageDetail,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
@@ -363,54 +433,6 @@ class OpenAIClient extends BaseClient {
     };
   }
 
-  /**
-   *
-   * @param {TMessage[]} _messages
-   * @returns {TMessage[]}
-   */
-  async addPreviousAttachments(_messages) {
-    if (!this.options.resendImages) {
-      return _messages;
-    }
-
-    /**
-     *
-     * @param {TMessage} message
-     */
-    const processMessage = async (message) => {
-      if (!this.message_file_map) {
-        /** @type {Record<string, MongoFile[]> */
-        this.message_file_map = {};
-      }
-
-      const fileIds = message.files.map((file) => file.file_id);
-      const files = await getFiles({
-        file_id: { $in: fileIds },
-      });
-
-      await this.addImageURLs(message, files);
-
-      this.message_file_map[message.messageId] = files;
-      return message;
-    };
-
-    const promises = [];
-
-    for (const message of _messages) {
-      if (!message.files) {
-        promises.push(message);
-        continue;
-      }
-
-      promises.push(processMessage(message));
-    }
-
-    const messages = await Promise.all(promises);
-
-    this.checkVisionRequest(this.message_file_map);
-    return messages;
-  }
-
   /**
    *
    * Adds image URLs to the message object and returns the files
@@ -420,9 +442,12 @@ class OpenAIClient extends BaseClient {
    * @returns {Promise<MongoFile[]>}
    */
   async addImageURLs(message, attachments) {
-    const { files, image_urls } = await encodeAndFormat(this.options.req, attachments);
-
-    message.image_urls = image_urls;
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      this.options.endpoint,
+    );
+    message.image_urls = image_urls.length ? image_urls : undefined;
     return files;
   }
 
@@ -450,23 +475,9 @@ class OpenAIClient extends BaseClient {
     let promptTokens;
 
     promptPrefix = (promptPrefix || this.options.promptPrefix || '').trim();
-    if (promptPrefix) {
-      promptPrefix = `Instructions:\n${promptPrefix}`;
-      instructions = {
-        role: 'system',
-        name: 'instructions',
-        content: promptPrefix,
-      };
-
-      if (this.contextStrategy) {
-        instructions.tokenCount = this.getTokenCountForMessage(instructions);
-      }
-    }
 
     if (this.options.attachments) {
-      const attachments = (await this.options.attachments).filter((file) =>
-        file.type.includes('image'),
-      );
+      const attachments = await this.options.attachments;
 
       if (this.message_file_map) {
         this.message_file_map[orderedMessages[orderedMessages.length - 1].messageId] = attachments;
@@ -484,6 +495,13 @@ class OpenAIClient extends BaseClient {
       this.options.attachments = files;
     }
 
+    if (this.message_file_map) {
+      this.contextHandlers = createContextHandlers(
+        this.options.req,
+        orderedMessages[orderedMessages.length - 1].text,
+      );
+    }
+
     const formattedMessages = orderedMessages.map((message, i) => {
       const formattedMessage = formatMessage({
         message,
@@ -502,6 +520,11 @@ class OpenAIClient extends BaseClient {
       if (this.message_file_map && this.message_file_map[message.messageId]) {
         const attachments = this.message_file_map[message.messageId];
         for (const file of attachments) {
+          if (file.embedded) {
+            this.contextHandlers?.processFile(file);
+            continue;
+          }
+
           orderedMessages[i].tokenCount += this.calculateImageTokenCost({
             width: file.width,
             height: file.height,
@@ -513,6 +536,24 @@ class OpenAIClient extends BaseClient {
       return formattedMessage;
     });
 
+    if (this.contextHandlers) {
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      promptPrefix = this.augmentedPrompt + promptPrefix;
+    }
+
+    if (promptPrefix) {
+      promptPrefix = `Instructions:\n${promptPrefix.trim()}`;
+      instructions = {
+        role: 'system',
+        name: 'instructions',
+        content: promptPrefix,
+      };
+
+      if (this.contextStrategy) {
+        instructions.tokenCount = this.getTokenCountForMessage(instructions);
+      }
+    }
+
     // TODO: need to handle interleaving instructions better
     if (this.contextStrategy) {
       ({ payload, tokenCountMap, promptTokens, messages } = await this.handleContextStrategy({
@@ -540,6 +581,7 @@ class OpenAIClient extends BaseClient {
     return result;
   }
 
+  /** @type {sendCompletion} */
   async sendCompletion(payload, opts = {}) {
     let reply = '';
     let result = null;
@@ -548,7 +590,7 @@ class OpenAIClient extends BaseClient {
     const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null;
     const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion);
     if (typeof opts.onProgress === 'function' && useOldMethod) {
-      await this.getCompletion(
+      const completionResult = await this.getCompletion(
         payload,
         (progressMessage) => {
           if (progressMessage === '[DONE]') {
@@ -581,12 +623,16 @@ class OpenAIClient extends BaseClient {
           opts.onProgress(token);
           reply += token;
         },
+        opts.onProgress,
         opts.abortController || new AbortController(),
       );
+
+      if (completionResult && typeof completionResult === 'string') {
+        reply = completionResult;
+      }
     } else if (typeof opts.onProgress === 'function' || this.options.useChatCompletion) {
       reply = await this.chatCompletion({
         payload,
-        clientOptions: opts,
         onProgress: opts.onProgress,
         abortController: opts.abortController,
       });
@@ -594,9 +640,14 @@ class OpenAIClient extends BaseClient {
       result = await this.getCompletion(
         payload,
         null,
+        opts.onProgress,
         opts.abortController || new AbortController(),
       );
 
+      if (result && typeof result === 'string') {
+        return result.trim();
+      }
+
       logger.debug('[OpenAIClient] sendCompletion: result', result);
 
       if (this.isChatCompletion) {
@@ -606,11 +657,11 @@ class OpenAIClient extends BaseClient {
       }
     }
 
-    if (streamResult && typeof opts.addMetadata === 'function') {
+    if (streamResult) {
       const { finish_reason } = streamResult.choices[0];
-      opts.addMetadata({ finish_reason });
+      this.metadata = { finish_reason };
     }
-    return reply.trim();
+    return (reply ?? '').trim();
   }
 
   initializeLLM({
@@ -624,6 +675,7 @@ class OpenAIClient extends BaseClient {
     context,
     tokenBuffer,
     initialMessageCount,
+    conversationId,
   }) {
     const modelOptions = {
       modelName: modelName ?? model,
@@ -653,6 +705,16 @@ class OpenAIClient extends BaseClient {
       };
     }
 
+    const { headers } = this.options;
+    if (headers && typeof headers === 'object' && !Array.isArray(headers)) {
+      configOptions.baseOptions = {
+        headers: resolveHeaders({
+          ...headers,
+          ...configOptions?.baseOptions?.headers,
+        }),
+      };
+    }
+
     if (this.options.proxy) {
       configOptions.httpAgent = new HttpsProxyAgent(this.options.proxy);
       configOptions.httpsAgent = new HttpsProxyAgent(this.options.proxy);
@@ -671,7 +733,7 @@ class OpenAIClient extends BaseClient {
       callbacks: runManager.createCallbacks({
         context,
         tokenBuffer,
-        conversationId: this.conversationId,
+        conversationId: this.conversationId ?? conversationId,
         initialMessageCount,
       }),
     });
@@ -687,12 +749,19 @@ class OpenAIClient extends BaseClient {
    *
    * @param {Object} params - The parameters for the conversation title generation.
    * @param {string} params.text - The user's input.
+   * @param {string} [params.conversationId] - The current conversationId, if not already defined on client initialization.
    * @param {string} [params.responseText=''] - The AI's immediate response to the user.
    *
    * @returns {Promise<string | 'New Chat'>} A promise that resolves to the generated conversation title.
    *                            In case of failure, it will return the default title, "New Chat".
    */
-  async titleConvo({ text, responseText = '' }) {
+  async titleConvo({ text, conversationId, responseText = '' }) {
+    this.conversationId = conversationId;
+
+    if (this.options.attachments) {
+      delete this.options.attachments;
+    }
+
     let title = 'New Chat';
     const convo = `||>User:
 "${truncateText(text)}"
@@ -701,7 +770,10 @@ class OpenAIClient extends BaseClient {
 
     const { OPENAI_TITLE_MODEL } = process.env ?? {};
 
-    const model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+    let model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+    if (model === Constants.CURRENT_MODEL) {
+      model = this.modelOptions.model;
+    }
 
     const modelOptions = {
       // TODO: remove the gpt fallback and make it specific to endpoint
@@ -712,6 +784,39 @@ class OpenAIClient extends BaseClient {
       max_tokens: 16,
     };
 
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const resetTitleOptions = !!(
+      (this.azure && azureConfig) ||
+      (azureConfig && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    );
+
+    if (resetTitleOptions) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+
+      this.options.headers = resolveHeaders(headers);
+      this.options.reverseProxyUrl = baseURL ?? null;
+      this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      this.options.forcePrompt = azureConfig.groupMap[groupName].forcePrompt;
+      this.azure = !serverless && azureOptions;
+    }
+
     const titleChatCompletion = async () => {
       modelOptions.model = model;
 
@@ -722,9 +827,8 @@ class OpenAIClient extends BaseClient {
 
       const instructionsPayload = [
         {
-          role: 'system',
-          content: `Detect user language and write in the same language an extremely concise title for this conversation, which you must accurately detect.
-Write in the detected language. Title in 5 Words or Less. No Punctuation or Quotation. Do not mention the language. All first letters of every word should be capitalized and write the title in User Language only.
+          role: this.options.titleMessageRole ?? 'system',
+          content: `Please generate ${titleInstruction}
 
 ${convo}
 
@@ -732,10 +836,22 @@ ${convo}
         },
       ];
 
+      const promptTokens = this.getTokenCountForMessage(instructionsPayload[0]);
+
       try {
+        let useChatCompletion = true;
+
+        if (this.options.reverseProxyUrl === CohereConstants.API_URL) {
+          useChatCompletion = false;
+        }
+
         title = (
-          await this.sendPayload(instructionsPayload, { modelOptions, useChatCompletion: true })
+          await this.sendPayload(instructionsPayload, { modelOptions, useChatCompletion })
         ).replaceAll('"', '');
+
+        const completionTokens = this.getTokenCount(title);
+
+        this.recordTokenUsage({ promptTokens, completionTokens, context: 'title' });
       } catch (e) {
         logger.error(
           '[OpenAIClient] There was an issue generating the title with the completion method',
@@ -752,7 +868,13 @@ ${convo}
 
     try {
       this.abortController = new AbortController();
-      const llm = this.initializeLLM({ ...modelOptions, context: 'title', tokenBuffer: 150 });
+      const llm = this.initializeLLM({
+        ...modelOptions,
+        conversationId,
+        context: 'title',
+        tokenBuffer: 150,
+      });
+
       title = await runTitleChain({ llm, text, convo, signal: this.abortController.signal });
     } catch (e) {
       if (e?.message?.toLowerCase()?.includes('abort')) {
@@ -778,8 +900,17 @@ ${convo}
 
     // TODO: remove the gpt fallback and make it specific to endpoint
     const { OPENAI_SUMMARY_MODEL = 'gpt-3.5-turbo' } = process.env ?? {};
-    const model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL;
-    const maxContextTokens = getModelMaxTokens(model) ?? 4095;
+    let model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL;
+    if (model === Constants.CURRENT_MODEL) {
+      model = this.modelOptions.model;
+    }
+
+    const maxContextTokens =
+      getModelMaxTokens(
+        model,
+        this.options.endpointType ?? this.options.endpoint,
+        this.options.endpointTokenConfig,
+      ) ?? 4095; // 1 less than maximum
 
     // 3 tokens for the assistant label, and 98 for the summarizer prompt (101)
     let promptBuffer = 101;
@@ -877,14 +1008,14 @@ ${convo}
     }
   }
 
-  async recordTokenUsage({ promptTokens, completionTokens }) {
-    logger.debug('[OpenAIClient] recordTokenUsage:', { promptTokens, completionTokens });
+  async recordTokenUsage({ promptTokens, completionTokens, context = 'message' }) {
     await spendTokens(
       {
-        user: this.user,
+        context,
         model: this.modelOptions.model,
-        context: 'message',
         conversationId: this.conversationId,
+        user: this.user ?? this.options.req.user?.id,
+        endpointTokenConfig: this.options.endpointTokenConfig,
       },
       { promptTokens, completionTokens },
     );
@@ -897,7 +1028,7 @@ ${convo}
     });
   }
 
-  async chatCompletion({ payload, onProgress, clientOptions, abortController = null }) {
+  async chatCompletion({ payload, onProgress, abortController = null }) {
     let error = null;
     const errorCallback = (err) => (error = err);
     let intermediateReply = '';
@@ -918,15 +1049,6 @@ ${convo}
       }
 
       const baseURL = extractBaseURL(this.completionsUrl);
-      // let { messages: _msgsToLog, ...modelOptionsToLog } = modelOptions;
-      // if (modelOptionsToLog.messages) {
-      //   _msgsToLog = modelOptionsToLog.messages.map((msg) => {
-      //     let { content, ...rest } = msg;
-
-      //     if (content)
-      //     return { ...rest, content: truncateText(content) };
-      //   });
-      // }
       logger.debug('[OpenAIClient] chatCompletion', { baseURL, modelOptions });
       const opts = {
         baseURL,
@@ -951,16 +1073,54 @@ ${convo}
         modelOptions.max_tokens = 4000;
       }
 
+      /** @type {TAzureConfig | undefined} */
+      const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+      if (
+        (this.azure && this.isVisionModel && azureConfig) ||
+        (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+      ) {
+        const { modelGroupMap, groupMap } = azureConfig;
+        const {
+          azureOptions,
+          baseURL,
+          headers = {},
+          serverless,
+        } = mapModelToAzureConfig({
+          modelName: modelOptions.model,
+          modelGroupMap,
+          groupMap,
+        });
+        opts.defaultHeaders = resolveHeaders(headers);
+        this.langchainProxy = extractBaseURL(baseURL);
+        this.apiKey = azureOptions.azureOpenAIApiKey;
+
+        const groupName = modelGroupMap[modelOptions.model].group;
+        this.options.addParams = azureConfig.groupMap[groupName].addParams;
+        this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+        // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+        this.azure = !serverless && azureOptions;
+        this.azureEndpoint =
+          !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+      }
+
       if (this.azure || this.options.azure) {
-        // Azure does not accept `model` in the body, so we need to remove it.
+        /* Azure Bug, extremely short default `max_tokens` response */
+        if (!modelOptions.max_tokens && modelOptions.model === 'gpt-4-vision-preview') {
+          modelOptions.max_tokens = 4000;
+        }
+
+        /* Azure does not accept `model` in the body, so we need to remove it. */
         delete modelOptions.model;
 
         opts.baseURL = this.langchainProxy
           ? constructAzureURL({
             baseURL: this.langchainProxy,
-            azure: this.azure,
+            azureOptions: this.azure,
           })
-          : this.azureEndpoint.split(/\/(chat|completion)/)[0];
+          : this.azureEndpoint.split(/(?<!\/)\/(chat|completion)\//)[0];
+
         opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
         opts.defaultHeaders = { ...opts.defaultHeaders, 'api-key': this.apiKey };
       }
@@ -970,17 +1130,35 @@ ${convo}
       }
 
       let chatCompletion;
+      /** @type {OpenAI} */
       const openai = new OpenAI({
+        fetch: this.fetch,
         apiKey: this.apiKey,
         ...opts,
       });
 
-      /* hacky fix for Mistral AI API not allowing a singular system message in payload */
-      if (opts.baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+      /* Re-orders system message to the top of the messages payload, as not allowed anywhere else */
+      if (modelOptions.messages && (opts.baseURL.includes('api.mistral.ai') || this.isOllama)) {
         const { messages } = modelOptions;
-        if (messages.length === 1 && messages[0].role === 'system') {
-          modelOptions.messages[0].role = 'user';
+
+        const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+        if (systemMessageIndex > 0) {
+          const [systemMessage] = messages.splice(systemMessageIndex, 1);
+          messages.unshift(systemMessage);
         }
+
+        modelOptions.messages = messages;
+      }
+
+      /* If there is only one message and it's a system message, change the role to user */
+      if (
+        (opts.baseURL.includes('api.mistral.ai') || opts.baseURL.includes('api.perplexity.ai')) &&
+        modelOptions.messages &&
+        modelOptions.messages.length === 1 &&
+        modelOptions.messages[0]?.role === 'system'
+      ) {
+        modelOptions.messages[0].role = 'user';
       }
 
       if (this.options.addParams && typeof this.options.addParams === 'object') {
@@ -988,12 +1166,31 @@ ${convo}
           ...modelOptions,
           ...this.options.addParams,
         };
+        logger.debug('[OpenAIClient] chatCompletion: added params', {
+          addParams: this.options.addParams,
+          modelOptions,
+        });
       }
 
       if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
         this.options.dropParams.forEach((param) => {
           delete modelOptions[param];
         });
+        logger.debug('[OpenAIClient] chatCompletion: dropped params', {
+          dropParams: this.options.dropParams,
+          modelOptions,
+        });
+      }
+
+      const streamRate = this.options.streamRate ?? Constants.DEFAULT_STREAM_RATE;
+
+      if (this.message_file_map && this.isOllama) {
+        const ollamaClient = new OllamaClient({ baseURL, streamRate });
+        return await ollamaClient.chatCompletion({
+          payload: modelOptions,
+          onProgress,
+          abortController,
+        });
       }
 
       let UnexpectedRoleError = false;
@@ -1009,6 +1206,16 @@ ${convo}
           .on('error', (err) => {
             handleOpenAIErrors(err, errorCallback, 'stream');
           })
+          .on('finalChatCompletion', (finalChatCompletion) => {
+            const finalMessage = finalChatCompletion?.choices?.[0]?.message;
+            if (finalMessage && finalMessage?.role !== 'assistant') {
+              finalChatCompletion.choices[0].message.role = 'assistant';
+            }
+
+            if (finalMessage && !finalMessage?.content?.trim()) {
+              finalChatCompletion.choices[0].message.content = intermediateReply;
+            }
+          })
           .on('finalMessage', (message) => {
             if (message?.role !== 'assistant') {
               stream.messages.push({ role: 'assistant', content: intermediateReply });
@@ -1024,6 +1231,8 @@ ${convo}
             stream.controller.abort();
             break;
           }
+
+          await sleep(streamRate);
         }
 
         if (!UnexpectedRoleError) {
@@ -1054,12 +1263,20 @@ ${convo}
       }
 
       const { message, finish_reason } = chatCompletion.choices[0];
-      if (chatCompletion && typeof clientOptions.addMetadata === 'function') {
-        clientOptions.addMetadata({ finish_reason });
+      if (chatCompletion) {
+        this.metadata = { finish_reason };
       }
 
       logger.debug('[OpenAIClient] chatCompletion response', chatCompletion);
 
+      if (!message?.content?.trim() && intermediateReply.length) {
+        logger.debug(
+          '[OpenAIClient] chatCompletion: using intermediateReply due to empty message.content',
+          { intermediateReply },
+        );
+        return intermediateReply;
+      }
+
       return message.content;
     } catch (err) {
       if (
@@ -1072,6 +1289,9 @@ ${convo}
         err?.message?.includes(
           'OpenAI error: Invalid final message: OpenAI expects final message to include role=assistant',
         ) ||
+        err?.message?.includes(
+          'stream ended without producing a ChatCompletionMessage with role=assistant',
+        ) ||
         err?.message?.includes('The server had an error processing your request') ||
         err?.message?.includes('missing finish_reason') ||
         err?.message?.includes('missing role') ||

api/app/clients/PluginsClient.js

@@ -1,8 +1,10 @@
 const OpenAIClient = require('./OpenAIClient');
 const { CallbackManager } = require('langchain/callbacks');
+const { CacheKeys, Time } = require('librechat-data-provider');
 const { BufferMemory, ChatMessageHistory } = require('langchain/memory');
 const { initializeCustomAgent, initializeFunctionsAgent } = require('./agents');
 const { addImages, buildErrorInput, buildPromptPrefix } = require('./output_parsers');
+const { processFileURL } = require('~/server/services/Files/process');
 const { EModelEndpoint } = require('librechat-data-provider');
 const { formatLangChainMessages } = require('./prompts');
 const checkBalance = require('~/models/checkBalance');
@@ -10,6 +12,7 @@ const { SelfReflectionTool } = require('./tools');
 const { isEnabled } = require('~/server/utils');
 const { extractBaseURL } = require('~/utils');
 const { loadTools } = require('./tools/util');
+const { getLogStores } = require('~/cache');
 const { logger } = require('~/config');
 
 class PluginsClient extends OpenAIClient {
@@ -30,10 +33,6 @@ class PluginsClient extends OpenAIClient {
 
     super.setOptions(options);
 
-    if (this.functionsAgent && this.agentOptions.model && !this.useOpenRouter) {
-      this.agentOptions.model = this.getFunctionModelName(this.agentOptions.model);
-    }
-
     this.isGpt3 = this.modelOptions?.model?.includes('gpt-3');
 
     if (this.options.reverseProxyUrl) {
@@ -45,8 +44,12 @@ class PluginsClient extends OpenAIClient {
     return {
       chatGptLabel: this.options.chatGptLabel,
       promptPrefix: this.options.promptPrefix,
+      tools: this.options.tools,
       ...this.modelOptions,
       agentOptions: this.agentOptions,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
     };
   }
 
@@ -113,6 +116,7 @@ class PluginsClient extends OpenAIClient {
         openAIApiKey: this.openAIApiKey,
         conversationId: this.conversationId,
         fileStrategy: this.options.req.app.locals.fileStrategy,
+        processFileURL,
         message,
       },
     });
@@ -146,9 +150,11 @@ class PluginsClient extends OpenAIClient {
       signal,
       pastMessages,
       tools: this.tools,
-      currentDateString: this.currentDateString,
       verbose: this.options.debug,
       returnIntermediateSteps: true,
+      customName: this.options.chatGptLabel,
+      currentDateString: this.currentDateString,
+      customInstructions: this.options.promptPrefix,
       callbackManager: CallbackManager.fromHandlers({
         async handleAgentAction(action, runId) {
           handleAction(action, runId, onAgentAction);
@@ -216,6 +222,13 @@ class PluginsClient extends OpenAIClient {
     }
   }
 
+  /**
+   *
+   * @param {TMessage} responseMessage
+   * @param {Partial<TMessage>} saveOptions
+   * @param {string} user
+   * @returns
+   */
   async handleResponseMessage(responseMessage, saveOptions, user) {
     const { output, errorMessage, ...result } = this.result;
     logger.debug('[PluginsClient][handleResponseMessage] Output:', {
@@ -234,19 +247,40 @@ class PluginsClient extends OpenAIClient {
       await this.recordTokenUsage(responseMessage);
     }
 
-    await this.saveMessageToDatabase(responseMessage, saveOptions, user);
+    this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user);
+    const messageCache = getLogStores(CacheKeys.MESSAGES);
+    messageCache.set(
+      responseMessage.messageId,
+      {
+        text: responseMessage.text,
+        complete: true,
+      },
+      Time.FIVE_MINUTES,
+    );
     delete responseMessage.tokenCount;
     return { ...responseMessage, ...result };
   }
 
   async sendMessage(message, opts = {}) {
+    /** @type {{ filteredTools: string[], includedTools: string[] }} */
+    const { filteredTools = [], includedTools = [] } = this.options.req.app.locals;
+
+    if (includedTools.length > 0) {
+      const tools = this.options.tools.filter((plugin) => includedTools.includes(plugin));
+      this.options.tools = tools;
+    } else {
+      const tools = this.options.tools.filter((plugin) => !filteredTools.includes(plugin));
+      this.options.tools = tools;
+    }
+
     // If a message is edited, no tools can be used.
     const completionMode = this.options.tools.length === 0 || opts.isEdited;
     if (completionMode) {
       this.setOptions(opts);
       return super.sendMessage(message, opts);
     }
-    logger.debug('[PluginsClient] sendMessage', { message, opts });
+
+    logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts });
     const {
       user,
       isEdited,
@@ -260,6 +294,14 @@ class PluginsClient extends OpenAIClient {
       onToolEnd,
     } = await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     this.currentMessages.push(userMessage);
 
     let {
@@ -288,7 +330,15 @@ class PluginsClient extends OpenAIClient {
     if (payload) {
       this.currentMessages = payload;
     }
-    await this.saveMessageToDatabase(userMessage, saveOptions, user);
+
+    if (!this.skipSaveUserMessage) {
+      this.userMessagePromise = this.saveMessageToDatabase(userMessage, saveOptions, user);
+      if (typeof opts?.getReqData === 'function') {
+        opts.getReqData({
+          userMessagePromise: this.userMessagePromise,
+        });
+      }
+    }
 
     if (isEnabled(process.env.CHECK_BALANCE)) {
       await checkBalance({
@@ -306,6 +356,8 @@ class PluginsClient extends OpenAIClient {
     }
 
     const responseMessage = {
+      endpoint: EModelEndpoint.gptPlugins,
+      iconURL: this.options.iconURL,
       messageId: responseMessageId,
       conversationId,
       parentMessageId: userMessage.messageId,

api/app/clients/agents/CustomAgent/initializeCustomAgent.js

@@ -13,10 +13,18 @@ const initializeCustomAgent = async ({
   tools,
   model,
   pastMessages,
+  customName,
+  customInstructions,
   currentDateString,
   ...rest
 }) => {
   let prompt = CustomAgent.createPrompt(tools, { currentDateString, model: model.modelName });
+  if (customName) {
+    prompt = `You are "${customName}".\n${prompt}`;
+  }
+  if (customInstructions) {
+    prompt = `${prompt}\n${customInstructions}`;
+  }
 
   const chatPrompt = ChatPromptTemplate.fromMessages([
     new SystemMessagePromptTemplate(prompt),

api/app/clients/agents/CustomAgent/instructions.js

@@ -1,44 +1,3 @@
-/*
-module.exports = `You are ChatGPT, a Large Language model with useful tools.
-
-Talk to the human and provide meaningful answers when questions are asked.
-
-Use the tools when you need them, but use your own knowledge if you are confident of the answer. Keep answers short and concise.
-
-A tool is not usually needed for creative requests, so do your best to answer them without tools.
-
-Avoid repeating identical answers if it appears before. Only fulfill the human's requests, do not create extra steps beyond what the human has asked for.
-
-Your input for 'Action' should be the name of tool used only.
-
-Be honest. If you can't answer something, or a tool is not appropriate, say you don't know or answer to the best of your ability.
-
-Attempt to fulfill the human's requests in as few actions as possible`;
-*/
-
-// module.exports = `You are ChatGPT, a highly knowledgeable and versatile large language model.
-
-// Engage with the Human conversationally, providing concise and meaningful answers to questions. Utilize built-in tools when necessary, except for creative requests, where relying on your own knowledge is preferred. Aim for variety and avoid repetitive answers.
-
-// For your 'Action' input, state the name of the tool used only, and honor user requests without adding extra steps. Always be honest; if you cannot provide an appropriate answer or tool, admit that or do your best.
-
-// Strive to meet the user's needs efficiently with minimal actions.`;
-
-// import {
-//   BasePromptTemplate,
-//   BaseStringPromptTemplate,
-//   SerializedBasePromptTemplate,
-//   renderTemplate,
-// } from "langchain/prompts";
-
-// prefix: `You are ChatGPT, a highly knowledgeable and versatile large language model.
-// Your objective is to help users by understanding their intent and choosing the best action. Prioritize direct, specific responses. Use concise, varied answers and rely on your knowledge for creative tasks. Utilize tools when needed, and structure results for machine compatibility.
-// prefix: `Objective: to comprehend human intentions based on user input and available tools. Goal: identify the best action to directly address the human's query. In your subsequent steps, you will utilize the chosen action. You may select multiple actions and list them in a meaningful order. Prioritize actions that directly relate to the user's query over general ones. Ensure that the generated thought is highly specific and explicit to best match the user's expectations. Construct the result in a manner that an online open-API would most likely expect. Provide concise and meaningful answers to human queries. Utilize tools when necessary. Relying on your own knowledge is preferred for creative requests. Aim for variety and avoid repetitive answers.
-
-// # Available Actions & Tools:
-// N/A: no suitable action, use your own knowledge.`,
-// suffix: `Remember, all your responses MUST adhere to the described format and only respond if the format is followed. Output exactly with the requested format, avoiding any other text as this will be parsed by a machine. Following 'Action:', provide only one of the actions listed above. If a tool is not necessary, deduce this quickly and finish your response. Honor the human's requests without adding extra steps. Carry out tasks in the sequence written by the human. Always be honest; if you cannot provide an appropriate answer or tool, do your best with your own knowledge. Strive to meet the user's needs efficiently with minimal actions.`;
-
 module.exports = {
   'gpt3-v1': {
     prefix: `Objective: Understand human intentions using user input and available tools. Goal: Identify the most suitable actions to directly address user queries.

api/app/clients/agents/Functions/initializeFunctionsAgent.js

@@ -10,6 +10,8 @@ const initializeFunctionsAgent = async ({
   tools,
   model,
   pastMessages,
+  customName,
+  customInstructions,
   currentDateString,
   ...rest
 }) => {
@@ -24,7 +26,13 @@ const initializeFunctionsAgent = async ({
     returnMessages: true,
   });
 
-  const prefix = addToolDescriptions(`Current Date: ${currentDateString}\n${PREFIX}`, tools);
+  let prefix = addToolDescriptions(`Current Date: ${currentDateString}\n${PREFIX}`, tools);
+  if (customName) {
+    prefix = `You are "${customName}".\n${prefix}`;
+  }
+  if (customInstructions) {
+    prefix = `${prefix}\n${customInstructions}`;
+  }
 
   return await initializeAgentExecutorWithOptions(tools, model, {
     agentType: 'openai-functions',

api/app/clients/llm/createCoherePayload.js

@@ -0,0 +1,85 @@
+const { CohereConstants } = require('librechat-data-provider');
+const { titleInstruction } = require('../prompts/titlePrompts');
+
+// Mapping OpenAI roles to Cohere roles
+const roleMap = {
+  user: CohereConstants.ROLE_USER,
+  assistant: CohereConstants.ROLE_CHATBOT,
+  system: CohereConstants.ROLE_SYSTEM, // Recognize and map the system role explicitly
+};
+
+/**
+ * Adjusts an OpenAI ChatCompletionPayload to conform with Cohere's expected chat payload format.
+ * Now includes handling for "system" roles explicitly mentioned.
+ *
+ * @param {Object} options - Object containing the model options.
+ * @param {ChatCompletionPayload} options.modelOptions - The OpenAI model payload options.
+ * @returns {CohereChatStreamRequest} Cohere-compatible chat API payload.
+ */
+function createCoherePayload({ modelOptions }) {
+  /** @type {string | undefined} */
+  let preamble;
+  let latestUserMessageContent = '';
+  const {
+    stream,
+    stop,
+    top_p,
+    temperature,
+    frequency_penalty,
+    presence_penalty,
+    max_tokens,
+    messages,
+    model,
+    ...rest
+  } = modelOptions;
+
+  // Filter out the latest user message and transform remaining messages to Cohere's chat_history format
+  let chatHistory = messages.reduce((acc, message, index, arr) => {
+    const isLastUserMessage = index === arr.length - 1 && message.role === 'user';
+
+    const messageContent =
+      typeof message.content === 'string'
+        ? message.content
+        : message.content.map((part) => (part.type === 'text' ? part.text : '')).join(' ');
+
+    if (isLastUserMessage) {
+      latestUserMessageContent = messageContent;
+    } else {
+      acc.push({
+        role: roleMap[message.role] || CohereConstants.ROLE_USER,
+        message: messageContent,
+      });
+    }
+
+    return acc;
+  }, []);
+
+  if (
+    chatHistory.length === 1 &&
+    chatHistory[0].role === CohereConstants.ROLE_SYSTEM &&
+    !latestUserMessageContent.length
+  ) {
+    const message = chatHistory[0].message;
+    latestUserMessageContent = message.includes(titleInstruction)
+      ? CohereConstants.TITLE_MESSAGE
+      : '.';
+    preamble = message;
+  }
+
+  return {
+    message: latestUserMessageContent,
+    model: model,
+    chatHistory,
+    stream: stream ?? false,
+    temperature: temperature,
+    frequencyPenalty: frequency_penalty,
+    presencePenalty: presence_penalty,
+    maxTokens: max_tokens,
+    stopSequences: stop,
+    preamble,
+    p: top_p,
+    ...rest,
+  };
+}
+
+module.exports = createCoherePayload;

api/app/clients/llm/createLLM.js

@@ -55,16 +55,18 @@ function createLLM({
   }
 
   if (azure && configOptions.basePath) {
-    configOptions.basePath = constructAzureURL({
+    const azureURL = constructAzureURL({
       baseURL: configOptions.basePath,
-      azure: azureOptions,
+      azureOptions,
     });
+    azureOptions.azureOpenAIBasePath = azureURL.split(
+      `/${azureOptions.azureOpenAIApiDeploymentName}`,
+    )[0];
   }
 
   return new ChatOpenAI(
     {
       streaming,
-      verbose: true,
       credentials,
       configuration,
       ...azureOptions,

api/app/clients/llm/index.js

@@ -1,7 +1,9 @@
 const createLLM = require('./createLLM');
 const RunManager = require('./RunManager');
+const createCoherePayload = require('./createCoherePayload');
 
 module.exports = {
   createLLM,
   RunManager,
+  createCoherePayload,
 };

api/app/clients/prompts/createContextHandlers.js

@@ -0,0 +1,160 @@
+const axios = require('axios');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
+
+const footer = `Use the context as your learned knowledge to better answer the user.
+
+In your response, remember to follow these guidelines:
+- If you don't know the answer, simply say that you don't know.
+- If you are unsure how to answer, ask for clarification.
+- Avoid mentioning that you obtained the information from the context.
+`;
+
+function createContextHandlers(req, userMessageContent) {
+  if (!process.env.RAG_API_URL) {
+    return;
+  }
+
+  const queryPromises = [];
+  const processedFiles = [];
+  const processedIds = new Set();
+  const jwtToken = req.headers.authorization.split(' ')[1];
+  const useFullContext = isEnabled(process.env.RAG_USE_FULL_CONTEXT);
+
+  const query = async (file) => {
+    if (useFullContext) {
+      return axios.get(`${process.env.RAG_API_URL}/documents/${file.file_id}/context`, {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+        },
+      });
+    }
+
+    return axios.post(
+      `${process.env.RAG_API_URL}/query`,
+      {
+        file_id: file.file_id,
+        query: userMessageContent,
+        k: 4,
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+  };
+
+  const processFile = async (file) => {
+    if (file.embedded && !processedIds.has(file.file_id)) {
+      try {
+        const promise = query(file);
+        queryPromises.push(promise);
+        processedFiles.push(file);
+        processedIds.add(file.file_id);
+      } catch (error) {
+        logger.error(`Error processing file ${file.filename}:`, error);
+      }
+    }
+  };
+
+  const createContext = async () => {
+    try {
+      if (!queryPromises.length || !processedFiles.length) {
+        return '';
+      }
+
+      const oneFile = processedFiles.length === 1;
+      const header = `The user has attached ${oneFile ? 'a' : processedFiles.length} file${
+        !oneFile ? 's' : ''
+      } to the conversation:`;
+
+      const files = `${
+        oneFile
+          ? ''
+          : `
+      <files>`
+      }${processedFiles
+        .map(
+          (file) => `
+              <file>
+                <filename>${file.filename}</filename>
+                <type>${file.type}</type>
+              </file>`,
+        )
+        .join('')}${
+        oneFile
+          ? ''
+          : `
+        </files>`
+      }`;
+
+      const resolvedQueries = await Promise.all(queryPromises);
+
+      const context =
+        resolvedQueries.length === 0
+          ? '\n\tThe semantic search did not return any results.'
+          : resolvedQueries
+            .map((queryResult, index) => {
+              const file = processedFiles[index];
+              let contextItems = queryResult.data;
+
+              const generateContext = (currentContext) =>
+                `
+          <file>
+            <filename>${file.filename}</filename>
+            <context>${currentContext}
+            </context>
+          </file>`;
+
+              if (useFullContext) {
+                return generateContext(`\n${contextItems}`);
+              }
+
+              contextItems = queryResult.data
+                .map((item) => {
+                  const pageContent = item[0].page_content;
+                  return `
+            <contextItem>
+              <![CDATA[${pageContent?.trim()}]]>
+            </contextItem>`;
+                })
+                .join('');
+
+              return generateContext(contextItems);
+            })
+            .join('');
+
+      if (useFullContext) {
+        const prompt = `${header}
+          ${context}
+          ${footer}`;
+
+        return prompt;
+      }
+
+      const prompt = `${header}
+        ${files}
+
+        A semantic search was executed with the user's message as the query, retrieving the following context inside <context></context> XML tags.
+
+        <context>${context}
+        </context>
+
+        ${footer}`;
+
+      return prompt;
+    } catch (error) {
+      logger.error('Error creating context:', error);
+      throw error;
+    }
+  };
+
+  return {
+    processFile,
+    createContext,
+  };
+}
+
+module.exports = createContextHandlers;

api/app/clients/prompts/createVisionPrompt.js

@@ -0,0 +1,34 @@
+/**
+ * Generates a prompt instructing the user to describe an image in detail, tailored to different types of visual content.
+ * @param {boolean} pluralized - Whether to pluralize the prompt for multiple images.
+ * @returns {string} - The generated vision prompt.
+ */
+const createVisionPrompt = (pluralized = false) => {
+  return `Please describe the image${
+    pluralized ? 's' : ''
+  } in detail, covering relevant aspects such as:
+
+  For photographs, illustrations, or artwork:
+  - The main subject(s) and their appearance, positioning, and actions
+  - The setting, background, and any notable objects or elements
+  - Colors, lighting, and overall mood or atmosphere
+  - Any interesting details, textures, or patterns
+  - The style, technique, or medium used (if discernible)
+  
+  For screenshots or images containing text:
+  - The content and purpose of the text
+  - The layout, formatting, and organization of the information
+  - Any notable visual elements, such as logos, icons, or graphics
+  - The overall context or message conveyed by the screenshot
+  
+  For graphs, charts, or data visualizations:
+  - The type of graph or chart (e.g., bar graph, line chart, pie chart)
+  - The variables being compared or analyzed
+  - Any trends, patterns, or outliers in the data
+  - The axis labels, scales, and units of measurement
+  - The title, legend, and any additional context provided
+  
+  Be as specific and descriptive as possible while maintaining clarity and concision.`;
+};
+
+module.exports = createVisionPrompt;

api/app/clients/prompts/formatMessages.js

@@ -1,3 +1,4 @@
+const { EModelEndpoint } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
 
 /**
@@ -7,10 +8,16 @@ const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
  * @param {Object} params.message - The message object to format.
  * @param {string} [params.message.role] - The role of the message sender (must be 'user').
  * @param {string} [params.message.content] - The text content of the message.
+ * @param {EModelEndpoint} [params.endpoint] - Identifier for specific endpoint handling
  * @param {Array<string>} [params.image_urls] - The image_urls to attach to the message.
  * @returns {(Object)} - The formatted message.
  */
-const formatVisionMessage = ({ message, image_urls }) => {
+const formatVisionMessage = ({ message, image_urls, endpoint }) => {
+  if (endpoint === EModelEndpoint.anthropic) {
+    message.content = [...image_urls, { type: 'text', text: message.content }];
+    return message;
+  }
+
   message.content = [{ type: 'text', text: message.content }, ...image_urls];
 
   return message;
@@ -29,10 +36,11 @@ const formatVisionMessage = ({ message, image_urls }) => {
  * @param {Array<string>} [params.message.image_urls] - The image_urls attached to the message for Vision API.
  * @param {string} [params.userName] - The name of the user.
  * @param {string} [params.assistantName] - The name of the assistant.
+ * @param {string} [params.endpoint] - Identifier for specific endpoint handling
  * @param {boolean} [params.langChain=false] - Whether to return a LangChain message object.
  * @returns {(Object|HumanMessage|AIMessage|SystemMessage)} - The formatted message.
  */
-const formatMessage = ({ message, userName, assistantName, langChain = false }) => {
+const formatMessage = ({ message, userName, assistantName, endpoint, langChain = false }) => {
   let { role: _role, _name, sender, text, content: _content, lc_id } = message;
   if (lc_id && lc_id[2] && !langChain) {
     const roleMapping = {
@@ -51,7 +59,11 @@ const formatMessage = ({ message, userName, assistantName, langChain = false })
 
   const { image_urls } = message;
   if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
-    return formatVisionMessage({ message: formattedMessage, image_urls: message.image_urls });
+    return formatVisionMessage({
+      message: formattedMessage,
+      image_urls: message.image_urls,
+      endpoint,
+    });
   }
 
   if (_name) {

api/app/clients/prompts/formatMessages.spec.js

@@ -1,5 +1,6 @@
-const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
+const { Constants } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
+const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
 
 describe('formatMessage', () => {
   it('formats user message', () => {
@@ -61,7 +62,7 @@ describe('formatMessage', () => {
         isCreatedByUser: true,
         isEdited: false,
         model: null,
-        parentMessageId: '00000000-0000-0000-0000-000000000000',
+        parentMessageId: Constants.NO_PARENT,
         sender: 'User',
         text: 'hi',
         tokenCount: 5,

api/app/clients/prompts/index.js

@@ -4,6 +4,8 @@ const handleInputs = require('./handleInputs');
 const instructions = require('./instructions');
 const titlePrompts = require('./titlePrompts');
 const truncateText = require('./truncateText');
+const createVisionPrompt = require('./createVisionPrompt');
+const createContextHandlers = require('./createContextHandlers');
 
 module.exports = {
   ...formatMessages,
@@ -11,5 +13,7 @@ module.exports = {
   ...handleInputs,
   ...instructions,
   ...titlePrompts,
-  truncateText,
+  ...truncateText,
+  createVisionPrompt,
+  createContextHandlers,
 };

api/app/clients/prompts/titlePrompts.js

@@ -27,7 +27,96 @@ ${convo}`,
   return titlePrompt;
 };
 
+const titleInstruction =
+  'a concise, 5-word-or-less title for the conversation, using its same language, with no punctuation. Apply title case conventions appropriate for the language. Never directly mention the language name or the word "title"';
+const titleFunctionPrompt = `In this environment you have access to a set of tools you can use to generate the conversation title.
+  
+You may call them like this:
+<function_calls>
+<invoke>
+<tool_name>$TOOL_NAME</tool_name>
+<parameters>
+<$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME>
+...
+</parameters>
+</invoke>
+</function_calls>
+
+Here are the tools available:
+<tools>
+<tool_description>
+<tool_name>submit_title</tool_name>
+<description>
+Submit a brief title in the conversation's language, following the parameter description closely.
+</description>
+<parameters>
+<parameter>
+<name>title</name>
+<type>string</type>
+<description>${titleInstruction}</description>
+</parameter>
+</parameters>
+</tool_description>
+</tools>`;
+
+const genTranslationPrompt = (
+  translationPrompt,
+) => `In this environment you have access to a set of tools you can use to translate text.
+  
+You may call them like this:
+<function_calls>
+<invoke>
+<tool_name>$TOOL_NAME</tool_name>
+<parameters>
+<$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME>
+...
+</parameters>
+</invoke>
+</function_calls>
+
+Here are the tools available:
+<tools>
+<tool_description>
+<tool_name>submit_translation</tool_name>
+<description>
+Submit a translation in the target language, following the parameter description and its language closely.
+</description>
+<parameters>
+<parameter>
+<name>translation</name>
+<type>string</type>
+<description>${translationPrompt}
+ONLY include the generated translation without quotations, nor its related key</description>
+</parameter>
+</parameters>
+</tool_description>
+</tools>`;
+
+/**
+ * Parses specified parameter from the provided prompt.
+ * @param {string} prompt - The prompt containing the desired parameter.
+ * @param {string} paramName - The name of the parameter to extract.
+ * @returns {string} The parsed parameter's value or a default value if not found.
+ */
+function parseParamFromPrompt(prompt, paramName) {
+  const paramRegex = new RegExp(`<${paramName}>([\\s\\S]+?)</${paramName}>`);
+  const paramMatch = prompt.match(paramRegex);
+
+  if (paramMatch && paramMatch[1]) {
+    return paramMatch[1].trim();
+  }
+
+  if (prompt && prompt.length) {
+    return `NO TOOL INVOCATION: ${prompt}`;
+  }
+  return `No ${paramName} provided`;
+}
+
 module.exports = {
   langPrompt,
+  titleInstruction,
   createTitlePrompt,
+  titleFunctionPrompt,
+  parseParamFromPrompt,
+  genTranslationPrompt,
 };

api/app/clients/prompts/truncateText.js

@@ -1,10 +1,40 @@
 const MAX_CHAR = 255;
 
-function truncateText(text) {
-  if (text.length > MAX_CHAR) {
-    return `${text.slice(0, MAX_CHAR)}... [text truncated for brevity]`;
+/**
+ * Truncates a given text to a specified maximum length, appending ellipsis and a notification
+ * if the original text exceeds the maximum length.
+ *
+ * @param {string} text - The text to be truncated.
+ * @param {number} [maxLength=MAX_CHAR] - The maximum length of the text after truncation. Defaults to MAX_CHAR.
+ * @returns {string} The truncated text if the original text length exceeds maxLength, otherwise returns the original text.
+ */
+function truncateText(text, maxLength = MAX_CHAR) {
+  if (text.length > maxLength) {
+    return `${text.slice(0, maxLength)}... [text truncated for brevity]`;
   }
   return text;
 }
 
-module.exports = truncateText;
+/**
+ * Truncates a given text to a specified maximum length by showing the first half and the last half of the text,
+ * separated by ellipsis. This method ensures the output does not exceed the maximum length, including the addition
+ * of ellipsis and notification if the original text exceeds the maximum length.
+ *
+ * @param {string} text - The text to be truncated.
+ * @param {number} [maxLength=MAX_CHAR] - The maximum length of the output text after truncation. Defaults to MAX_CHAR.
+ * @returns {string} The truncated text showing the first half and the last half, or the original text if it does not exceed maxLength.
+ */
+function smartTruncateText(text, maxLength = MAX_CHAR) {
+  const ellipsis = '...';
+  const notification = ' [text truncated for brevity]';
+  const halfMaxLength = Math.floor((maxLength - ellipsis.length - notification.length) / 2);
+
+  if (text.length > maxLength) {
+    const startLastHalf = text.length - halfMaxLength;
+    return `${text.slice(0, halfMaxLength)}${ellipsis}${text.slice(startLastHalf)}${notification}`;
+  }
+
+  return text;
+}
+
+module.exports = { truncateText, smartTruncateText };

api/app/clients/specs/AnthropicClient.test.js

@@ -1,4 +1,6 @@
-const AnthropicClient = require('../AnthropicClient');
+const { anthropicSettings } = require('librechat-data-provider');
+const AnthropicClient = require('~/app/clients/AnthropicClient');
+
 const HUMAN_PROMPT = '\n\nHuman:';
 const AI_PROMPT = '\n\nAssistant:';
 
@@ -22,7 +24,7 @@ describe('AnthropicClient', () => {
     const options = {
       modelOptions: {
         model,
-        temperature: 0.7,
+        temperature: anthropicSettings.temperature.default,
       },
     };
     client = new AnthropicClient('test-api-key');
@@ -33,7 +35,42 @@ describe('AnthropicClient', () => {
     it('should set the options correctly', () => {
       expect(client.apiKey).toBe('test-api-key');
       expect(client.modelOptions.model).toBe(model);
-      expect(client.modelOptions.temperature).toBe(0.7);
+      expect(client.modelOptions.temperature).toBe(anthropicSettings.temperature.default);
+    });
+
+    it('should set legacy maxOutputTokens for non-Claude-3 models', () => {
+      const client = new AnthropicClient('test-api-key');
+      client.setOptions({
+        modelOptions: {
+          model: 'claude-2',
+          maxOutputTokens: anthropicSettings.maxOutputTokens.default,
+        },
+      });
+      expect(client.modelOptions.maxOutputTokens).toBe(
+        anthropicSettings.legacy.maxOutputTokens.default,
+      );
+    });
+    it('should not set maxOutputTokens if not provided', () => {
+      const client = new AnthropicClient('test-api-key');
+      client.setOptions({
+        modelOptions: {
+          model: 'claude-3',
+        },
+      });
+      expect(client.modelOptions.maxOutputTokens).toBeUndefined();
+    });
+
+    it('should not set legacy maxOutputTokens for Claude-3 models', () => {
+      const client = new AnthropicClient('test-api-key');
+      client.setOptions({
+        modelOptions: {
+          model: 'claude-3-opus-20240229',
+          maxOutputTokens: anthropicSettings.legacy.maxOutputTokens.default,
+        },
+      });
+      expect(client.modelOptions.maxOutputTokens).toBe(
+        anthropicSettings.legacy.maxOutputTokens.default,
+      );
     });
   });
 
@@ -136,4 +173,57 @@ describe('AnthropicClient', () => {
       expect(prompt).toContain('You are Claude-2');
     });
   });
+
+  describe('getClient', () => {
+    it('should set legacy maxOutputTokens for non-Claude-3 models', () => {
+      const client = new AnthropicClient('test-api-key');
+      client.setOptions({
+        modelOptions: {
+          model: 'claude-2',
+          maxOutputTokens: anthropicSettings.legacy.maxOutputTokens.default,
+        },
+      });
+      expect(client.modelOptions.maxOutputTokens).toBe(
+        anthropicSettings.legacy.maxOutputTokens.default,
+      );
+    });
+
+    it('should not set legacy maxOutputTokens for Claude-3 models', () => {
+      const client = new AnthropicClient('test-api-key');
+      client.setOptions({
+        modelOptions: {
+          model: 'claude-3-opus-20240229',
+          maxOutputTokens: anthropicSettings.legacy.maxOutputTokens.default,
+        },
+      });
+      expect(client.modelOptions.maxOutputTokens).toBe(
+        anthropicSettings.legacy.maxOutputTokens.default,
+      );
+    });
+
+    it('should add beta header for claude-3-5-sonnet model', () => {
+      const client = new AnthropicClient('test-api-key');
+      const modelOptions = {
+        model: 'claude-3-5-sonnet-20240307',
+      };
+      client.setOptions({ modelOptions });
+      const anthropicClient = client.getClient(modelOptions);
+      expect(anthropicClient._options.defaultHeaders).toBeDefined();
+      expect(anthropicClient._options.defaultHeaders).toHaveProperty('anthropic-beta');
+      expect(anthropicClient._options.defaultHeaders['anthropic-beta']).toBe(
+        'max-tokens-3-5-sonnet-2024-07-15',
+      );
+    });
+
+    it('should not add beta header for other models', () => {
+      const client = new AnthropicClient('test-api-key');
+      client.setOptions({
+        modelOptions: {
+          model: 'claude-2',
+        },
+      });
+      const anthropicClient = client.getClient();
+      expect(anthropicClient.defaultHeaders).not.toHaveProperty('anthropic-beta');
+    });
+  });
 });

api/app/clients/specs/BaseClient.test.js

@@ -1,6 +1,7 @@
+const { Constants } = require('librechat-data-provider');
 const { initializeFakeClient } = require('./FakeClient');
 
-jest.mock('../../../lib/db/connectDb');
+jest.mock('~/lib/db/connectDb');
 jest.mock('~/models', () => ({
   User: jest.fn(),
   Key: jest.fn(),
@@ -307,7 +308,7 @@ describe('BaseClient', () => {
     const unorderedMessages = [
       { id: '3', parentMessageId: '2', text: 'Message 3' },
       { id: '2', parentMessageId: '1', text: 'Message 2' },
-      { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+      { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
     ];
 
     it('should return ordered messages based on parentMessageId', () => {
@@ -316,7 +317,7 @@ describe('BaseClient', () => {
         parentMessageId: '3',
       });
       expect(result).toEqual([
-        { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+        { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
         { id: '2', parentMessageId: '1', text: 'Message 2' },
         { id: '3', parentMessageId: '2', text: 'Message 3' },
       ]);
@@ -575,7 +576,11 @@ describe('BaseClient', () => {
       const onStart = jest.fn();
       const opts = { onStart };
       await TestClient.sendMessage('Hello, world!', opts);
-      expect(onStart).toHaveBeenCalledWith(expect.objectContaining({ text: 'Hello, world!' }));
+
+      expect(onStart).toHaveBeenCalledWith(
+        expect.objectContaining({ text: 'Hello, world!' }),
+        expect.any(String),
+      );
     });
 
     test('saveMessageToDatabase is called with the correct arguments', async () => {
@@ -626,5 +631,32 @@ describe('BaseClient', () => {
         }),
       );
     });
+
+    test('userMessagePromise is awaited before saving response message', async () => {
+      // Mock the saveMessageToDatabase method
+      TestClient.saveMessageToDatabase = jest.fn().mockImplementation(() => {
+        return new Promise((resolve) => setTimeout(resolve, 100)); // Simulate a delay
+      });
+
+      // Send a message
+      const messagePromise = TestClient.sendMessage('Hello, world!');
+
+      // Wait a short time to ensure the user message save has started
+      await new Promise((resolve) => setTimeout(resolve, 50));
+
+      // Check that saveMessageToDatabase has been called once (for the user message)
+      expect(TestClient.saveMessageToDatabase).toHaveBeenCalledTimes(1);
+
+      // Wait for the message to be fully processed
+      await messagePromise;
+
+      // Check that saveMessageToDatabase has been called twice (once for user message, once for response)
+      expect(TestClient.saveMessageToDatabase).toHaveBeenCalledTimes(2);
+
+      // Check the order of calls
+      const calls = TestClient.saveMessageToDatabase.mock.calls;
+      expect(calls[0][0].isCreatedByUser).toBe(true); // First call should be for user message
+      expect(calls[1][0].isCreatedByUser).toBe(false); // Second call should be for response message
+    });
   });
 });

api/app/clients/specs/FakeClient.js

@@ -40,7 +40,8 @@ class FakeClient extends BaseClient {
       };
     }
 
-    this.maxContextTokens = getModelMaxTokens(this.modelOptions.model) ?? 4097;
+    this.maxContextTokens =
+      this.options.maxContextTokens ?? getModelMaxTokens(this.modelOptions.model) ?? 4097;
   }
   buildMessages() {}
   getTokenCount(str) {

api/app/clients/specs/OpenAIClient.test.js

@@ -144,6 +144,7 @@ describe('OpenAIClient', () => {
 
   const defaultOptions = {
     // debug: true,
+    req: {},
     openaiApiKey: 'new-api-key',
     modelOptions: {
       model,
@@ -157,12 +158,19 @@ describe('OpenAIClient', () => {
     azureOpenAIApiVersion: '2020-07-01-preview',
   };
 
+  let originalWarn;
+
   beforeAll(() => {
-    jest.spyOn(console, 'warn').mockImplementation(() => {});
+    originalWarn = console.warn;
+    console.warn = jest.fn();
   });
 
   afterAll(() => {
-    console.warn.mockRestore();
+    console.warn = originalWarn;
+  });
+
+  beforeEach(() => {
+    console.warn.mockClear();
   });
 
   beforeEach(() => {
@@ -662,4 +670,35 @@ describe('OpenAIClient', () => {
       expect(constructorArgs.baseURL).toBe(expectedURL);
     });
   });
+
+  describe('checkVisionRequest functionality', () => {
+    let client;
+    const attachments = [{ type: 'image/png' }];
+
+    beforeEach(() => {
+      client = new OpenAIClient('test-api-key', {
+        endpoint: 'ollama',
+        modelOptions: {
+          model: 'initial-model',
+        },
+        modelsConfig: {
+          ollama: ['initial-model', 'llava', 'other-model'],
+        },
+      });
+
+      client.defaultVisionModel = 'non-valid-default-model';
+    });
+
+    afterEach(() => {
+      jest.restoreAllMocks();
+    });
+
+    it('should set "llava" as the model if it is the first valid model when default validation fails', () => {
+      client.checkVisionRequest(attachments);
+
+      expect(client.modelOptions.model).toBe('llava');
+      expect(client.isVisionModel).toBeTruthy();
+      expect(client.modelOptions.stop).toBeUndefined();
+    });
+  });
 });

api/app/clients/specs/PluginsClient.test.js

@@ -1,9 +1,10 @@
+const crypto = require('crypto');
+const { Constants } = require('librechat-data-provider');
 const { HumanChatMessage, AIChatMessage } = require('langchain/schema');
 const PluginsClient = require('../PluginsClient');
-const crypto = require('crypto');
 
-jest.mock('../../../lib/db/connectDb');
-jest.mock('../../../models/Conversation', () => {
+jest.mock('~/lib/db/connectDb');
+jest.mock('~/models/Conversation', () => {
   return function () {
     return {
       save: jest.fn(),
@@ -12,6 +13,12 @@ jest.mock('../../../models/Conversation', () => {
   };
 });
 
+const defaultAzureOptions = {
+  azureOpenAIApiInstanceName: 'your-instance-name',
+  azureOpenAIApiDeploymentName: 'your-deployment-name',
+  azureOpenAIApiVersion: '2020-07-01-preview',
+};
+
 describe('PluginsClient', () => {
   let TestAgent;
   let options = {
@@ -60,7 +67,7 @@ describe('PluginsClient', () => {
         TestAgent.setOptions(opts);
       }
       const conversationId = opts.conversationId || crypto.randomUUID();
-      const parentMessageId = opts.parentMessageId || '00000000-0000-0000-0000-000000000000';
+      const parentMessageId = opts.parentMessageId || Constants.NO_PARENT;
       const userMessageId = opts.overrideParentMessageId || crypto.randomUUID();
       this.pastMessages = await TestAgent.loadHistory(
         conversationId,
@@ -187,4 +194,121 @@ describe('PluginsClient', () => {
       expect(client.getFunctionModelName('')).toBe('gpt-3.5-turbo');
     });
   });
+
+  describe('Azure OpenAI tests specific to Plugins', () => {
+    // TODO: add more tests for Azure OpenAI integration with Plugins
+    // let client;
+    // beforeEach(() => {
+    //   client = new PluginsClient('dummy_api_key');
+    // });
+
+    test('should not call getFunctionModelName when azure options are set', () => {
+      const spy = jest.spyOn(PluginsClient.prototype, 'getFunctionModelName');
+      const model = 'gpt-4-turbo';
+
+      // note, without the azure change in PR #1766, `getFunctionModelName` is called twice
+      const testClient = new PluginsClient('dummy_api_key', {
+        agentOptions: {
+          model,
+          agent: 'functions',
+        },
+        azure: defaultAzureOptions,
+      });
+
+      expect(spy).not.toHaveBeenCalled();
+      expect(testClient.agentOptions.model).toBe(model);
+
+      spy.mockRestore();
+    });
+  });
+
+  describe('sendMessage with filtered tools', () => {
+    let TestAgent;
+    const apiKey = 'fake-api-key';
+    const mockTools = [{ name: 'tool1' }, { name: 'tool2' }, { name: 'tool3' }, { name: 'tool4' }];
+
+    beforeEach(() => {
+      TestAgent = new PluginsClient(apiKey, {
+        tools: mockTools,
+        modelOptions: {
+          model: 'gpt-3.5-turbo',
+          temperature: 0,
+          max_tokens: 2,
+        },
+        agentOptions: {
+          model: 'gpt-3.5-turbo',
+        },
+      });
+
+      TestAgent.options.req = {
+        app: {
+          locals: {},
+        },
+      };
+
+      TestAgent.sendMessage = jest.fn().mockImplementation(async () => {
+        const { filteredTools = [], includedTools = [] } = TestAgent.options.req.app.locals;
+
+        if (includedTools.length > 0) {
+          const tools = TestAgent.options.tools.filter((plugin) =>
+            includedTools.includes(plugin.name),
+          );
+          TestAgent.options.tools = tools;
+        } else {
+          const tools = TestAgent.options.tools.filter(
+            (plugin) => !filteredTools.includes(plugin.name),
+          );
+          TestAgent.options.tools = tools;
+        }
+
+        return {
+          text: 'Mocked response',
+          tools: TestAgent.options.tools,
+        };
+      });
+    });
+
+    test('should filter out tools when filteredTools is provided', async () => {
+      TestAgent.options.req.app.locals.filteredTools = ['tool1', 'tool3'];
+      const response = await TestAgent.sendMessage('Test message');
+      expect(response.tools).toHaveLength(2);
+      expect(response.tools).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ name: 'tool2' }),
+          expect.objectContaining({ name: 'tool4' }),
+        ]),
+      );
+    });
+
+    test('should only include specified tools when includedTools is provided', async () => {
+      TestAgent.options.req.app.locals.includedTools = ['tool2', 'tool4'];
+      const response = await TestAgent.sendMessage('Test message');
+      expect(response.tools).toHaveLength(2);
+      expect(response.tools).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ name: 'tool2' }),
+          expect.objectContaining({ name: 'tool4' }),
+        ]),
+      );
+    });
+
+    test('should prioritize includedTools over filteredTools', async () => {
+      TestAgent.options.req.app.locals.filteredTools = ['tool1', 'tool3'];
+      TestAgent.options.req.app.locals.includedTools = ['tool1', 'tool2'];
+      const response = await TestAgent.sendMessage('Test message');
+      expect(response.tools).toHaveLength(2);
+      expect(response.tools).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ name: 'tool1' }),
+          expect.objectContaining({ name: 'tool2' }),
+        ]),
+      );
+    });
+
+    test('should not modify tools when no filters are provided', async () => {
+      const response = await TestAgent.sendMessage('Test message');
+      expect(response.tools).toHaveLength(4);
+      expect(response.tools).toEqual(expect.arrayContaining(mockTools));
+    });
+  });
 });

api/app/clients/tools/AzureAiSearch.js

@@ -16,6 +16,9 @@ class AzureAISearch extends StructuredTool {
 
   constructor(fields = {}) {
     super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
 
     // Initialize properties using helper function
     this.serviceEndpoint = this._initializeField(
@@ -68,15 +71,6 @@ class AzureAISearch extends StructuredTool {
     });
   }
 
-  // Simplified getter methods
-  get name() {
-    return 'azure-ai-search';
-  }
-
-  get description() {
-    return 'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
-  }
-
   // Improved error handling and logging
   async _call(data) {
     const { query } = data;

api/app/clients/tools/DALL-E.js

@@ -3,42 +3,39 @@ const OpenAI = require('openai');
 const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
+const { FileContext } = require('librechat-data-provider');
 const { getImageBasename } = require('~/server/services/Files/images');
-const { processFileURL } = require('~/server/services/Files/process');
 const extractBaseURL = require('~/utils/extractBaseURL');
 const { logger } = require('~/config');
 
-const {
-  DALLE2_SYSTEM_PROMPT,
-  DALLE_REVERSE_PROXY,
-  PROXY,
-  DALLE2_AZURE_API_VERSION,
-  DALLE2_BASEURL,
-  DALLE2_API_KEY,
-  DALLE_API_KEY,
-} = process.env;
 class OpenAICreateImage extends Tool {
   constructor(fields = {}) {
     super();
 
     this.userId = fields.userId;
     this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
     let apiKey = fields.DALLE2_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
 
     const config = { apiKey };
-    if (DALLE_REVERSE_PROXY) {
-      config.baseURL = extractBaseURL(DALLE_REVERSE_PROXY);
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
     }
 
-    if (DALLE2_AZURE_API_VERSION && DALLE2_BASEURL) {
-      config.baseURL = DALLE2_BASEURL;
-      config.defaultQuery = { 'api-version': DALLE2_AZURE_API_VERSION };
-      config.defaultHeaders = { 'api-key': DALLE2_API_KEY, 'Content-Type': 'application/json' };
-      config.apiKey = DALLE2_API_KEY;
+    if (process.env.DALLE2_AZURE_API_VERSION && process.env.DALLE2_BASEURL) {
+      config.baseURL = process.env.DALLE2_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE2_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE2_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE2_API_KEY;
     }
 
-    if (PROXY) {
-      config.httpAgent = new HttpsProxyAgent(PROXY);
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
     }
 
     this.openai = new OpenAI(config);
@@ -51,7 +48,7 @@ Guidelines:
 "Subject: [subject], Style: [style], Color: [color], Details: [details], Emotion: [emotion]"
 - Generate images only once per human query unless explicitly requested by the user`;
     this.description_for_model =
-      DALLE2_SYSTEM_PROMPT ??
+      process.env.DALLE2_SYSTEM_PROMPT ??
       `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
 // 1. Prompts must be in English. Translate to English if needed.
 // 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
@@ -67,7 +64,7 @@ Guidelines:
   }
 
   getApiKey() {
-    const apiKey = DALLE2_API_KEY ?? DALLE_API_KEY ?? '';
+    const apiKey = process.env.DALLE2_API_KEY ?? process.env.DALLE_API_KEY ?? '';
     if (!apiKey) {
       throw new Error('Missing DALLE_API_KEY environment variable.');
     }
@@ -86,13 +83,21 @@ Guidelines:
   }
 
   async _call(input) {
-    const resp = await this.openai.images.generate({
-      prompt: this.replaceUnwantedChars(input),
-      // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
-      n: 1,
-      // size: '1024x1024'
-      size: '512x512',
-    });
+    let resp;
+
+    try {
+      resp = await this.openai.images.generate({
+        prompt: this.replaceUnwantedChars(input),
+        // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
+        n: 1,
+        // size: '1024x1024'
+        size: '512x512',
+      });
+    } catch (error) {
+      logger.error('[DALL-E] Problem generating the image:', error);
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
+Error Message: ${error.message}`;
+    }
 
     const theImageUrl = resp.data[0].url;
 
@@ -116,15 +121,16 @@ Guidelines:
     });
 
     try {
-      const result = await processFileURL({
+      const result = await this.processFileURL({
         fileStrategy: this.fileStrategy,
         userId: this.userId,
         URL: theImageUrl,
         fileName: imageName,
         basePath: 'images',
+        context: FileContext.image_generation,
       });
 
-      this.result = this.wrapInMarkdown(result);
+      this.result = this.wrapInMarkdown(result.filepath);
     } catch (error) {
       logger.error('Error while saving the image:', error);
       this.result = `Failed to save the image locally. ${error.message}`;

api/app/clients/tools/GoogleSearch.js

@@ -1,121 +0,0 @@
-const { google } = require('googleapis');
-const { Tool } = require('langchain/tools');
-const { logger } = require('~/config');
-
-/**
- * Represents a tool that allows an agent to use the Google Custom Search API.
- * @extends Tool
- */
-class GoogleSearchAPI extends Tool {
-  constructor(fields = {}) {
-    super();
-    this.cx = fields.GOOGLE_CSE_ID || this.getCx();
-    this.apiKey = fields.GOOGLE_API_KEY || this.getApiKey();
-    this.customSearch = undefined;
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'google';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-  description_for_model =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-
-  getCx() {
-    const cx = process.env.GOOGLE_CSE_ID || '';
-    if (!cx) {
-      throw new Error('Missing GOOGLE_CSE_ID environment variable.');
-    }
-    return cx;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.GOOGLE_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing GOOGLE_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getCustomSearch() {
-    if (!this.customSearch) {
-      const version = 'v1';
-      this.customSearch = google.customsearch(version);
-    }
-    return this.customSearch;
-  }
-
-  resultsToReadableFormat(results) {
-    let output = 'Results:\n';
-
-    results.forEach((resultObj, index) => {
-      output += `Title: ${resultObj.title}\n`;
-      output += `Link: ${resultObj.link}\n`;
-      if (resultObj.snippet) {
-        output += `Snippet: ${resultObj.snippet}\n`;
-      }
-
-      if (index < results.length - 1) {
-        output += '\n';
-      }
-    });
-
-    return output;
-  }
-
-  /**
-   * Calls the tool with the provided input and returns a promise that resolves with a response from the Google Custom Search API.
-   * @param {string} input - The input to provide to the API.
-   * @returns {Promise<String>} A promise that resolves with a response from the Google Custom Search API.
-   */
-  async _call(input) {
-    try {
-      const metadataResults = [];
-      const response = await this.getCustomSearch().cse.list({
-        q: input,
-        cx: this.cx,
-        auth: this.apiKey,
-        num: 5, // Limit the number of results to 5
-      });
-
-      // return response.data;
-      // logger.debug(response.data);
-
-      if (!response.data.items || response.data.items.length === 0) {
-        return this.resultsToReadableFormat([
-          { title: 'No good Google Search Result was found', link: '' },
-        ]);
-      }
-
-      // const results = response.items.slice(0, numResults);
-      const results = response.data.items;
-
-      for (const result of results) {
-        const metadataResult = {
-          title: result.title || '',
-          link: result.link || '',
-        };
-        if (result.snippet) {
-          metadataResult.snippet = result.snippet;
-        }
-        metadataResults.push(metadataResult);
-      }
-
-      return this.resultsToReadableFormat(metadataResults);
-    } catch (error) {
-      logger.error('[GoogleSearchAPI]', error);
-      // throw error;
-      return 'There was an error searching Google.';
-    }
-  }
-}
-
-module.exports = GoogleSearchAPI;

api/app/clients/tools/index.js

@@ -1,35 +1,44 @@
-const GoogleSearchAPI = require('./GoogleSearch');
-const OpenAICreateImage = require('./DALL-E');
-const DALLE3 = require('./structured/DALLE3');
-const StructuredSD = require('./structured/StableDiffusion');
-const StableDiffusionAPI = require('./StableDiffusion');
+const availableTools = require('./manifest.json');
+// Basic Tools
+const CodeBrew = require('./CodeBrew');
 const WolframAlphaAPI = require('./Wolfram');
-const StructuredWolfram = require('./structured/Wolfram');
-const SelfReflectionTool = require('./SelfReflection');
 const AzureAiSearch = require('./AzureAiSearch');
-const StructuredACS = require('./structured/AzureAISearch');
+const OpenAICreateImage = require('./DALL-E');
+const StableDiffusionAPI = require('./StableDiffusion');
+const SelfReflectionTool = require('./SelfReflection');
+
+// Structured Tools
+const DALLE3 = require('./structured/DALLE3');
 const ChatTool = require('./structured/ChatTool');
 const E2BTools = require('./structured/E2BTools');
 const CodeSherpa = require('./structured/CodeSherpa');
+const StructuredSD = require('./structured/StableDiffusion');
+const StructuredACS = require('./structured/AzureAISearch');
 const CodeSherpaTools = require('./structured/CodeSherpaTools');
-const availableTools = require('./manifest.json');
-const CodeBrew = require('./CodeBrew');
+const GoogleSearchAPI = require('./structured/GoogleSearch');
+const StructuredWolfram = require('./structured/Wolfram');
+const TavilySearchResults = require('./structured/TavilySearchResults');
+const TraversaalSearch = require('./structured/TraversaalSearch');
 
 module.exports = {
   availableTools,
-  GoogleSearchAPI,
-  OpenAICreateImage,
-  DALLE3,
-  StableDiffusionAPI,
-  StructuredSD,
-  WolframAlphaAPI,
-  StructuredWolfram,
-  SelfReflectionTool,
-  AzureAiSearch,
-  StructuredACS,
-  E2BTools,
-  ChatTool,
-  CodeSherpa,
-  CodeSherpaTools,
+  // Basic Tools
   CodeBrew,
+  AzureAiSearch,
+  GoogleSearchAPI,
+  WolframAlphaAPI,
+  OpenAICreateImage,
+  StableDiffusionAPI,
+  SelfReflectionTool,
+  // Structured Tools
+  DALLE3,
+  ChatTool,
+  E2BTools,
+  CodeSherpa,
+  StructuredSD,
+  StructuredACS,
+  CodeSherpaTools,
+  StructuredWolfram,
+  TavilySearchResults,
+  TraversaalSearch,
 };

api/app/clients/tools/manifest.json

@@ -1,4 +1,17 @@
 [
+  {
+    "name": "Traversaal",
+    "pluginKey": "traversaal_search",
+    "description": "Traversaal is a robust search API tailored for LLM Agents. Get an API key here: https://api.traversaal.ai",
+    "icon": "https://traversaal.ai/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TRAVERSAAL_API_KEY",
+        "label": "Traversaal API Key",
+        "description": "Get your API key here: <a href=\"https://api.traversaal.ai\" target=\"_blank\">https://api.traversaal.ai</a>"
+      }
+    ]
+  },
   {
     "name": "Google",
     "pluginKey": "google",
@@ -11,7 +24,7 @@
         "description": "This is your Google Custom Search Engine ID. For instructions on how to obtain this, see <a href='https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md'>Our Docs</a>."
       },
       {
-        "authField": "GOOGLE_API_KEY",
+        "authField": "GOOGLE_SEARCH_API_KEY",
         "label": "Google API Key",
         "description": "This is your Google Custom Search API Key. For instructions on how to obtain this, see <a href='https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md'>Our Docs</a>."
       }
@@ -47,7 +60,7 @@
     "name": "CodeSherpa",
     "pluginKey": "codesherpa_tools",
     "description": "[Experimental] A REPL for your chat. Requires https://github.com/iamgreggarcia/codesherpa",
-    "icon": "https://github.com/iamgreggarcia/codesherpa/blob/main/localserver/_logo.png",
+    "icon": "https://raw.githubusercontent.com/iamgreggarcia/codesherpa/main/localserver/_logo.png",
     "authConfig": [
       {
         "authField": "CODESHERPA_SERVER_URL",
@@ -89,7 +102,7 @@
     "icon": "https://i.imgur.com/u2TzXzH.png",
     "authConfig": [
       {
-        "authField": "DALLE2_API_KEY",
+        "authField": "DALLE2_API_KEY||DALLE_API_KEY",
         "label": "OpenAI API Key",
         "description": "You can use DALL-E with your API Key from OpenAI."
       }
@@ -102,12 +115,25 @@
     "icon": "https://i.imgur.com/u2TzXzH.png",
     "authConfig": [
       {
-        "authField": "DALLE3_API_KEY",
+        "authField": "DALLE3_API_KEY||DALLE_API_KEY",
         "label": "OpenAI API Key",
         "description": "You can use DALL-E with your API Key from OpenAI."
       }
     ]
   },
+  {
+    "name": "Tavily Search",
+    "pluginKey": "tavily_search_results_json",
+    "description": "Tavily Search is a robust search API tailored for LLM Agents. It seamlessly integrates with diverse data sources to ensure a superior, relevant search experience.",
+    "icon": "https://tavily.com/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TAVILY_API_KEY",
+        "label": "Tavily API Key",
+        "description": "Get your API key here: https://app.tavily.com/"
+      }
+    ]
+  },
   {
     "name": "Calculator",
     "pluginKey": "calculator",

api/app/clients/tools/structured/AzureAISearch.js

@@ -16,6 +16,16 @@ class AzureAISearch extends StructuredTool {
 
   constructor(fields = {}) {
     super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
+    // Define schema
+    this.schema = z.object({
+      query: z.string().describe('Search word or phrase to Azure AI Search'),
+    });
 
     // Initialize properties using helper function
     this.serviceEndpoint = this._initializeField(
@@ -48,12 +58,16 @@ class AzureAISearch extends StructuredTool {
     );
 
     // Check for required fields
-    if (!this.serviceEndpoint || !this.indexName || !this.apiKey) {
+    if (!this.override && (!this.serviceEndpoint || !this.indexName || !this.apiKey)) {
       throw new Error(
         'Missing AZURE_AI_SEARCH_SERVICE_ENDPOINT, AZURE_AI_SEARCH_INDEX_NAME, or AZURE_AI_SEARCH_API_KEY environment variable.',
       );
     }
 
+    if (this.override) {
+      return;
+    }
+
     // Create SearchClient
     this.client = new SearchClient(
       this.serviceEndpoint,
@@ -61,20 +75,6 @@ class AzureAISearch extends StructuredTool {
       new AzureKeyCredential(this.apiKey),
       { apiVersion: this.apiVersion },
     );
-
-    // Define schema
-    this.schema = z.object({
-      query: z.string().describe('Search word or phrase to Azure AI Search'),
-    });
-  }
-
-  // Simplified getter methods
-  get name() {
-    return 'azure-ai-search';
-  }
-
-  get description() {
-    return 'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
   }
 
   // Improved error handling and logging

api/app/clients/tools/structured/DALLE3.js

@@ -4,42 +4,47 @@ const OpenAI = require('openai');
 const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
+const { FileContext } = require('librechat-data-provider');
 const { getImageBasename } = require('~/server/services/Files/images');
-const { processFileURL } = require('~/server/services/Files/process');
 const extractBaseURL = require('~/utils/extractBaseURL');
 const { logger } = require('~/config');
 
-const {
-  DALLE3_SYSTEM_PROMPT,
-  DALLE_REVERSE_PROXY,
-  PROXY,
-  DALLE3_AZURE_API_VERSION,
-  DALLE3_BASEURL,
-  DALLE3_API_KEY,
-} = process.env;
 class DALLE3 extends Tool {
   constructor(fields = {}) {
     super();
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
 
     this.userId = fields.userId;
     this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      /** @type {processFileURL} Necessary for output to contain all image metadata. */
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
+
     let apiKey = fields.DALLE3_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
     const config = { apiKey };
-    if (DALLE_REVERSE_PROXY) {
-      config.baseURL = extractBaseURL(DALLE_REVERSE_PROXY);
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
     }
 
-    if (DALLE3_AZURE_API_VERSION && DALLE3_BASEURL) {
-      config.baseURL = DALLE3_BASEURL;
-      config.defaultQuery = { 'api-version': DALLE3_AZURE_API_VERSION };
-      config.defaultHeaders = { 'api-key': DALLE3_API_KEY, 'Content-Type': 'application/json' };
-      config.apiKey = DALLE3_API_KEY;
+    if (process.env.DALLE3_AZURE_API_VERSION && process.env.DALLE3_BASEURL) {
+      config.baseURL = process.env.DALLE3_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE3_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE3_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE3_API_KEY;
     }
 
-    if (PROXY) {
-      config.httpAgent = new HttpsProxyAgent(PROXY);
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
     }
 
+    /** @type {OpenAI} */
     this.openai = new OpenAI(config);
     this.name = 'dalle';
     this.description = `Use DALLE to create images from text descriptions.
@@ -47,7 +52,7 @@ class DALLE3 extends Tool {
     - Create only one image, without repeating or listing descriptions outside the "prompts" field.
     - Maintains the original intent of the description, with parameters for image style, quality, and size to tailor the output.`;
     this.description_for_model =
-      DALLE3_SYSTEM_PROMPT ??
+      process.env.DALLE3_SYSTEM_PROMPT ??
       `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
     // 1. Prompts must be in English. Translate to English if needed.
     // 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
@@ -86,7 +91,7 @@ class DALLE3 extends Tool {
 
   getApiKey() {
     const apiKey = process.env.DALLE3_API_KEY ?? process.env.DALLE_API_KEY ?? '';
-    if (!apiKey) {
+    if (!apiKey && !this.override) {
       throw new Error('Missing DALLE_API_KEY environment variable.');
     }
     return apiKey;
@@ -120,6 +125,7 @@ class DALLE3 extends Tool {
         n: 1,
       });
     } catch (error) {
+      logger.error('[DALL-E-3] Problem generating the image:', error);
       return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
 Error Message: ${error.message}`;
     }
@@ -150,15 +156,20 @@ Error Message: ${error.message}`;
     });
 
     try {
-      const result = await processFileURL({
+      const result = await this.processFileURL({
         fileStrategy: this.fileStrategy,
         userId: this.userId,
         URL: theImageUrl,
         fileName: imageName,
         basePath: 'images',
+        context: FileContext.image_generation,
       });
 
-      this.result = this.wrapInMarkdown(result);
+      if (this.returnMetadata) {
+        this.result = result;
+      } else {
+        this.result = this.wrapInMarkdown(result.filepath);
+      }
     } catch (error) {
       logger.error('Error while saving the image:', error);
       this.result = `Failed to save the image locally. ${error.message}`;

api/app/clients/tools/structured/GoogleSearch.js

@@ -0,0 +1,65 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class GoogleSearchResults extends Tool {
+  static lc_name() {
+    return 'GoogleSearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVarApiKey = 'GOOGLE_SEARCH_API_KEY';
+    this.envVarSearchEngineId = 'GOOGLE_CSE_ID';
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? getEnvironmentVariable(this.envVarApiKey);
+    this.searchEngineId =
+      fields.searchEngineId ?? getEnvironmentVariable(this.envVarSearchEngineId);
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'google';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 10.'),
+      // Note: Google API has its own parameters for search customization, adjust as needed.
+    });
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, max_results = 5 } = validationResult.data;
+
+    const response = await fetch(
+      `https://www.googleapis.com/customsearch/v1?key=${this.apiKey}&cx=${
+        this.searchEngineId
+      }&q=${encodeURIComponent(query)}&num=${max_results}`,
+      {
+        method: 'GET',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error.message}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = GoogleSearchResults;

api/app/clients/tools/structured/StableDiffusion.js

@@ -4,12 +4,28 @@ const { z } = require('zod');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { v4: uuidv4 } = require('uuid');
 const { StructuredTool } = require('langchain/tools');
+const { FileContext } = require('librechat-data-provider');
+const paths = require('~/config/paths');
 const { logger } = require('~/config');
 
 class StableDiffusionAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /** @type {string} User ID */
+    this.userId = fields.userId;
+    /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
+    this.req = fields.req;
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
+    if (fields.uploadImageBuffer) {
+      /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */
+      this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);
+    }
+
     this.name = 'stable-diffusion';
     this.url = fields.SD_WEBUI_URL || this.getServerURL();
     this.description_for_model = `// Generate images and visuals using text.
@@ -44,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getMarkdownImageUrl(imageName) {
     const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
+      .join(this.relativePath, this.userId, imageName)
       .replace(/\\/g, '/')
       .replace('public/', '');
     return `![generated image](/${imageUrl})`;
@@ -52,7 +68,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getServerURL() {
     const url = process.env.SD_WEBUI_URL || '';
-    if (!url) {
+    if (!url && !this.override) {
       throw new Error('Missing SD_WEBUI_URL environment variable.');
     }
     return url;
@@ -64,52 +80,78 @@ class StableDiffusionAPI extends StructuredTool {
     const payload = {
       prompt,
       negative_prompt,
-      sampler_index: 'DPM++ 2M Karras',
       cfg_scale: 4.5,
       steps: 22,
       width: 1024,
       height: 1024,
     };
-    const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
-    const image = response.data.images[0];
-    const pngPayload = { image: `data:image/png;base64,${image}` };
-    const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);
-    const info = response2.data.info;
+    let generationResponse;
+    try {
+      generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while generating image:', error);
+      return 'Error making API request.';
+    }
+    const image = generationResponse.data.images[0];
 
-    // Generate unique name
-    const imageName = `${Date.now()}.png`;
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+    /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */
+    let info = {};
+    try {
+      info = JSON.parse(generationResponse.data.info);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while getting image metadata:', error);
+    }
 
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
+    const file_id = uuidv4();
+    const imageName = `${file_id}.png`;
+    const { imageOutput: imageOutputPath, clientPath } = paths;
+    const filepath = path.join(imageOutputPath, this.userId, imageName);
+    this.relativePath = path.relative(clientPath, imageOutputPath);
+
+    if (!fs.existsSync(path.join(imageOutputPath, this.userId))) {
+      fs.mkdirSync(path.join(imageOutputPath, this.userId), { recursive: true });
     }
 
     try {
       const buffer = Buffer.from(image.split(',', 1)[0], 'base64');
+      if (this.returnMetadata && this.uploadImageBuffer && this.req) {
+        const file = await this.uploadImageBuffer({
+          req: this.req,
+          context: FileContext.image_generation,
+          resize: false,
+          metadata: {
+            buffer,
+            height: info.height,
+            width: info.width,
+            bytes: Buffer.byteLength(buffer),
+            filename: imageName,
+            type: 'image/png',
+            file_id,
+          },
+        });
+
+        const generationInfo = info.infotexts[0].split('\n').pop();
+        return {
+          ...file,
+          prompt,
+          metadata: {
+            negative_prompt,
+            seed: info.seed,
+            info: generationInfo,
+          },
+        };
+      }
+
       await sharp(buffer)
         .withMetadata({
           iptcpng: {
-            parameters: info,
+            parameters: info.infotexts[0],
           },
         })
-        .toFile(this.outputPath + '/' + imageName);
+        .toFile(filepath);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
       logger.error('[StableDiffusion] Error while saving the image:', error);
-      // this.result = theImageUrl;
     }
 
     return this.result;

api/app/clients/tools/structured/TavilySearchResults.js

@@ -0,0 +1,92 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class TavilySearchResults extends Tool {
+  static lc_name() {
+    return 'TavilySearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVar = 'TAVILY_API_KEY';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? this.getApiKey();
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'tavily_search_results_json';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 5.'),
+      search_depth: z
+        .enum(['basic', 'advanced'])
+        .optional()
+        .describe(
+          'The depth of the search, affecting result quality and response time (`basic` or `advanced`). Default is basic for quick results and advanced for indepth high quality results but longer response time. Advanced calls equals 2 requests.',
+        ),
+      include_images: z
+        .boolean()
+        .optional()
+        .describe(
+          'Whether to include a list of query-related images in the response. Default is False.',
+        ),
+      include_answer: z
+        .boolean()
+        .optional()
+        .describe('Whether to include answers in the search results. Default is False.'),
+      // include_raw_content: z.boolean().optional().describe('Whether to include raw content in the search results. Default is False.'),
+      // include_domains: z.array(z.string()).optional().describe('A list of domains to specifically include in the search results.'),
+      // exclude_domains: z.array(z.string()).optional().describe('A list of domains to specifically exclude from the search results.'),
+    });
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable(this.envVar);
+    if (!apiKey && !this.override) {
+      throw new Error(`Missing ${this.envVar} environment variable.`);
+    }
+    return apiKey;
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, ...rest } = validationResult.data;
+
+    const requestBody = {
+      api_key: this.apiKey,
+      query,
+      ...rest,
+      ...this.kwargs,
+    };
+
+    const response = await fetch('https://api.tavily.com/search', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(requestBody),
+    });
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = TavilySearchResults;

api/app/clients/tools/structured/TraversaalSearch.js

@@ -0,0 +1,89 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+const { logger } = require('~/config');
+
+/**
+ * Tool for the Traversaal AI search API, Ares.
+ */
+class TraversaalSearch extends Tool {
+  static lc_name() {
+    return 'TraversaalSearch';
+  }
+  constructor(fields) {
+    super(fields);
+    this.name = 'traversaal_search';
+    this.description = `An AI search engine optimized for comprehensive, accurate, and trusted results.
+    Useful for when you need to answer questions about current events. Input should be a search query.`;
+    this.description_for_model =
+      '\'Please create a specific sentence for the AI to understand and use as a query to search the web based on the user\'s request. For example, "Find information about the highest mountains in the world." or "Show me the latest news articles about climate change and its impact on polar ice caps."\'';
+    this.schema = z.object({
+      query: z
+        .string()
+        .describe(
+          'A properly written sentence to be interpreted by an AI to search the web according to the user\'s request.',
+        ),
+    });
+
+    this.apiKey = fields?.TRAVERSAAL_API_KEY ?? this.getApiKey();
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable('TRAVERSAAL_API_KEY');
+    if (!apiKey && this.override) {
+      throw new Error(
+        'No Traversaal API key found. Either set an environment variable named "TRAVERSAAL_API_KEY" or pass an API key as "apiKey".',
+      );
+    }
+    return apiKey;
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  async _call({ query }, _runManager) {
+    const body = {
+      query: [query],
+    };
+    try {
+      const response = await fetch('https://api-ares.traversaal.ai/live/predict', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-api-key': this.apiKey,
+        },
+        body: JSON.stringify({ ...body }),
+      });
+      const json = await response.json();
+      if (!response.ok) {
+        throw new Error(
+          `Request failed with status code ${response.status}: ${json.error ?? json.message}`,
+        );
+      }
+      if (!json.data) {
+        throw new Error('Could not parse Traversaal API results. Please try again.');
+      }
+
+      const baseText = json.data?.response_text ?? '';
+      const sources = json.data?.web_url;
+      const noResponse = 'No response found in Traversaal API results';
+
+      if (!baseText && !sources) {
+        return noResponse;
+      }
+
+      const sourcesText = sources?.length ? '\n\nSources:\n - ' + sources.join('\n - ') : '';
+
+      const result = baseText + sourcesText;
+
+      if (!result) {
+        return noResponse;
+      }
+
+      return result;
+    } catch (error) {
+      logger.error('Traversaal API request failed', error);
+      return `Traversaal API request failed: ${error.message}`;
+    }
+  }
+}
+
+module.exports = TraversaalSearch;

api/app/clients/tools/structured/Wolfram.js

@@ -7,6 +7,9 @@ const { logger } = require('~/config');
 class WolframAlphaAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
     this.name = 'wolfram';
     this.apiKey = fields.WOLFRAM_APP_ID || this.getAppId();
     this.description_for_model = `// Access dynamic computation and curated data from WolframAlpha and Wolfram Cloud.
@@ -55,7 +58,7 @@ class WolframAlphaAPI extends StructuredTool {
 
   getAppId() {
     const appId = process.env.WOLFRAM_APP_ID || '';
-    if (!appId) {
+    if (!appId && !this.override) {
       throw new Error('Missing WOLFRAM_APP_ID environment variable.');
     }
     return appId;

api/app/clients/tools/structured/specs/DALLE3.spec.js

@@ -1,14 +1,11 @@
 const OpenAI = require('openai');
 const DALLE3 = require('../DALLE3');
-const { processFileURL } = require('~/server/services/Files/process');
 
 const { logger } = require('~/config');
 
 jest.mock('openai');
 
-jest.mock('~/server/services/Files/process', () => ({
-  processFileURL: jest.fn(),
-}));
+const processFileURL = jest.fn();
 
 jest.mock('~/server/services/Files/images', () => ({
   getImageBasename: jest.fn().mockImplementation((url) => {
@@ -69,7 +66,7 @@ describe('DALLE3', () => {
     jest.resetModules();
     process.env = { ...originalEnv, DALLE_API_KEY: mockApiKey };
     // Instantiate DALLE3 for tests that do not depend on DALLE3_SYSTEM_PROMPT
-    dalle = new DALLE3();
+    dalle = new DALLE3({ processFileURL });
   });
 
   afterEach(() => {
@@ -78,7 +75,8 @@ describe('DALLE3', () => {
     process.env = originalEnv;
   });
 
-  it('should throw an error if DALLE_API_KEY is missing', () => {
+  it('should throw an error if all potential API keys are missing', () => {
+    delete process.env.DALLE3_API_KEY;
     delete process.env.DALLE_API_KEY;
     expect(() => new DALLE3()).toThrow('Missing DALLE_API_KEY environment variable.');
   });
@@ -112,7 +110,9 @@ describe('DALLE3', () => {
     };
 
     generate.mockResolvedValue(mockResponse);
-    processFileURL.mockResolvedValue('http://example.com/img-test.png');
+    processFileURL.mockResolvedValue({
+      filepath: 'http://example.com/img-test.png',
+    });
 
     const result = await dalle._call(mockData);
 

api/app/clients/tools/util/handleTools.js

@@ -6,19 +6,23 @@ const { OpenAIEmbeddings } = require('langchain/embeddings/openai');
 const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const {
   availableTools,
+  // Basic Tools
+  CodeBrew,
+  AzureAISearch,
   GoogleSearchAPI,
   WolframAlphaAPI,
-  StructuredWolfram,
   OpenAICreateImage,
   StableDiffusionAPI,
+  // Structured Tools
   DALLE3,
-  StructuredSD,
-  AzureAISearch,
-  StructuredACS,
   E2BTools,
   CodeSherpa,
+  StructuredSD,
+  StructuredACS,
   CodeSherpaTools,
-  CodeBrew,
+  TraversaalSearch,
+  StructuredWolfram,
+  TavilySearchResults,
 } = require('../');
 const { loadToolSuite } = require('./loadToolSuite');
 const { loadSpecs } = require('./loadSpecs');
@@ -30,6 +34,14 @@ const getOpenAIKey = async (options, user) => {
   return openAIApiKey || (await getUserPluginAuthValue(user, 'OPENAI_API_KEY'));
 };
 
+/**
+ * Validates the availability and authentication of tools for a user based on environment variables or user-specific plugin authentication values.
+ * Tools without required authentication or with valid authentication are considered valid.
+ *
+ * @param {Object} user The user object for whom to validate tool access.
+ * @param {Array<string>} tools An array of tool identifiers to validate. Defaults to an empty array.
+ * @returns {Promise<Array<string>>} A promise that resolves to an array of valid tool identifiers.
+ */
 const validateTools = async (user, tools = []) => {
   try {
     const validToolsSet = new Set(tools);
@@ -37,16 +49,34 @@ const validateTools = async (user, tools = []) => {
       validToolsSet.has(tool.pluginKey),
     );
 
+    /**
+     * Validates the credentials for a given auth field or set of alternate auth fields for a tool.
+     * If valid admin or user authentication is found, the function returns early. Otherwise, it removes the tool from the set of valid tools.
+     *
+     * @param {string} authField The authentication field or fields (separated by "||" for alternates) to validate.
+     * @param {string} toolName The identifier of the tool being validated.
+     */
     const validateCredentials = async (authField, toolName) => {
-      const adminAuth = process.env[authField];
-      if (adminAuth && adminAuth.length > 0) {
-        return;
+      const fields = authField.split('||');
+      for (const field of fields) {
+        const adminAuth = process.env[field];
+        if (adminAuth && adminAuth.length > 0) {
+          return;
+        }
+
+        let userAuth = null;
+        try {
+          userAuth = await getUserPluginAuthValue(user, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !userAuth) {
+            throw err;
+          }
+        }
+        if (userAuth && userAuth.length > 0) {
+          return;
+        }
       }
 
-      const userAuth = await getUserPluginAuthValue(user, authField);
-      if (userAuth && userAuth.length > 0) {
-        return;
-      }
       validToolsSet.delete(toolName);
     };
 
@@ -63,20 +93,55 @@ const validateTools = async (user, tools = []) => {
     return Array.from(validToolsSet.values());
   } catch (err) {
     logger.error('[validateTools] There was a problem validating tools', err);
-    throw new Error(err);
+    throw new Error('There was a problem validating tools');
   }
 };
 
-const loadToolWithAuth = async (userId, authFields, ToolConstructor, options = {}) => {
+/**
+ * Initializes a tool with authentication values for the given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {string} userId The user ID for which the tool is being loaded.
+ * @param {Array<string>} authFields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+ * @param {typeof import('langchain/tools').Tool} ToolConstructor The constructor function for the tool to be initialized.
+ * @param {Object} options Optional parameters to be passed to the tool constructor alongside authentication values.
+ * @returns {Function} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication.
+ */
+const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => {
   return async function () {
     let authValues = {};
 
-    for (const authField of authFields) {
-      let authValue = process.env[authField];
-      if (!authValue) {
-        authValue = await getUserPluginAuthValue(userId, authField);
+    /**
+     * Finds the first non-empty value for the given authentication field, supporting alternate fields.
+     * @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+     * @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found.
+     */
+    const findAuthValue = async (fields) => {
+      for (const field of fields) {
+        let value = process.env[field];
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+        try {
+          value = await getUserPluginAuthValue(userId, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !value) {
+            throw err;
+          }
+        }
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+      }
+      return null;
+    };
+
+    for (let authField of authFields) {
+      const fields = authField.split('||');
+      const result = await findAuthValue(fields);
+      if (result) {
+        authValues[result.authField] = result.authValue;
       }
-      authValues[authField] = authValue;
     }
 
     return new ToolConstructor({ ...options, ...authValues, userId });
@@ -90,8 +155,10 @@ const loadTools = async ({
   returnMap = false,
   tools = [],
   options = {},
+  skipSpecs = false,
 }) => {
   const toolConstructors = {
+    tavily_search_results_json: TavilySearchResults,
     calculator: Calculator,
     google: GoogleSearchAPI,
     wolfram: functions ? StructuredWolfram : WolframAlphaAPI,
@@ -99,6 +166,7 @@ const loadTools = async ({
     'stable-diffusion': functions ? StructuredSD : StableDiffusionAPI,
     'azure-ai-search': functions ? StructuredACS : AzureAISearch,
     CodeBrew: CodeBrew,
+    traversaal_search: TraversaalSearch,
   };
 
   const openAIApiKey = await getOpenAIKey(options, user);
@@ -168,10 +236,19 @@ const loadTools = async ({
     toolConstructors.codesherpa = CodeSherpa;
   }
 
+  const imageGenOptions = {
+    req: options.req,
+    fileStrategy: options.fileStrategy,
+    processFileURL: options.processFileURL,
+    returnMetadata: options.returnMetadata,
+    uploadImageBuffer: options.uploadImageBuffer,
+  };
+
   const toolOptions = {
     serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' },
-    dalle: { fileStrategy: options.fileStrategy },
-    'dall-e': { fileStrategy: options.fileStrategy },
+    dalle: imageGenOptions,
+    'dall-e': imageGenOptions,
+    'stable-diffusion': imageGenOptions,
   };
 
   const toolAuthFields = {};
@@ -194,7 +271,7 @@ const loadTools = async ({
 
     if (toolConstructors[tool]) {
       const options = toolOptions[tool] || {};
-      const toolInstance = await loadToolWithAuth(
+      const toolInstance = loadToolWithAuth(
         user,
         toolAuthFields[tool],
         toolConstructors[tool],
@@ -210,7 +287,7 @@ const loadTools = async ({
   }
 
   let specs = null;
-  if (functions && remainingTools.length > 0) {
+  if (functions && remainingTools.length > 0 && skipSpecs !== true) {
     specs = await loadSpecs({
       llm: model,
       user,
@@ -237,6 +314,9 @@ const loadTools = async ({
   let result = [];
   for (const tool of tools) {
     const validTool = requestedTools[tool];
+    if (!validTool) {
+      continue;
+    }
     const plugin = await validTool();
 
     if (Array.isArray(plugin)) {
@@ -250,6 +330,7 @@ const loadTools = async ({
 };
 
 module.exports = {
+  loadToolWithAuth,
   validateTools,
   loadTools,
 };

api/app/clients/tools/util/handleTools.test.js

@@ -4,26 +4,33 @@ const mockUser = {
   findByIdAndDelete: jest.fn(),
 };
 
-var mockPluginService = {
+const mockPluginService = {
   updateUserPluginAuth: jest.fn(),
   deleteUserPluginAuth: jest.fn(),
   getUserPluginAuthValue: jest.fn(),
 };
 
-jest.mock('../../../../models/User', () => {
+jest.mock('~/models/User', () => {
   return function () {
     return mockUser;
   };
 });
 
-jest.mock('../../../../server/services/PluginService', () => mockPluginService);
+jest.mock('~/server/services/PluginService', () => mockPluginService);
 
-const User = require('../../../../models/User');
-const { validateTools, loadTools } = require('./');
-const PluginService = require('../../../../server/services/PluginService');
-const { BaseChatModel } = require('langchain/chat_models/openai');
 const { Calculator } = require('langchain/tools/calculator');
-const { availableTools, OpenAICreateImage, GoogleSearchAPI, StructuredSD } = require('../');
+const { BaseChatModel } = require('langchain/chat_models/openai');
+
+const User = require('~/models/User');
+const PluginService = require('~/server/services/PluginService');
+const { validateTools, loadTools, loadToolWithAuth } = require('./handleTools');
+const {
+  availableTools,
+  OpenAICreateImage,
+  GoogleSearchAPI,
+  StructuredSD,
+  WolframAlphaAPI,
+} = require('../');
 
 describe('Tool Handlers', () => {
   let fakeUser;
@@ -44,7 +51,10 @@ describe('Tool Handlers', () => {
     });
     mockPluginService.updateUserPluginAuth.mockImplementation(
       (userId, authField, _pluginKey, credential) => {
-        userAuthValues[`${userId}-${authField}`] = credential;
+        const fields = authField.split('||');
+        fields.forEach((field) => {
+          userAuthValues[`${userId}-${field}`] = credential;
+        });
       },
     );
 
@@ -53,6 +63,7 @@ describe('Tool Handlers', () => {
       username: 'fakeuser',
       email: 'fakeuser@example.com',
       emailVerified: false,
+      // file deepcode ignore NoHardcodedPasswords/test: fake value
       password: 'fakepassword123',
       avatar: '',
       provider: 'local',
@@ -133,6 +144,18 @@ describe('Tool Handlers', () => {
       loadTool2 = toolFunctions[sampleTools[1]];
       loadTool3 = toolFunctions[sampleTools[2]];
     });
+
+    let originalEnv;
+
+    beforeEach(() => {
+      originalEnv = process.env;
+      process.env = { ...originalEnv };
+    });
+
+    afterEach(() => {
+      process.env = originalEnv;
+    });
+
     it('returns the expected load functions for requested tools', async () => {
       expect(loadTool1).toBeDefined();
       expect(loadTool2).toBeDefined();
@@ -149,6 +172,86 @@ describe('Tool Handlers', () => {
       expect(authTool).toBeInstanceOf(ToolClass);
       expect(tool).toBeInstanceOf(ToolClass2);
     });
+
+    it('should initialize an authenticated tool with primary auth field', async () => {
+      process.env.DALLE2_API_KEY = 'mocked_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool with alternate auth field when primary is missing', async () => {
+      delete process.env.DALLE2_API_KEY; // Ensure the primary key is not set
+      process.env.DALLE_API_KEY = 'mocked_alternate_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'DALLE2_API_KEY',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when env vars are missing', async () => {
+      mockPluginService.updateUserPluginAuth('userId', 'DALLE_API_KEY', 'dalle', 'mocked_api_key');
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(2);
+    });
+
+    it('should initialize an authenticated tool with singular auth field', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool when env var is set', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when singular env var is missing', async () => {
+      delete process.env.WOLFRAM_APP_ID; // Ensure the environment variable is not set
+      mockPluginService.getUserPluginAuthValue.mockResolvedValue('mocked_user_auth_value');
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
     it('should throw an error for an unauthenticated tool', async () => {
       try {
         await loadTool2();

api/app/clients/tools/util/loadToolSuite.js

@@ -1,17 +1,49 @@
-const { getUserPluginAuthValue } = require('../../../../server/services/PluginService');
+const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const { availableTools } = require('../');
+const { logger } = require('~/config');
 
-const loadToolSuite = async ({ pluginKey, tools, user, options }) => {
+/**
+ * Loads a suite of tools with authentication values for a given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {Object} params Parameters for loading the tool suite.
+ * @param {string} params.pluginKey Key identifying the plugin whose tools are to be loaded.
+ * @param {Array<Function>} params.tools Array of tool constructor functions.
+ * @param {Object} params.user User object for whom the tools are being loaded.
+ * @param {Object} [params.options={}] Optional parameters to be passed to each tool constructor.
+ * @returns {Promise<Array>} A promise that resolves to an array of instantiated tools.
+ */
+const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
   const authConfig = availableTools.find((tool) => tool.pluginKey === pluginKey).authConfig;
   const suite = [];
   const authValues = {};
 
-  for (const auth of authConfig) {
-    let authValue = process.env[auth.authField];
-    if (!authValue) {
-      authValue = await getUserPluginAuthValue(user, auth.authField);
+  const findAuthValue = async (authField) => {
+    const fields = authField.split('||');
+    for (const field of fields) {
+      let value = process.env[field];
+      if (value) {
+        return value;
+      }
+      try {
+        value = await getUserPluginAuthValue(user, field);
+        if (value) {
+          return value;
+        }
+      } catch (err) {
+        logger.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
+      }
+    }
+    return null;
+  };
+
+  for (const auth of authConfig) {
+    const authValue = await findAuthValue(auth.authField);
+    if (authValue !== null) {
+      authValues[auth.authField] = authValue;
+    } else {
+      logger.warn(`[loadToolSuite] No auth value found for ${auth.authField}`);
     }
-    authValues[auth.authField] = authValue;
   }
 
   for (const tool of tools) {

api/cache/banViolation.js

@@ -1,6 +1,7 @@
-const Session = require('~/models/Session');
-const getLogStores = require('./getLogStores');
+const { ViolationTypes } = require('librechat-data-provider');
 const { isEnabled, math, removePorts } = require('~/server/utils');
+const getLogStores = require('./getLogStores');
+const Session = require('~/models/Session');
 const { logger } = require('~/config');
 
 const { BAN_VIOLATIONS, BAN_INTERVAL } = process.env ?? {};
@@ -48,7 +49,7 @@ const banViolation = async (req, res, errorMessage) => {
   await Session.deleteAllUserSessions(user_id);
   res.clearCookie('refreshToken');
 
-  const banLogs = getLogStores('ban');
+  const banLogs = getLogStores(ViolationTypes.BAN);
   const duration = errorMessage.duration || banLogs.opts.ttl;
 
   if (duration <= 0) {

api/cache/banViolation.spec.js

@@ -6,6 +6,7 @@ jest.mock('../models/Session');
 jest.mock('./getLogStores', () => {
   return jest.fn().mockImplementation(() => {
     const EventEmitter = require('events');
+    const { CacheKeys } = require('librechat-data-provider');
     const math = require('../server/utils/math');
     const mockGet = jest.fn();
     const mockSet = jest.fn();
@@ -33,7 +34,7 @@ jest.mock('./getLogStores', () => {
     }
 
     return new KeyvMongo('', {
-      namespace: 'bans',
+      namespace: CacheKeys.BANS,
       ttl: math(process.env.BAN_DURATION, 7200000),
     });
   });

api/cache/getLogStores.js

@@ -1,5 +1,5 @@
 const Keyv = require('keyv');
-const { CacheKeys } = require('librechat-data-provider');
+const { CacheKeys, ViolationTypes, Time } = require('librechat-data-provider');
 const { logFile, violationFile } = require('./keyvFiles');
 const { math, isEnabled } = require('~/server/utils');
 const keyvRedis = require('./keyvRedis');
@@ -23,17 +23,67 @@ const config = isEnabled(USE_REDIS)
   ? new Keyv({ store: keyvRedis })
   : new Keyv({ namespace: CacheKeys.CONFIG_STORE });
 
+const roles = isEnabled(USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.ROLES });
+
+const audioRuns = isEnabled(USE_REDIS)
+  ? new Keyv({ store: keyvRedis, ttl: Time.TEN_MINUTES })
+  : new Keyv({ namespace: CacheKeys.AUDIO_RUNS, ttl: Time.TEN_MINUTES });
+
+const messages = isEnabled(USE_REDIS)
+  ? new Keyv({ store: keyvRedis, ttl: Time.FIVE_MINUTES })
+  : new Keyv({ namespace: CacheKeys.MESSAGES, ttl: Time.FIVE_MINUTES });
+
+const tokenConfig = isEnabled(USE_REDIS) // ttl: 30 minutes
+  ? new Keyv({ store: keyvRedis, ttl: Time.THIRTY_MINUTES })
+  : new Keyv({ namespace: CacheKeys.TOKEN_CONFIG, ttl: Time.THIRTY_MINUTES });
+
+const genTitle = isEnabled(USE_REDIS) // ttl: 2 minutes
+  ? new Keyv({ store: keyvRedis, ttl: Time.TWO_MINUTES })
+  : new Keyv({ namespace: CacheKeys.GEN_TITLE, ttl: Time.TWO_MINUTES });
+
+const modelQueries = isEnabled(process.env.USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.MODEL_QUERIES });
+
+const abortKeys = isEnabled(USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.ABORT_KEYS, ttl: Time.TEN_MINUTES });
+
 const namespaces = {
+  [CacheKeys.ROLES]: roles,
   [CacheKeys.CONFIG_STORE]: config,
   pending_req,
-  ban: new Keyv({ store: keyvMongo, namespace: 'bans', ttl: duration }),
+  [ViolationTypes.BAN]: new Keyv({ store: keyvMongo, namespace: CacheKeys.BANS, ttl: duration }),
+  [CacheKeys.ENCODED_DOMAINS]: new Keyv({
+    store: keyvMongo,
+    namespace: CacheKeys.ENCODED_DOMAINS,
+    ttl: 0,
+  }),
   general: new Keyv({ store: logFile, namespace: 'violations' }),
   concurrent: createViolationInstance('concurrent'),
   non_browser: createViolationInstance('non_browser'),
   message_limit: createViolationInstance('message_limit'),
-  token_balance: createViolationInstance('token_balance'),
+  token_balance: createViolationInstance(ViolationTypes.TOKEN_BALANCE),
   registrations: createViolationInstance('registrations'),
+  [ViolationTypes.TTS_LIMIT]: createViolationInstance(ViolationTypes.TTS_LIMIT),
+  [ViolationTypes.STT_LIMIT]: createViolationInstance(ViolationTypes.STT_LIMIT),
+  [ViolationTypes.FILE_UPLOAD_LIMIT]: createViolationInstance(ViolationTypes.FILE_UPLOAD_LIMIT),
+  [ViolationTypes.VERIFY_EMAIL_LIMIT]: createViolationInstance(ViolationTypes.VERIFY_EMAIL_LIMIT),
+  [ViolationTypes.RESET_PASSWORD_LIMIT]: createViolationInstance(
+    ViolationTypes.RESET_PASSWORD_LIMIT,
+  ),
+  [ViolationTypes.ILLEGAL_MODEL_REQUEST]: createViolationInstance(
+    ViolationTypes.ILLEGAL_MODEL_REQUEST,
+  ),
   logins: createViolationInstance('logins'),
+  [CacheKeys.ABORT_KEYS]: abortKeys,
+  [CacheKeys.TOKEN_CONFIG]: tokenConfig,
+  [CacheKeys.GEN_TITLE]: genTitle,
+  [CacheKeys.MODEL_QUERIES]: modelQueries,
+  [CacheKeys.AUDIO_RUNS]: audioRuns,
+  [CacheKeys.MESSAGES]: messages,
 };
 
 /**

api/cache/logViolation.js

@@ -1,6 +1,6 @@
+const { isEnabled } = require('~/server/utils');
 const getLogStores = require('./getLogStores');
 const banViolation = require('./banViolation');
-const { isEnabled } = require('../server/utils');
 
 /**
  * Logs the violation.

api/config/parsers.js

@@ -27,22 +27,25 @@ function getMatchingSensitivePatterns(valueStr) {
 }
 
 /**
- * Redacts sensitive information from a console message.
- *
+ * Redacts sensitive information from a console message and trims it to a specified length if provided.
  * @param {string} str - The console message to be redacted.
- * @returns {string} - The redacted console message.
+ * @param {number} [trimLength] - The optional length at which to trim the redacted message.
+ * @returns {string} - The redacted and optionally trimmed console message.
  */
-function redactMessage(str) {
-  const patterns = getMatchingSensitivePatterns(str);
-
-  if (patterns.length === 0) {
-    return str;
+function redactMessage(str, trimLength) {
+  if (!str) {
+    return '';
   }
 
+  const patterns = getMatchingSensitivePatterns(str);
   patterns.forEach((pattern) => {
     str = str.replace(pattern, '$1[REDACTED]');
   });
 
+  if (trimLength !== undefined && str.length > trimLength) {
+    return `${str.substring(0, trimLength)}...`;
+  }
+
   return str;
 }
 
@@ -106,6 +109,14 @@ const condenseArray = (item) => {
  * @returns {string} - The formatted log message.
  */
 const debugTraverse = winston.format.printf(({ level, message, timestamp, ...metadata }) => {
+  if (!message) {
+    return `${timestamp} ${level}`;
+  }
+
+  if (!message?.trim || typeof message !== 'string') {
+    return `${timestamp} ${level}: ${JSON.stringify(message)}`;
+  }
+
   let msg = `${timestamp} ${level}: ${truncateLongStrings(message?.trim(), 150)}`;
   try {
     if (level !== 'debug') {

api/config/paths.js

@@ -1,7 +1,14 @@
 const path = require('path');
 
 module.exports = {
+  root: path.resolve(__dirname, '..', '..'),
+  uploads: path.resolve(__dirname, '..', '..', 'uploads'),
+  clientPath: path.resolve(__dirname, '..', '..', 'client'),
   dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),
   publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),
+  fonts: path.resolve(__dirname, '..', '..', 'client', 'public', 'fonts'),
+  assets: path.resolve(__dirname, '..', '..', 'client', 'public', 'assets'),
   imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),
+  structuredTools: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'structured'),
+  pluginManifest: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'manifest.json'),
 };

api/config/winston.js

@@ -5,7 +5,15 @@ const { redactFormat, redactMessage, debugTraverse } = require('./parsers');
 
 const logDir = path.join(__dirname, '..', 'logs');
 
-const { NODE_ENV, DEBUG_LOGGING = true, DEBUG_CONSOLE = false } = process.env;
+const { NODE_ENV, DEBUG_LOGGING = true, DEBUG_CONSOLE = false, CONSOLE_JSON = false } = process.env;
+
+const useConsoleJson =
+  (typeof CONSOLE_JSON === 'string' && CONSOLE_JSON?.toLowerCase() === 'true') ||
+  CONSOLE_JSON === true;
+
+const useDebugConsole =
+  (typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE?.toLowerCase() === 'true') ||
+  DEBUG_CONSOLE === true;
 
 const levels = {
   error: 0,
@@ -33,7 +41,7 @@ const level = () => {
 
 const fileFormat = winston.format.combine(
   redactFormat(),
-  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.timestamp({ format: () => new Date().toISOString() }),
   winston.format.errors({ stack: true }),
   winston.format.splat(),
   // redactErrors(),
@@ -99,14 +107,20 @@ const consoleFormat = winston.format.combine(
   }),
 );
 
-if (
-  (typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE?.toLowerCase() === 'true') ||
-  DEBUG_CONSOLE === true
-) {
+if (useDebugConsole) {
   transports.push(
     new winston.transports.Console({
       level: 'debug',
-      format: winston.format.combine(consoleFormat, debugTraverse),
+      format: useConsoleJson
+        ? winston.format.combine(fileFormat, debugTraverse, winston.format.json())
+        : winston.format.combine(fileFormat, debugTraverse),
+    }),
+  );
+} else if (useConsoleJson) {
+  transports.push(
+    new winston.transports.Console({
+      level: 'info',
+      format: winston.format.combine(fileFormat, winston.format.json()),
     }),
   );
 } else {

api/lib/db/indexSync.js

@@ -1,11 +1,28 @@
 const { MeiliSearch } = require('meilisearch');
-const Message = require('~/models/schema/messageSchema');
 const Conversation = require('~/models/schema/convoSchema');
+const Message = require('~/models/schema/messageSchema');
 const { logger } = require('~/config');
 
 const searchEnabled = process.env?.SEARCH?.toLowerCase() === 'true';
 let currentTimeout = null;
 
+class MeiliSearchClient {
+  static instance = null;
+
+  static getInstance() {
+    if (!MeiliSearchClient.instance) {
+      if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY) {
+        throw new Error('Meilisearch configuration is missing.');
+      }
+      MeiliSearchClient.instance = new MeiliSearch({
+        host: process.env.MEILI_HOST,
+        apiKey: process.env.MEILI_MASTER_KEY,
+      });
+    }
+    return MeiliSearchClient.instance;
+  }
+}
+
 // eslint-disable-next-line no-unused-vars
 async function indexSync(req, res, next) {
   if (!searchEnabled) {
@@ -13,20 +30,10 @@ async function indexSync(req, res, next) {
   }
 
   try {
-    if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY || !searchEnabled) {
-      throw new Error('Meilisearch not configured, search will be disabled.');
-    }
-
-    const client = new MeiliSearch({
-      host: process.env.MEILI_HOST,
-      apiKey: process.env.MEILI_MASTER_KEY,
-    });
+    const client = MeiliSearchClient.getInstance();
 
     const { status } = await client.health();
-    // logger.debug(`[indexSync] Meilisearch: ${status}`);
-    const result = status === 'available' && !!process.env.SEARCH;
-
-    if (!result) {
+    if (status !== 'available' || !process.env.SEARCH) {
       throw new Error('Meilisearch not available');
     }
 
@@ -37,12 +44,8 @@ async function indexSync(req, res, next) {
     const messagesIndexed = messages.numberOfDocuments;
     const convosIndexed = convos.numberOfDocuments;
 
-    logger.debug(
-      `[indexSync] There are ${messageCount} messages in the database, ${messagesIndexed} indexed`,
-    );
-    logger.debug(
-      `[indexSync] There are ${convoCount} convos in the database, ${convosIndexed} indexed`,
-    );
+    logger.debug(`[indexSync] There are ${messageCount} messages and ${messagesIndexed} indexed`);
+    logger.debug(`[indexSync] There are ${convoCount} convos and ${convosIndexed} indexed`);
 
     if (messageCount !== messagesIndexed) {
       logger.debug('[indexSync] Messages out of sync, indexing');
@@ -54,7 +57,6 @@ async function indexSync(req, res, next) {
       Conversation.syncWithMeili();
     }
   } catch (err) {
-    // logger.debug('[indexSync] in index sync');
     if (err.message.includes('not found')) {
       logger.debug('[indexSync] Creating indices...');
       currentTimeout = setTimeout(async () => {

api/models/Action.js

@@ -0,0 +1,85 @@
+const mongoose = require('mongoose');
+const actionSchema = require('./schema/action');
+
+const Action = mongoose.model('action', actionSchema);
+
+/**
+ * Update an action with new data without overwriting existing properties,
+ * or create a new action if it doesn't exist, within a transaction session if provided.
+ *
+ * @param {Object} searchParams - The search parameters to find the action to update.
+ * @param {string} searchParams.action_id - The ID of the action to update.
+ * @param {string} searchParams.user - The user ID of the action's author.
+ * @param {Object} updateData - An object containing the properties to update.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use.
+ * @returns {Promise<Object>} The updated or newly created action document as a plain object.
+ */
+const updateAction = async (searchParams, updateData, session = null) => {
+  const options = { new: true, upsert: true, session };
+  return await Action.findOneAndUpdate(searchParams, updateData, options).lean();
+};
+
+/**
+ * Retrieves all actions that match the given search parameters.
+ *
+ * @param {Object} searchParams - The search parameters to find matching actions.
+ * @param {boolean} includeSensitive - Flag to include sensitive data in the metadata.
+ * @returns {Promise<Array<Object>>} A promise that resolves to an array of action documents as plain objects.
+ */
+const getActions = async (searchParams, includeSensitive = false) => {
+  const actions = await Action.find(searchParams).lean();
+
+  if (!includeSensitive) {
+    for (let i = 0; i < actions.length; i++) {
+      const metadata = actions[i].metadata;
+      if (!metadata) {
+        continue;
+      }
+
+      const sensitiveFields = ['api_key', 'oauth_client_id', 'oauth_client_secret'];
+      for (let field of sensitiveFields) {
+        if (metadata[field]) {
+          delete metadata[field];
+        }
+      }
+    }
+  }
+
+  return actions;
+};
+
+/**
+ * Deletes an action by params, within a transaction session if provided.
+ *
+ * @param {Object} searchParams - The search parameters to find the action to delete.
+ * @param {string} searchParams.action_id - The ID of the action to delete.
+ * @param {string} searchParams.user - The user ID of the action's author.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use (optional).
+ * @returns {Promise<Object>} A promise that resolves to the deleted action document as a plain object, or null if no document was found.
+ */
+const deleteAction = async (searchParams, session = null) => {
+  const options = session ? { session } : {};
+  return await Action.findOneAndDelete(searchParams, options).lean();
+};
+
+/**
+ * Deletes actions by params, within a transaction session if provided.
+ *
+ * @param {Object} searchParams - The search parameters to find the actions to delete.
+ * @param {string} searchParams.action_id - The ID of the action(s) to delete.
+ * @param {string} searchParams.user - The user ID of the action's author.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use (optional).
+ * @returns {Promise<Number>} A promise that resolves to the number of deleted action documents.
+ */
+const deleteActions = async (searchParams, session = null) => {
+  const options = session ? { session } : {};
+  const result = await Action.deleteMany(searchParams, options);
+  return result.deletedCount;
+};
+
+module.exports = {
+  getActions,
+  updateAction,
+  deleteAction,
+  deleteActions,
+};

api/models/Assistant.js

@@ -0,0 +1,59 @@
+const mongoose = require('mongoose');
+const assistantSchema = require('./schema/assistant');
+
+const Assistant = mongoose.model('assistant', assistantSchema);
+
+/**
+ * Update an assistant with new data without overwriting existing properties,
+ * or create a new assistant if it doesn't exist, within a transaction session if provided.
+ *
+ * @param {Object} searchParams - The search parameters to find the assistant to update.
+ * @param {string} searchParams.assistant_id - The ID of the assistant to update.
+ * @param {string} searchParams.user - The user ID of the assistant's author.
+ * @param {Object} updateData - An object containing the properties to update.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use (optional).
+ * @returns {Promise<Object>} The updated or newly created assistant document as a plain object.
+ */
+const updateAssistantDoc = async (searchParams, updateData, session = null) => {
+  const options = { new: true, upsert: true, session };
+  return await Assistant.findOneAndUpdate(searchParams, updateData, options).lean();
+};
+
+/**
+ * Retrieves an assistant document based on the provided ID.
+ *
+ * @param {Object} searchParams - The search parameters to find the assistant to update.
+ * @param {string} searchParams.assistant_id - The ID of the assistant to update.
+ * @param {string} searchParams.user - The user ID of the assistant's author.
+ * @returns {Promise<Object|null>} The assistant document as a plain object, or null if not found.
+ */
+const getAssistant = async (searchParams) => await Assistant.findOne(searchParams).lean();
+
+/**
+ * Retrieves all assistants that match the given search parameters.
+ *
+ * @param {Object} searchParams - The search parameters to find matching assistants.
+ * @returns {Promise<Array<Object>>} A promise that resolves to an array of action documents as plain objects.
+ */
+const getAssistants = async (searchParams) => {
+  return await Assistant.find(searchParams).lean();
+};
+
+/**
+ * Deletes an assistant based on the provided ID.
+ *
+ * @param {Object} searchParams - The search parameters to find the assistant to delete.
+ * @param {string} searchParams.assistant_id - The ID of the assistant to delete.
+ * @param {string} searchParams.user - The user ID of the assistant's author.
+ * @returns {Promise<void>} Resolves when the assistant has been successfully deleted.
+ */
+const deleteAssistant = async (searchParams) => {
+  return await Assistant.findOneAndDelete(searchParams);
+};
+
+module.exports = {
+  updateAssistantDoc,
+  deleteAssistant,
+  getAssistants,
+  getAssistant,
+};

api/models/Balance.js

@@ -10,8 +10,9 @@ balanceSchema.statics.check = async function ({
   valueKey,
   tokenType,
   amount,
+  endpointTokenConfig,
 }) {
-  const multiplier = getMultiplier({ valueKey, tokenType, model, endpoint });
+  const multiplier = getMultiplier({ valueKey, tokenType, model, endpoint, endpointTokenConfig });
   const tokenCost = amount * multiplier;
   const { tokenCredits: balance } = (await this.findOne({ user }, 'tokenCredits').lean()) ?? {};
 
@@ -24,6 +25,7 @@ balanceSchema.statics.check = async function ({
     amount,
     balance,
     multiplier,
+    endpointTokenConfig: !!endpointTokenConfig,
   });
 
   if (!balance) {

api/models/Categories.js

@@ -0,0 +1,61 @@
+const { logger } = require('~/config');
+// const { Categories } = require('./schema/categories');
+const options = [
+  {
+    label: '',
+    value: '',
+  },
+  {
+    label: 'idea',
+    value: 'idea',
+  },
+  {
+    label: 'travel',
+    value: 'travel',
+  },
+  {
+    label: 'teach_or_explain',
+    value: 'teach_or_explain',
+  },
+  {
+    label: 'write',
+    value: 'write',
+  },
+  {
+    label: 'shop',
+    value: 'shop',
+  },
+  {
+    label: 'code',
+    value: 'code',
+  },
+  {
+    label: 'misc',
+    value: 'misc',
+  },
+  {
+    label: 'roleplay',
+    value: 'roleplay',
+  },
+  {
+    label: 'finance',
+    value: 'finance',
+  },
+];
+
+module.exports = {
+  /**
+   * Retrieves the categories asynchronously.
+   * @returns {Promise<TGetCategoriesResponse>} An array of category objects.
+   * @throws {Error} If there is an error retrieving the categories.
+   */
+  getCategories: async () => {
+    try {
+      // const categories = await Categories.find();
+      return options;
+    } catch (error) {
+      logger.error('Error getting categories', error);
+      return [];
+    }
+  },
+};

api/models/Conversation.js

@@ -2,6 +2,12 @@ const Conversation = require('./schema/convoSchema');
 const { getMessages, deleteMessages } = require('./Message');
 const logger = require('~/config/winston');
 
+/**
+ * Retrieves a single conversation for a given user and conversation ID.
+ * @param {string} user - The user's ID.
+ * @param {string} conversationId - The conversation's ID.
+ * @returns {Promise<TConversation>} The conversation object.
+ */
 const getConvo = async (user, conversationId) => {
   try {
     return await Conversation.findOne({ user, conversationId }).lean();
@@ -13,29 +19,75 @@ const getConvo = async (user, conversationId) => {
 
 module.exports = {
   Conversation,
-  saveConvo: async (user, { conversationId, newConversationId, ...convo }) => {
+  /**
+   * Saves a conversation to the database.
+   * @param {Object} req - The request object.
+   * @param {string} conversationId - The conversation's ID.
+   * @param {Object} metadata - Additional metadata to log for operation.
+   * @returns {Promise<TConversation>} The conversation object.
+   */
+  saveConvo: async (req, { conversationId, newConversationId, ...convo }, metadata) => {
     try {
-      const messages = await getMessages({ conversationId });
-      const update = { ...convo, messages, user };
+      if (metadata && metadata?.context) {
+        logger.debug(`[saveConvo] ${metadata.context}`);
+      }
+      const messages = await getMessages({ conversationId }, '_id');
+      const update = { ...convo, messages, user: req.user.id };
       if (newConversationId) {
         update.conversationId = newConversationId;
       }
 
-      return await Conversation.findOneAndUpdate({ conversationId: conversationId, user }, update, {
-        new: true,
-        upsert: true,
-      });
+      const conversation = await Conversation.findOneAndUpdate(
+        { conversationId, user: req.user.id },
+        update,
+        {
+          new: true,
+          upsert: true,
+        },
+      );
+
+      return conversation.toObject();
     } catch (error) {
       logger.error('[saveConvo] Error saving conversation', error);
+      if (metadata && metadata?.context) {
+        logger.info(`[saveConvo] ${metadata.context}`);
+      }
       return { message: 'Error saving conversation' };
     }
   },
-  getConvosByPage: async (user, pageNumber = 1, pageSize = 14) => {
+  bulkSaveConvos: async (conversations) => {
     try {
-      const totalConvos = (await Conversation.countDocuments({ user })) || 1;
+      const bulkOps = conversations.map((convo) => ({
+        updateOne: {
+          filter: { conversationId: convo.conversationId, user: convo.user },
+          update: convo,
+          upsert: true,
+          timestamps: false,
+        },
+      }));
+
+      const result = await Conversation.bulkWrite(bulkOps);
+      return result;
+    } catch (error) {
+      logger.error('[saveBulkConversations] Error saving conversations in bulk', error);
+      throw new Error('Failed to save conversations in bulk.');
+    }
+  },
+  getConvosByPage: async (user, pageNumber = 1, pageSize = 25, isArchived = false, tags) => {
+    const query = { user };
+    if (isArchived) {
+      query.isArchived = true;
+    } else {
+      query.$or = [{ isArchived: false }, { isArchived: { $exists: false } }];
+    }
+    if (Array.isArray(tags) && tags.length > 0) {
+      query.tags = { $in: tags };
+    }
+    try {
+      const totalConvos = (await Conversation.countDocuments(query)) || 1;
       const totalPages = Math.ceil(totalConvos / pageSize);
-      const convos = await Conversation.find({ user })
-        .sort({ createdAt: -1 })
+      const convos = await Conversation.find(query)
+        .sort({ updatedAt: -1 })
         .skip((pageNumber - 1) * pageSize)
         .limit(pageSize)
         .lean();
@@ -45,7 +97,7 @@ module.exports = {
       return { message: 'Error getting conversations' };
     }
   },
-  getConvosQueried: async (user, convoIds, pageNumber = 1, pageSize = 14) => {
+  getConvosQueried: async (user, convoIds, pageNumber = 1, pageSize = 25) => {
     try {
       if (!convoIds || convoIds.length === 0) {
         return { conversations: [], pages: 1, pageNumber, pageSize };

api/models/ConversationTag.js

@@ -0,0 +1,268 @@
+//const crypto = require('crypto');
+
+const logger = require('~/config/winston');
+const Conversation = require('./schema/convoSchema');
+const ConversationTag = require('./schema/conversationTagSchema');
+
+const SAVED_TAG = 'Saved';
+
+const updateTagsForConversation = async (user, conversationId, tags) => {
+  try {
+    const conversation = await Conversation.findOne({ user, conversationId });
+    if (!conversation) {
+      return { message: 'Conversation not found' };
+    }
+
+    const addedTags = tags.tags.filter((tag) => !conversation.tags.includes(tag));
+    const removedTags = conversation.tags.filter((tag) => !tags.tags.includes(tag));
+    for (const tag of addedTags) {
+      await ConversationTag.updateOne({ tag, user }, { $inc: { count: 1 } }, { upsert: true });
+    }
+    for (const tag of removedTags) {
+      await ConversationTag.updateOne({ tag, user }, { $inc: { count: -1 } });
+    }
+    conversation.tags = tags.tags;
+    await conversation.save({ timestamps: { updatedAt: false } });
+    return conversation.tags;
+  } catch (error) {
+    logger.error('[updateTagsToConversation] Error updating tags', error);
+    return { message: 'Error updating tags' };
+  }
+};
+
+const createConversationTag = async (user, data) => {
+  try {
+    const cTag = await ConversationTag.findOne({ user, tag: data.tag });
+    if (cTag) {
+      return cTag;
+    }
+
+    const addToConversation = data.addToConversation && data.conversationId;
+    const newTag = await ConversationTag.create({
+      user,
+      tag: data.tag,
+      count: 0,
+      description: data.description,
+      position: 1,
+    });
+
+    await ConversationTag.updateMany(
+      { user, position: { $gte: 1 }, _id: { $ne: newTag._id } },
+      { $inc: { position: 1 } },
+    );
+
+    if (addToConversation) {
+      const conversation = await Conversation.findOne({
+        user,
+        conversationId: data.conversationId,
+      });
+      if (conversation) {
+        const tags = [...(conversation.tags || []), data.tag];
+        await updateTagsForConversation(user, data.conversationId, { tags });
+      } else {
+        logger.warn('[updateTagsForConversation] Conversation not found', data.conversationId);
+      }
+    }
+
+    return await ConversationTag.findOne({ user, tag: data.tag });
+  } catch (error) {
+    logger.error('[createConversationTag] Error updating conversation tag', error);
+    return { message: 'Error updating conversation tag' };
+  }
+};
+
+const replaceOrRemoveTagInConversations = async (user, oldtag, newtag) => {
+  try {
+    const conversations = await Conversation.find({ user, tags: { $in: [oldtag] } });
+    for (const conversation of conversations) {
+      if (newtag && newtag !== '') {
+        conversation.tags = conversation.tags.map((tag) => (tag === oldtag ? newtag : tag));
+      } else {
+        conversation.tags = conversation.tags.filter((tag) => tag !== oldtag);
+      }
+      await conversation.save({ timestamps: { updatedAt: false } });
+    }
+  } catch (error) {
+    logger.error('[replaceOrRemoveTagInConversations] Error updating conversation tags', error);
+    return { message: 'Error updating conversation tags' };
+  }
+};
+
+const updateTagPosition = async (user, tag, newPosition) => {
+  try {
+    const cTag = await ConversationTag.findOne({ user, tag });
+    if (!cTag) {
+      return { message: 'Tag not found' };
+    }
+
+    const oldPosition = cTag.position;
+
+    if (newPosition === oldPosition) {
+      return cTag;
+    }
+
+    const updateOperations = [];
+
+    if (newPosition > oldPosition) {
+      // Move other tags up
+      updateOperations.push({
+        updateMany: {
+          filter: {
+            user,
+            position: { $gt: oldPosition, $lte: newPosition },
+            tag: { $ne: SAVED_TAG },
+          },
+          update: { $inc: { position: -1 } },
+        },
+      });
+    } else {
+      // Move other tags down
+      updateOperations.push({
+        updateMany: {
+          filter: {
+            user,
+            position: { $gte: newPosition, $lt: oldPosition },
+            tag: { $ne: SAVED_TAG },
+          },
+          update: { $inc: { position: 1 } },
+        },
+      });
+    }
+
+    // Update the target tag's position
+    updateOperations.push({
+      updateOne: {
+        filter: { _id: cTag._id },
+        update: { $set: { position: newPosition } },
+      },
+    });
+
+    await ConversationTag.bulkWrite(updateOperations);
+
+    return await ConversationTag.findById(cTag._id);
+  } catch (error) {
+    logger.error('[updateTagPosition] Error updating tag position', error);
+    return { message: 'Error updating tag position' };
+  }
+};
+module.exports = {
+  SAVED_TAG,
+  ConversationTag,
+  getConversationTags: async (user) => {
+    try {
+      const cTags = await ConversationTag.find({ user }).sort({ position: 1 }).lean();
+      cTags.sort((a, b) => (a.tag === SAVED_TAG ? -1 : b.tag === SAVED_TAG ? 1 : 0));
+
+      return cTags;
+    } catch (error) {
+      logger.error('[getShare] Error getting share link', error);
+      return { message: 'Error getting share link' };
+    }
+  },
+
+  createConversationTag,
+  updateConversationTag: async (user, tag, data) => {
+    try {
+      const cTag = await ConversationTag.findOne({ user, tag });
+      if (!cTag) {
+        return createConversationTag(user, data);
+      }
+
+      if (cTag.tag !== data.tag || cTag.description !== data.description) {
+        cTag.tag = data.tag;
+        cTag.description = data.description === undefined ? cTag.description : data.description;
+        await cTag.save();
+      }
+
+      if (data.position !== undefined && cTag.position !== data.position) {
+        await updateTagPosition(user, tag, data.position);
+      }
+
+      // update conversation tags properties
+      replaceOrRemoveTagInConversations(user, tag, data.tag);
+      return await ConversationTag.findOne({ user, tag: data.tag });
+    } catch (error) {
+      logger.error('[updateConversationTag] Error updating conversation tag', error);
+      return { message: 'Error updating conversation tag' };
+    }
+  },
+
+  deleteConversationTag: async (user, tag) => {
+    try {
+      const currentTag = await ConversationTag.findOne({ user, tag });
+      if (!currentTag) {
+        return;
+      }
+
+      await currentTag.deleteOne({ user, tag });
+
+      await replaceOrRemoveTagInConversations(user, tag, null);
+      return currentTag;
+    } catch (error) {
+      logger.error('[deleteConversationTag] Error deleting conversation tag', error);
+      return { message: 'Error deleting conversation tag' };
+    }
+  },
+
+  updateTagsForConversation,
+  rebuildConversationTags: async (user) => {
+    try {
+      const conversations = await Conversation.find({ user }).select('tags');
+      const tagCountMap = {};
+
+      // Count the occurrences of each tag
+      conversations.forEach((conversation) => {
+        conversation.tags.forEach((tag) => {
+          if (tagCountMap[tag]) {
+            tagCountMap[tag]++;
+          } else {
+            tagCountMap[tag] = 1;
+          }
+        });
+      });
+
+      const tags = await ConversationTag.find({ user }).sort({ position: -1 });
+
+      // Update existing tags and add new tags
+      for (const [tag, count] of Object.entries(tagCountMap)) {
+        const existingTag = tags.find((t) => t.tag === tag);
+        if (existingTag) {
+          existingTag.count = count;
+          await existingTag.save();
+        } else {
+          const newTag = new ConversationTag({ user, tag, count });
+          tags.push(newTag);
+          await newTag.save();
+        }
+      }
+
+      // Set count to 0 for tags that are not in the grouped tags
+      for (const tag of tags) {
+        if (!tagCountMap[tag.tag]) {
+          tag.count = 0;
+          await tag.save();
+        }
+      }
+
+      // Sort tags by position in descending order
+      tags.sort((a, b) => a.position - b.position);
+
+      // Move the tag with name "saved" to the first position
+      const savedTagIndex = tags.findIndex((tag) => tag.tag === SAVED_TAG);
+      if (savedTagIndex !== -1) {
+        const [savedTag] = tags.splice(savedTagIndex, 1);
+        tags.unshift(savedTag);
+      }
+
+      // Reassign positions starting from 0
+      tags.forEach((tag, index) => {
+        tag.position = index;
+        tag.save();
+      });
+      return tags;
+    } catch (error) {
+      logger.error('[rearrangeTags] Error rearranging tags', error);
+      return { message: 'Error rearranging tags' };
+    }
+  },
+};

api/models/File.js

@@ -14,24 +14,32 @@ const findFileById = async (file_id, options = {}) => {
 };
 
 /**
- * Retrieves files matching a given filter.
+ * Retrieves files matching a given filter, sorted by the most recently updated.
  * @param {Object} filter - The filter criteria to apply.
+ * @param {Object} [_sortOptions] - Optional sort parameters.
  * @returns {Promise<Array<MongoFile>>} A promise that resolves to an array of file documents.
  */
-const getFiles = async (filter) => {
-  return await File.find(filter).lean();
+const getFiles = async (filter, _sortOptions) => {
+  const sortOptions = { updatedAt: -1, ..._sortOptions };
+  return await File.find(filter).sort(sortOptions).lean();
 };
 
 /**
  * Creates a new file with a TTL of 1 hour.
  * @param {MongoFile} data - The file data to be created, must contain file_id.
+ * @param {boolean} disableTTL - Whether to disable the TTL.
  * @returns {Promise<MongoFile>} A promise that resolves to the created file document.
  */
-const createFile = async (data) => {
+const createFile = async (data, disableTTL) => {
   const fileData = {
     ...data,
     expiresAt: new Date(Date.now() + 3600 * 1000),
   };
+
+  if (disableTTL) {
+    delete fileData.expiresAt;
+  }
+
   return await File.findOneAndUpdate({ file_id: data.file_id }, fileData, {
     new: true,
     upsert: true,
@@ -61,7 +69,7 @@ const updateFileUsage = async (data) => {
   const { file_id, inc = 1 } = data;
   const updateOperation = {
     $inc: { usage: inc },
-    $unset: { expiresAt: '' },
+    $unset: { expiresAt: '', temp_file_id: '' },
   };
   return await File.findOneAndUpdate({ file_id }, updateOperation, { new: true }).lean();
 };
@@ -75,13 +83,26 @@ const deleteFile = async (file_id) => {
   return await File.findOneAndDelete({ file_id }).lean();
 };
 
+/**
+ * Deletes a file identified by a filter.
+ * @param {object} filter - The filter criteria to apply.
+ * @returns {Promise<MongoFile>} A promise that resolves to the deleted file document or null.
+ */
+const deleteFileByFilter = async (filter) => {
+  return await File.findOneAndDelete(filter).lean();
+};
+
 /**
  * Deletes multiple files identified by an array of file_ids.
  * @param {Array<string>} file_ids - The unique identifiers of the files to delete.
  * @returns {Promise<Object>} A promise that resolves to the result of the deletion operation.
  */
-const deleteFiles = async (file_ids) => {
-  return await File.deleteMany({ file_id: { $in: file_ids } });
+const deleteFiles = async (file_ids, user) => {
+  let deleteQuery = { file_id: { $in: file_ids } };
+  if (user) {
+    deleteQuery = { user: user };
+  }
+  return await File.deleteMany(deleteQuery);
 };
 
 module.exports = {
@@ -93,4 +114,5 @@ module.exports = {
   updateFileUsage,
   deleteFile,
   deleteFiles,
+  deleteFileByFilter,
 };

api/models/Message.js

@@ -1,132 +1,342 @@
 const { z } = require('zod');
 const Message = require('./schema/messageSchema');
-const logger = require('~/config/winston');
+const { logger } = require('~/config');
 
 const idSchema = z.string().uuid();
 
+/**
+ * Saves a message in the database.
+ *
+ * @async
+ * @function saveMessage
+ * @param {Express.Request} req - The request object containing user information.
+ * @param {Object} params - The message data object.
+ * @param {string} params.endpoint - The endpoint where the message originated.
+ * @param {string} params.iconURL - The URL of the sender's icon.
+ * @param {string} params.messageId - The unique identifier for the message.
+ * @param {string} params.newMessageId - The new unique identifier for the message (if applicable).
+ * @param {string} params.conversationId - The identifier of the conversation.
+ * @param {string} [params.parentMessageId] - The identifier of the parent message, if any.
+ * @param {string} params.sender - The identifier of the sender.
+ * @param {string} params.text - The text content of the message.
+ * @param {boolean} params.isCreatedByUser - Indicates if the message was created by the user.
+ * @param {string} [params.error] - Any error associated with the message.
+ * @param {boolean} [params.unfinished] - Indicates if the message is unfinished.
+ * @param {Object[]} [params.files] - An array of files associated with the message.
+ * @param {boolean} [params.isEdited] - Indicates if the message was edited.
+ * @param {string} [params.finish_reason] - Reason for finishing the message.
+ * @param {number} [params.tokenCount] - The number of tokens in the message.
+ * @param {string} [params.plugin] - Plugin associated with the message.
+ * @param {string[]} [params.plugins] - An array of plugins associated with the message.
+ * @param {string} [params.model] - The model used to generate the message.
+ * @param {Object} [metadata] - Additional metadata for this operation
+ * @param {string} [metadata.context] - The context of the operation
+ * @returns {Promise<TMessage>} The updated or newly inserted message document.
+ * @throws {Error} If there is an error in saving the message.
+ */
+async function saveMessage(req, params, metadata) {
+  try {
+    if (!req || !req.user || !req.user.id) {
+      throw new Error('User not authenticated');
+    }
+
+    const {
+      text,
+      error,
+      model,
+      files,
+      plugin,
+      sender,
+      plugins,
+      iconURL,
+      endpoint,
+      isEdited,
+      messageId,
+      unfinished,
+      tokenCount,
+      newMessageId,
+      finish_reason,
+      conversationId,
+      parentMessageId,
+      isCreatedByUser,
+    } = params;
+
+    const validConvoId = idSchema.safeParse(conversationId);
+    if (!validConvoId.success) {
+      logger.warn(`Invalid conversation ID: ${conversationId}`);
+      if (metadata && metadata?.context) {
+        logger.info(`---\`saveMessage\` context: ${metadata.context}`);
+      }
+
+      logger.info(`---Invalid conversation ID Params:
+
+${JSON.stringify(params, null, 2)}
+
+`);
+      return;
+    }
+
+    const update = {
+      user: req.user.id,
+      iconURL,
+      endpoint,
+      messageId: newMessageId || messageId,
+      conversationId,
+      parentMessageId,
+      sender,
+      text,
+      isCreatedByUser,
+      isEdited,
+      finish_reason,
+      error,
+      unfinished,
+      tokenCount,
+      plugin,
+      plugins,
+      model,
+    };
+
+    if (files) {
+      update.files = files;
+    }
+
+    const message = await Message.findOneAndUpdate({ messageId, user: req.user.id }, update, {
+      upsert: true,
+      new: true,
+    });
+
+    return message.toObject();
+  } catch (err) {
+    logger.error('Error saving message:', err);
+    if (metadata && metadata?.context) {
+      logger.info(`---\`saveMessage\` context: ${metadata.context}`);
+    }
+    throw err;
+  }
+}
+
+/**
+ * Saves multiple messages in the database in bulk.
+ *
+ * @async
+ * @function bulkSaveMessages
+ * @param {Object[]} messages - An array of message objects to save.
+ * @returns {Promise<Object>} The result of the bulk write operation.
+ * @throws {Error} If there is an error in saving messages in bulk.
+ */
+async function bulkSaveMessages(messages) {
+  try {
+    const bulkOps = messages.map((message) => ({
+      updateOne: {
+        filter: { messageId: message.messageId },
+        update: message,
+        upsert: true,
+      },
+    }));
+
+    const result = await Message.bulkWrite(bulkOps);
+    return result;
+  } catch (err) {
+    logger.error('Error saving messages in bulk:', err);
+    throw err;
+  }
+}
+
+/**
+ * Records a message in the database.
+ *
+ * @async
+ * @function recordMessage
+ * @param {Object} params - The message data object.
+ * @param {string} params.user - The identifier of the user.
+ * @param {string} params.endpoint - The endpoint where the message originated.
+ * @param {string} params.messageId - The unique identifier for the message.
+ * @param {string} params.conversationId - The identifier of the conversation.
+ * @param {string} [params.parentMessageId] - The identifier of the parent message, if any.
+ * @param {Partial<TMessage>} rest - Any additional properties from the TMessage typedef not explicitly listed.
+ * @returns {Promise<Object>} The updated or newly inserted message document.
+ * @throws {Error} If there is an error in saving the message.
+ */
+async function recordMessage({
+  user,
+  endpoint,
+  messageId,
+  conversationId,
+  parentMessageId,
+  ...rest
+}) {
+  try {
+    // No parsing of convoId as may use threadId
+    const message = {
+      user,
+      endpoint,
+      messageId,
+      conversationId,
+      parentMessageId,
+      ...rest,
+    };
+
+    return await Message.findOneAndUpdate({ user, messageId }, message, {
+      upsert: true,
+      new: true,
+    });
+  } catch (err) {
+    logger.error('Error recording message:', err);
+    throw err;
+  }
+}
+
+/**
+ * Updates the text of a message.
+ *
+ * @async
+ * @function updateMessageText
+ * @param {Object} params - The update data object.
+ * @param {Object} req - The request object.
+ * @param {string} params.messageId - The unique identifier for the message.
+ * @param {string} params.text - The new text content of the message.
+ * @returns {Promise<void>}
+ * @throws {Error} If there is an error in updating the message text.
+ */
+async function updateMessageText(req, { messageId, text }) {
+  try {
+    await Message.updateOne({ messageId, user: req.user.id }, { text });
+  } catch (err) {
+    logger.error('Error updating message text:', err);
+    throw err;
+  }
+}
+
+/**
+ * Updates a message.
+ *
+ * @async
+ * @function updateMessage
+ * @param {Object} message - The message object containing update data.
+ * @param {Object} req - The request object.
+ * @param {string} message.messageId - The unique identifier for the message.
+ * @param {string} [message.text] - The new text content of the message.
+ * @param {Object[]} [message.files] - The files associated with the message.
+ * @param {boolean} [message.isCreatedByUser] - Indicates if the message was created by the user.
+ * @param {string} [message.sender] - The identifier of the sender.
+ * @param {number} [message.tokenCount] - The number of tokens in the message.
+ * @param {Object} [metadata] - The operation metadata
+ * @param {string} [metadata.context] - The operation metadata
+ * @returns {Promise<TMessage>} The updated message document.
+ * @throws {Error} If there is an error in updating the message or if the message is not found.
+ */
+async function updateMessage(req, message, metadata) {
+  try {
+    const { messageId, ...update } = message;
+    update.isEdited = true;
+    const updatedMessage = await Message.findOneAndUpdate(
+      { messageId, user: req.user.id },
+      update,
+      {
+        new: true,
+      },
+    );
+
+    if (!updatedMessage) {
+      throw new Error('Message not found or user not authorized.');
+    }
+
+    return {
+      messageId: updatedMessage.messageId,
+      conversationId: updatedMessage.conversationId,
+      parentMessageId: updatedMessage.parentMessageId,
+      sender: updatedMessage.sender,
+      text: updatedMessage.text,
+      isCreatedByUser: updatedMessage.isCreatedByUser,
+      tokenCount: updatedMessage.tokenCount,
+      isEdited: true,
+    };
+  } catch (err) {
+    logger.error('Error updating message:', err);
+    if (metadata && metadata?.context) {
+      logger.info(`---\`updateMessage\` context: ${metadata.context}`);
+    }
+    throw err;
+  }
+}
+
+/**
+ * Deletes messages in a conversation since a specific message.
+ *
+ * @async
+ * @function deleteMessagesSince
+ * @param {Object} params - The parameters object.
+ * @param {Object} req - The request object.
+ * @param {string} params.messageId - The unique identifier for the message.
+ * @param {string} params.conversationId - The identifier of the conversation.
+ * @returns {Promise<Number>} The number of deleted messages.
+ * @throws {Error} If there is an error in deleting messages.
+ */
+async function deleteMessagesSince(req, { messageId, conversationId }) {
+  try {
+    const message = await Message.findOne({ messageId, user: req.user.id }).lean();
+
+    if (message) {
+      const query = Message.find({ conversationId, user: req.user.id });
+      return await query.deleteMany({
+        createdAt: { $gt: message.createdAt },
+      });
+    }
+    return undefined;
+  } catch (err) {
+    logger.error('Error deleting messages:', err);
+    throw err;
+  }
+}
+
+/**
+ * Retrieves messages from the database.
+ * @async
+ * @function getMessages
+ * @param {Record<string, unknown>} filter - The filter criteria.
+ * @param {string | undefined} [select] - The fields to select.
+ * @returns {Promise<TMessage[]>} The messages that match the filter criteria.
+ * @throws {Error} If there is an error in retrieving messages.
+ */
+async function getMessages(filter, select) {
+  try {
+    if (select) {
+      return await Message.find(filter).select(select).sort({ createdAt: 1 }).lean();
+    }
+
+    return await Message.find(filter).sort({ createdAt: 1 }).lean();
+  } catch (err) {
+    logger.error('Error getting messages:', err);
+    throw err;
+  }
+}
+
+/**
+ * Deletes messages from the database.
+ *
+ * @async
+ * @function deleteMessages
+ * @param {Object} filter - The filter criteria to find messages to delete.
+ * @returns {Promise<Number>} The number of deleted messages.
+ * @throws {Error} If there is an error in deleting messages.
+ */
+async function deleteMessages(filter) {
+  try {
+    return await Message.deleteMany(filter);
+  } catch (err) {
+    logger.error('Error deleting messages:', err);
+    throw err;
+  }
+}
+
 module.exports = {
   Message,
-
-  async saveMessage({
-    user,
-    endpoint,
-    messageId,
-    newMessageId,
-    conversationId,
-    parentMessageId,
-    sender,
-    text,
-    isCreatedByUser,
-    error,
-    unfinished,
-    files,
-    isEdited,
-    finish_reason,
-    tokenCount,
-    plugin,
-    plugins,
-    model,
-  }) {
-    try {
-      const validConvoId = idSchema.safeParse(conversationId);
-      if (!validConvoId.success) {
-        return;
-      }
-
-      const update = {
-        user,
-        endpoint,
-        messageId: newMessageId || messageId,
-        conversationId,
-        parentMessageId,
-        sender,
-        text,
-        isCreatedByUser,
-        isEdited,
-        finish_reason,
-        error,
-        unfinished,
-        tokenCount,
-        plugin,
-        plugins,
-        model,
-      };
-
-      if (files) {
-        update.files = files;
-      }
-      // may also need to update the conversation here
-      await Message.findOneAndUpdate({ messageId }, update, { upsert: true, new: true });
-
-      return {
-        messageId,
-        conversationId,
-        parentMessageId,
-        sender,
-        text,
-        isCreatedByUser,
-        tokenCount,
-      };
-    } catch (err) {
-      logger.error('Error saving message:', err);
-      throw new Error('Failed to save message.');
-    }
-  },
-  async updateMessage(message) {
-    try {
-      const { messageId, ...update } = message;
-      update.isEdited = true;
-      const updatedMessage = await Message.findOneAndUpdate({ messageId }, update, { new: true });
-
-      if (!updatedMessage) {
-        throw new Error('Message not found.');
-      }
-
-      return {
-        messageId: updatedMessage.messageId,
-        conversationId: updatedMessage.conversationId,
-        parentMessageId: updatedMessage.parentMessageId,
-        sender: updatedMessage.sender,
-        text: updatedMessage.text,
-        isCreatedByUser: updatedMessage.isCreatedByUser,
-        tokenCount: updatedMessage.tokenCount,
-        isEdited: true,
-      };
-    } catch (err) {
-      logger.error('Error updating message:', err);
-      throw new Error('Failed to update message.');
-    }
-  },
-  async deleteMessagesSince({ messageId, conversationId }) {
-    try {
-      const message = await Message.findOne({ messageId }).lean();
-
-      if (message) {
-        return await Message.find({ conversationId }).deleteMany({
-          createdAt: { $gt: message.createdAt },
-        });
-      }
-    } catch (err) {
-      logger.error('Error deleting messages:', err);
-      throw new Error('Failed to delete messages.');
-    }
-  },
-
-  async getMessages(filter) {
-    try {
-      return await Message.find(filter).sort({ createdAt: 1 }).lean();
-    } catch (err) {
-      logger.error('Error getting messages:', err);
-      throw new Error('Failed to get messages.');
-    }
-  },
-
-  async deleteMessages(filter) {
-    try {
-      return await Message.deleteMany(filter);
-    } catch (err) {
-      logger.error('Error deleting messages:', err);
-      throw new Error('Failed to delete messages.');
-    }
-  },
+  saveMessage,
+  bulkSaveMessages,
+  recordMessage,
+  updateMessageText,
+  updateMessage,
+  deleteMessagesSince,
+  getMessages,
+  deleteMessages,
 };

api/models/Message.spec.js

@@ -0,0 +1,239 @@
+const mongoose = require('mongoose');
+const { v4: uuidv4 } = require('uuid');
+
+jest.mock('mongoose');
+
+const mockFindQuery = {
+  select: jest.fn().mockReturnThis(),
+  sort: jest.fn().mockReturnThis(),
+  lean: jest.fn().mockReturnThis(),
+  deleteMany: jest.fn().mockResolvedValue({ deletedCount: 1 }),
+};
+
+const mockSchema = {
+  findOneAndUpdate: jest.fn(),
+  updateOne: jest.fn(),
+  findOne: jest.fn(() => ({
+    lean: jest.fn(),
+  })),
+  find: jest.fn(() => mockFindQuery),
+  deleteMany: jest.fn(),
+};
+
+mongoose.model.mockReturnValue(mockSchema);
+
+jest.mock('~/models/schema/messageSchema', () => mockSchema);
+
+jest.mock('~/config/winston', () => ({
+  error: jest.fn(),
+}));
+
+const {
+  saveMessage,
+  getMessages,
+  updateMessage,
+  deleteMessages,
+  updateMessageText,
+  deleteMessagesSince,
+} = require('~/models/Message');
+
+describe('Message Operations', () => {
+  let mockReq;
+  let mockMessage;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    mockReq = {
+      user: { id: 'user123' },
+    };
+
+    mockMessage = {
+      messageId: 'msg123',
+      conversationId: uuidv4(),
+      text: 'Hello, world!',
+      user: 'user123',
+    };
+
+    mockSchema.findOneAndUpdate.mockResolvedValue({
+      toObject: () => mockMessage,
+    });
+  });
+
+  describe('saveMessage', () => {
+    it('should save a message for an authenticated user', async () => {
+      const result = await saveMessage(mockReq, mockMessage);
+      expect(result).toEqual(mockMessage);
+      expect(mockSchema.findOneAndUpdate).toHaveBeenCalledWith(
+        { messageId: 'msg123', user: 'user123' },
+        expect.objectContaining({ user: 'user123' }),
+        expect.any(Object),
+      );
+    });
+
+    it('should throw an error for unauthenticated user', async () => {
+      mockReq.user = null;
+      await expect(saveMessage(mockReq, mockMessage)).rejects.toThrow('User not authenticated');
+    });
+
+    it('should throw an error for invalid conversation ID', async () => {
+      mockMessage.conversationId = 'invalid-id';
+      await expect(saveMessage(mockReq, mockMessage)).resolves.toBeUndefined();
+    });
+  });
+
+  describe('updateMessageText', () => {
+    it('should update message text for the authenticated user', async () => {
+      await updateMessageText(mockReq, { messageId: 'msg123', text: 'Updated text' });
+      expect(mockSchema.updateOne).toHaveBeenCalledWith(
+        { messageId: 'msg123', user: 'user123' },
+        { text: 'Updated text' },
+      );
+    });
+  });
+
+  describe('updateMessage', () => {
+    it('should update a message for the authenticated user', async () => {
+      mockSchema.findOneAndUpdate.mockResolvedValue(mockMessage);
+      const result = await updateMessage(mockReq, { messageId: 'msg123', text: 'Updated text' });
+      expect(result).toEqual(
+        expect.objectContaining({
+          messageId: 'msg123',
+          text: 'Hello, world!',
+          isEdited: true,
+        }),
+      );
+    });
+
+    it('should throw an error if message is not found', async () => {
+      mockSchema.findOneAndUpdate.mockResolvedValue(null);
+      await expect(
+        updateMessage(mockReq, { messageId: 'nonexistent', text: 'Test' }),
+      ).rejects.toThrow('Message not found or user not authorized.');
+    });
+  });
+
+  describe('deleteMessagesSince', () => {
+    it('should delete messages only for the authenticated user', async () => {
+      mockSchema.findOne().lean.mockResolvedValueOnce({ createdAt: new Date() });
+      mockFindQuery.deleteMany.mockResolvedValueOnce({ deletedCount: 1 });
+      const result = await deleteMessagesSince(mockReq, {
+        messageId: 'msg123',
+        conversationId: 'convo123',
+      });
+      expect(mockSchema.findOne).toHaveBeenCalledWith({ messageId: 'msg123', user: 'user123' });
+      expect(mockSchema.find).not.toHaveBeenCalled();
+      expect(result).toBeUndefined();
+    });
+
+    it('should return undefined if no message is found', async () => {
+      mockSchema.findOne().lean.mockResolvedValueOnce(null);
+      const result = await deleteMessagesSince(mockReq, {
+        messageId: 'nonexistent',
+        conversationId: 'convo123',
+      });
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('getMessages', () => {
+    it('should retrieve messages with the correct filter', async () => {
+      const filter = { conversationId: 'convo123' };
+      await getMessages(filter);
+      expect(mockSchema.find).toHaveBeenCalledWith(filter);
+      expect(mockFindQuery.sort).toHaveBeenCalledWith({ createdAt: 1 });
+      expect(mockFindQuery.lean).toHaveBeenCalled();
+    });
+  });
+
+  describe('deleteMessages', () => {
+    it('should delete messages with the correct filter', async () => {
+      await deleteMessages({ user: 'user123' });
+      expect(mockSchema.deleteMany).toHaveBeenCalledWith({ user: 'user123' });
+    });
+  });
+
+  describe('Conversation Hijacking Prevention', () => {
+    it('should not allow editing a message in another user\'s conversation', async () => {
+      const attackerReq = { user: { id: 'attacker123' } };
+      const victimConversationId = 'victim-convo-123';
+      const victimMessageId = 'victim-msg-123';
+
+      mockSchema.findOneAndUpdate.mockResolvedValue(null);
+
+      await expect(
+        updateMessage(attackerReq, {
+          messageId: victimMessageId,
+          conversationId: victimConversationId,
+          text: 'Hacked message',
+        }),
+      ).rejects.toThrow('Message not found or user not authorized.');
+
+      expect(mockSchema.findOneAndUpdate).toHaveBeenCalledWith(
+        { messageId: victimMessageId, user: 'attacker123' },
+        expect.anything(),
+        expect.anything(),
+      );
+    });
+
+    it('should not allow deleting messages from another user\'s conversation', async () => {
+      const attackerReq = { user: { id: 'attacker123' } };
+      const victimConversationId = 'victim-convo-123';
+      const victimMessageId = 'victim-msg-123';
+
+      mockSchema.findOne().lean.mockResolvedValueOnce(null); // Simulating message not found for this user
+      const result = await deleteMessagesSince(attackerReq, {
+        messageId: victimMessageId,
+        conversationId: victimConversationId,
+      });
+
+      expect(result).toBeUndefined();
+      expect(mockSchema.findOne).toHaveBeenCalledWith({
+        messageId: victimMessageId,
+        user: 'attacker123',
+      });
+    });
+
+    it('should not allow inserting a new message into another user\'s conversation', async () => {
+      const attackerReq = { user: { id: 'attacker123' } };
+      const victimConversationId = uuidv4(); // Use a valid UUID
+
+      await expect(
+        saveMessage(attackerReq, {
+          conversationId: victimConversationId,
+          text: 'Inserted malicious message',
+          messageId: 'new-msg-123',
+        }),
+      ).resolves.not.toThrow(); // It should not throw an error
+
+      // Check that the message was saved with the attacker's user ID
+      expect(mockSchema.findOneAndUpdate).toHaveBeenCalledWith(
+        { messageId: 'new-msg-123', user: 'attacker123' },
+        expect.objectContaining({
+          user: 'attacker123',
+          conversationId: victimConversationId,
+        }),
+        expect.anything(),
+      );
+    });
+
+    it('should allow retrieving messages from any conversation', async () => {
+      const victimConversationId = 'victim-convo-123';
+
+      await getMessages({ conversationId: victimConversationId });
+
+      expect(mockSchema.find).toHaveBeenCalledWith({
+        conversationId: victimConversationId,
+      });
+
+      mockSchema.find.mockReturnValueOnce({
+        select: jest.fn().mockReturnThis(),
+        sort: jest.fn().mockReturnThis(),
+        lean: jest.fn().mockResolvedValue([{ text: 'Test message' }]),
+      });
+
+      const result = await getMessages({ conversationId: victimConversationId });
+      expect(result).toEqual([{ text: 'Test message' }]);
+    });
+  });
+});

api/models/Preset.js

@@ -39,6 +39,12 @@ module.exports = {
     try {
       const setter = { $set: {} };
       const update = { presetId, ...preset };
+      if (preset.tools && Array.isArray(preset.tools)) {
+        update.tools =
+          preset.tools
+            .map((tool) => tool?.pluginKey ?? tool)
+            .filter((toolName) => typeof toolName === 'string') ?? [];
+      }
       if (newPresetId) {
         update.presetId = newPresetId;
       }

api/models/Project.js

@@ -0,0 +1,90 @@
+const { model } = require('mongoose');
+const projectSchema = require('~/models/schema/projectSchema');
+
+const Project = model('Project', projectSchema);
+
+/**
+ * Retrieve a project by ID and convert the found project document to a plain object.
+ *
+ * @param {string} projectId - The ID of the project to find and return as a plain object.
+ * @param {string|string[]} [fieldsToSelect] - The fields to include or exclude in the returned document.
+ * @returns {Promise<MongoProject>} A plain object representing the project document, or `null` if no project is found.
+ */
+const getProjectById = async function (projectId, fieldsToSelect = null) {
+  const query = Project.findById(projectId);
+
+  if (fieldsToSelect) {
+    query.select(fieldsToSelect);
+  }
+
+  return await query.lean();
+};
+
+/**
+ * Retrieve a project by name and convert the found project document to a plain object.
+ * If the project with the given name doesn't exist and the name is "instance", create it and return the lean version.
+ *
+ * @param {string} projectName - The name of the project to find or create.
+ * @param {string|string[]} [fieldsToSelect] - The fields to include or exclude in the returned document.
+ * @returns {Promise<MongoProject>} A plain object representing the project document.
+ */
+const getProjectByName = async function (projectName, fieldsToSelect = null) {
+  const query = { name: projectName };
+  const update = { $setOnInsert: { name: projectName } };
+  const options = {
+    new: true,
+    upsert: projectName === 'instance',
+    lean: true,
+    select: fieldsToSelect,
+  };
+
+  return await Project.findOneAndUpdate(query, update, options);
+};
+
+/**
+ * Add an array of prompt group IDs to a project's promptGroupIds array, ensuring uniqueness.
+ *
+ * @param {string} projectId - The ID of the project to update.
+ * @param {string[]} promptGroupIds - The array of prompt group IDs to add to the project.
+ * @returns {Promise<MongoProject>} The updated project document.
+ */
+const addGroupIdsToProject = async function (projectId, promptGroupIds) {
+  return await Project.findByIdAndUpdate(
+    projectId,
+    { $addToSet: { promptGroupIds: { $each: promptGroupIds } } },
+    { new: true },
+  );
+};
+
+/**
+ * Remove an array of prompt group IDs from a project's promptGroupIds array.
+ *
+ * @param {string} projectId - The ID of the project to update.
+ * @param {string[]} promptGroupIds - The array of prompt group IDs to remove from the project.
+ * @returns {Promise<MongoProject>} The updated project document.
+ */
+const removeGroupIdsFromProject = async function (projectId, promptGroupIds) {
+  return await Project.findByIdAndUpdate(
+    projectId,
+    { $pull: { promptGroupIds: { $in: promptGroupIds } } },
+    { new: true },
+  );
+};
+
+/**
+ * Remove a prompt group ID from all projects.
+ *
+ * @param {string} promptGroupId - The ID of the prompt group to remove from projects.
+ * @returns {Promise<void>}
+ */
+const removeGroupFromAllProjects = async (promptGroupId) => {
+  await Project.updateMany({}, { $pull: { promptGroupIds: promptGroupId } });
+};
+
+module.exports = {
+  getProjectById,
+  getProjectByName,
+  addGroupIdsToProject,
+  removeGroupIdsFromProject,
+  removeGroupFromAllProjects,
+};

api/models/Prompt.js

@@ -1,52 +1,528 @@
-const mongoose = require('mongoose');
+const { ObjectId } = require('mongodb');
+const { SystemRoles, SystemCategories } = require('librechat-data-provider');
+const {
+  getProjectByName,
+  addGroupIdsToProject,
+  removeGroupIdsFromProject,
+  removeGroupFromAllProjects,
+} = require('./Project');
+const { Prompt, PromptGroup } = require('./schema/promptSchema');
 const { logger } = require('~/config');
 
-const promptSchema = mongoose.Schema(
-  {
-    title: {
-      type: String,
-      required: true,
+/**
+ * Create a pipeline for the aggregation to get prompt groups
+ * @param {Object} query
+ * @param {number} skip
+ * @param {number} limit
+ * @returns {[Object]} - The pipeline for the aggregation
+ */
+const createGroupPipeline = (query, skip, limit) => {
+  return [
+    { $match: query },
+    { $sort: { createdAt: -1 } },
+    { $skip: skip },
+    { $limit: limit },
+    {
+      $lookup: {
+        from: 'prompts',
+        localField: 'productionId',
+        foreignField: '_id',
+        as: 'productionPrompt',
+      },
     },
-    prompt: {
-      type: String,
-      required: true,
+    { $unwind: { path: '$productionPrompt', preserveNullAndEmptyArrays: true } },
+    {
+      $project: {
+        name: 1,
+        numberOfGenerations: 1,
+        oneliner: 1,
+        category: 1,
+        projectIds: 1,
+        productionId: 1,
+        author: 1,
+        authorName: 1,
+        createdAt: 1,
+        updatedAt: 1,
+        'productionPrompt.prompt': 1,
+        // 'productionPrompt._id': 1,
+        // 'productionPrompt.type': 1,
+      },
     },
-    category: {
-      type: String,
-    },
-  },
-  { timestamps: true },
-);
+  ];
+};
 
-const Prompt = mongoose.models.Prompt || mongoose.model('Prompt', promptSchema);
+/**
+ * Create a pipeline for the aggregation to get all prompt groups
+ * @param {Object} query
+ * @param {Partial<MongoPromptGroup>} $project
+ * @returns {[Object]} - The pipeline for the aggregation
+ */
+const createAllGroupsPipeline = (
+  query,
+  $project = {
+    name: 1,
+    oneliner: 1,
+    category: 1,
+    author: 1,
+    authorName: 1,
+    createdAt: 1,
+    updatedAt: 1,
+    command: 1,
+    'productionPrompt.prompt': 1,
+  },
+) => {
+  return [
+    { $match: query },
+    { $sort: { createdAt: -1 } },
+    {
+      $lookup: {
+        from: 'prompts',
+        localField: 'productionId',
+        foreignField: '_id',
+        as: 'productionPrompt',
+      },
+    },
+    { $unwind: { path: '$productionPrompt', preserveNullAndEmptyArrays: true } },
+    {
+      $project,
+    },
+  ];
+};
+
+/**
+ * Get all prompt groups with filters
+ * @param {Object} req
+ * @param {TPromptGroupsWithFilterRequest} filter
+ * @returns {Promise<PromptGroupListResponse>}
+ */
+const getAllPromptGroups = async (req, filter) => {
+  try {
+    const { name, ...query } = filter;
+
+    if (!query.author) {
+      throw new Error('Author is required');
+    }
+
+    let searchShared = true;
+    let searchSharedOnly = false;
+    if (name) {
+      query.name = new RegExp(name, 'i');
+    }
+    if (!query.category) {
+      delete query.category;
+    } else if (query.category === SystemCategories.MY_PROMPTS) {
+      searchShared = false;
+      delete query.category;
+    } else if (query.category === SystemCategories.NO_CATEGORY) {
+      query.category = '';
+    } else if (query.category === SystemCategories.SHARED_PROMPTS) {
+      searchSharedOnly = true;
+      delete query.category;
+    }
+
+    let combinedQuery = query;
+
+    if (searchShared) {
+      const project = await getProjectByName('instance', 'promptGroupIds');
+      if (project && project.promptGroupIds.length > 0) {
+        const projectQuery = { _id: { $in: project.promptGroupIds }, ...query };
+        delete projectQuery.author;
+        combinedQuery = searchSharedOnly ? projectQuery : { $or: [projectQuery, query] };
+      }
+    }
+
+    const promptGroupsPipeline = createAllGroupsPipeline(combinedQuery);
+    return await PromptGroup.aggregate(promptGroupsPipeline).exec();
+  } catch (error) {
+    console.error('Error getting all prompt groups', error);
+    return { message: 'Error getting all prompt groups' };
+  }
+};
+
+/**
+ * Get prompt groups with filters
+ * @param {Object} req
+ * @param {TPromptGroupsWithFilterRequest} filter
+ * @returns {Promise<PromptGroupListResponse>}
+ */
+const getPromptGroups = async (req, filter) => {
+  try {
+    const { pageNumber = 1, pageSize = 10, name, ...query } = filter;
+
+    const validatedPageNumber = Math.max(parseInt(pageNumber, 10), 1);
+    const validatedPageSize = Math.max(parseInt(pageSize, 10), 1);
+
+    if (!query.author) {
+      throw new Error('Author is required');
+    }
+
+    let searchShared = true;
+    let searchSharedOnly = false;
+    if (name) {
+      query.name = new RegExp(name, 'i');
+    }
+    if (!query.category) {
+      delete query.category;
+    } else if (query.category === SystemCategories.MY_PROMPTS) {
+      searchShared = false;
+      delete query.category;
+    } else if (query.category === SystemCategories.NO_CATEGORY) {
+      query.category = '';
+    } else if (query.category === SystemCategories.SHARED_PROMPTS) {
+      searchSharedOnly = true;
+      delete query.category;
+    }
+
+    let combinedQuery = query;
+
+    if (searchShared) {
+      // const projects = req.user.projects || []; // TODO: handle multiple projects
+      const project = await getProjectByName('instance', 'promptGroupIds');
+      if (project && project.promptGroupIds.length > 0) {
+        const projectQuery = { _id: { $in: project.promptGroupIds }, ...query };
+        delete projectQuery.author;
+        combinedQuery = searchSharedOnly ? projectQuery : { $or: [projectQuery, query] };
+      }
+    }
+
+    const skip = (validatedPageNumber - 1) * validatedPageSize;
+    const limit = validatedPageSize;
+
+    const promptGroupsPipeline = createGroupPipeline(combinedQuery, skip, limit);
+    const totalPromptGroupsPipeline = [{ $match: combinedQuery }, { $count: 'total' }];
+
+    const [promptGroupsResults, totalPromptGroupsResults] = await Promise.all([
+      PromptGroup.aggregate(promptGroupsPipeline).exec(),
+      PromptGroup.aggregate(totalPromptGroupsPipeline).exec(),
+    ]);
+
+    const promptGroups = promptGroupsResults;
+    const totalPromptGroups =
+      totalPromptGroupsResults.length > 0 ? totalPromptGroupsResults[0].total : 0;
+
+    return {
+      promptGroups,
+      pageNumber: validatedPageNumber.toString(),
+      pageSize: validatedPageSize.toString(),
+      pages: Math.ceil(totalPromptGroups / validatedPageSize).toString(),
+    };
+  } catch (error) {
+    console.error('Error getting prompt groups', error);
+    return { message: 'Error getting prompt groups' };
+  }
+};
 
 module.exports = {
-  savePrompt: async ({ title, prompt }) => {
+  getPromptGroups,
+  getAllPromptGroups,
+  /**
+   * Create a prompt and its respective group
+   * @param {TCreatePromptRecord} saveData
+   * @returns {Promise<TCreatePromptResponse>}
+   */
+  createPromptGroup: async (saveData) => {
     try {
-      await Prompt.create({
-        title,
-        prompt,
-      });
-      return { title, prompt };
+      const { prompt, group, author, authorName } = saveData;
+
+      let newPromptGroup = await PromptGroup.findOneAndUpdate(
+        { ...group, author, authorName, productionId: null },
+        { $setOnInsert: { ...group, author, authorName, productionId: null } },
+        { new: true, upsert: true },
+      )
+        .lean()
+        .select('-__v')
+        .exec();
+
+      const newPrompt = await Prompt.findOneAndUpdate(
+        { ...prompt, author, groupId: newPromptGroup._id },
+        { $setOnInsert: { ...prompt, author, groupId: newPromptGroup._id } },
+        { new: true, upsert: true },
+      )
+        .lean()
+        .select('-__v')
+        .exec();
+
+      newPromptGroup = await PromptGroup.findByIdAndUpdate(
+        newPromptGroup._id,
+        { productionId: newPrompt._id },
+        { new: true },
+      )
+        .lean()
+        .select('-__v')
+        .exec();
+
+      return {
+        prompt: newPrompt,
+        group: {
+          ...newPromptGroup,
+          productionPrompt: { prompt: newPrompt.prompt },
+        },
+      };
+    } catch (error) {
+      logger.error('Error saving prompt group', error);
+      throw new Error('Error saving prompt group');
+    }
+  },
+  /**
+   * Save a prompt
+   * @param {TCreatePromptRecord} saveData
+   * @returns {Promise<TCreatePromptResponse>}
+   */
+  savePrompt: async (saveData) => {
+    try {
+      const { prompt, author } = saveData;
+      const newPromptData = {
+        ...prompt,
+        author,
+      };
+
+      /** @type {TPrompt} */
+      let newPrompt;
+      try {
+        newPrompt = await Prompt.create(newPromptData);
+      } catch (error) {
+        if (error?.message?.includes('groupId_1_version_1')) {
+          await Prompt.db.collection('prompts').dropIndex('groupId_1_version_1');
+        } else {
+          throw error;
+        }
+        newPrompt = await Prompt.create(newPromptData);
+      }
+
+      return { prompt: newPrompt };
     } catch (error) {
       logger.error('Error saving prompt', error);
-      return { prompt: 'Error saving prompt' };
+      return { message: 'Error saving prompt' };
     }
   },
   getPrompts: async (filter) => {
     try {
-      return await Prompt.find(filter).lean();
+      return await Prompt.find(filter).sort({ createdAt: -1 }).lean();
     } catch (error) {
       logger.error('Error getting prompts', error);
-      return { prompt: 'Error getting prompts' };
+      return { message: 'Error getting prompts' };
     }
   },
-  deletePrompts: async (filter) => {
+  getPrompt: async (filter) => {
     try {
-      return await Prompt.deleteMany(filter);
+      if (filter.groupId) {
+        filter.groupId = new ObjectId(filter.groupId);
+      }
+      return await Prompt.findOne(filter).lean();
     } catch (error) {
-      logger.error('Error deleting prompts', error);
-      return { prompt: 'Error deleting prompts' };
+      logger.error('Error getting prompt', error);
+      return { message: 'Error getting prompt' };
+    }
+  },
+  /**
+   * Get prompt groups with filters
+   * @param {TGetRandomPromptsRequest} filter
+   * @returns {Promise<TGetRandomPromptsResponse>}
+   */
+  getRandomPromptGroups: async (filter) => {
+    try {
+      const result = await PromptGroup.aggregate([
+        {
+          $match: {
+            category: { $ne: '' },
+          },
+        },
+        {
+          $group: {
+            _id: '$category',
+            promptGroup: { $first: '$$ROOT' },
+          },
+        },
+        {
+          $replaceRoot: { newRoot: '$promptGroup' },
+        },
+        {
+          $sample: { size: +filter.limit + +filter.skip },
+        },
+        {
+          $skip: +filter.skip,
+        },
+        {
+          $limit: +filter.limit,
+        },
+      ]);
+      return { prompts: result };
+    } catch (error) {
+      logger.error('Error getting prompt groups', error);
+      return { message: 'Error getting prompt groups' };
+    }
+  },
+  getPromptGroupsWithPrompts: async (filter) => {
+    try {
+      return await PromptGroup.findOne(filter)
+        .populate({
+          path: 'prompts',
+          select: '-_id -__v -user',
+        })
+        .select('-_id -__v -user')
+        .lean();
+    } catch (error) {
+      logger.error('Error getting prompt groups', error);
+      return { message: 'Error getting prompt groups' };
+    }
+  },
+  getPromptGroup: async (filter) => {
+    try {
+      return await PromptGroup.findOne(filter).lean();
+    } catch (error) {
+      logger.error('Error getting prompt group', error);
+      return { message: 'Error getting prompt group' };
+    }
+  },
+  /**
+   * Deletes a prompt and its corresponding prompt group if it is the last prompt in the group.
+   *
+   * @param {Object} options - The options for deleting the prompt.
+   * @param {ObjectId|string} options.promptId - The ID of the prompt to delete.
+   * @param {ObjectId|string} options.groupId - The ID of the prompt's group.
+   * @param {ObjectId|string} options.author - The ID of the prompt's author.
+   * @param {string} options.role - The role of the prompt's author.
+   * @return {Promise<TDeletePromptResponse>} An object containing the result of the deletion.
+   * If the prompt was deleted successfully, the object will have a property 'prompt' with the value 'Prompt deleted successfully'.
+   * If the prompt group was deleted successfully, the object will have a property 'promptGroup' with the message 'Prompt group deleted successfully' and id of the deleted group.
+   * If there was an error deleting the prompt, the object will have a property 'message' with the value 'Error deleting prompt'.
+   */
+  deletePrompt: async ({ promptId, groupId, author, role }) => {
+    const query = { _id: promptId, groupId, author };
+    if (role === SystemRoles.ADMIN) {
+      delete query.author;
+    }
+    const { deletedCount } = await Prompt.deleteOne(query);
+    if (deletedCount === 0) {
+      throw new Error('Failed to delete the prompt');
+    }
+
+    const remainingPrompts = await Prompt.find({ groupId })
+      .select('_id')
+      .sort({ createdAt: 1 })
+      .lean();
+
+    if (remainingPrompts.length === 0) {
+      await PromptGroup.deleteOne({ _id: groupId });
+      await removeGroupFromAllProjects(groupId);
+
+      return {
+        prompt: 'Prompt deleted successfully',
+        promptGroup: {
+          message: 'Prompt group deleted successfully',
+          id: groupId,
+        },
+      };
+    } else {
+      const promptGroup = await PromptGroup.findById(groupId).lean();
+      if (promptGroup.productionId.toString() === promptId.toString()) {
+        await PromptGroup.updateOne(
+          { _id: groupId },
+          { productionId: remainingPrompts[remainingPrompts.length - 1]._id },
+        );
+      }
+
+      return { prompt: 'Prompt deleted successfully' };
+    }
+  },
+  /**
+   * Update prompt group
+   * @param {Partial<MongoPromptGroup>} filter - Filter to find prompt group
+   * @param {Partial<MongoPromptGroup>} data - Data to update
+   * @returns {Promise<TUpdatePromptGroupResponse>}
+   */
+  updatePromptGroup: async (filter, data) => {
+    try {
+      const updateOps = {};
+      if (data.removeProjectIds) {
+        for (const projectId of data.removeProjectIds) {
+          await removeGroupIdsFromProject(projectId, [filter._id]);
+        }
+
+        updateOps.$pull = { projectIds: { $in: data.removeProjectIds } };
+        delete data.removeProjectIds;
+      }
+
+      if (data.projectIds) {
+        for (const projectId of data.projectIds) {
+          await addGroupIdsToProject(projectId, [filter._id]);
+        }
+
+        updateOps.$addToSet = { projectIds: { $each: data.projectIds } };
+        delete data.projectIds;
+      }
+
+      const updateData = { ...data, ...updateOps };
+      const updatedDoc = await PromptGroup.findOneAndUpdate(filter, updateData, {
+        new: true,
+        upsert: false,
+      });
+
+      if (!updatedDoc) {
+        throw new Error('Prompt group not found');
+      }
+
+      return updatedDoc;
+    } catch (error) {
+      logger.error('Error updating prompt group', error);
+      return { message: 'Error updating prompt group' };
+    }
+  },
+  /**
+   * Function to make a prompt production based on its ID.
+   * @param {String} promptId - The ID of the prompt to make production.
+   * @returns {Object} The result of the production operation.
+   */
+  makePromptProduction: async (promptId) => {
+    try {
+      const prompt = await Prompt.findById(promptId).lean();
+
+      if (!prompt) {
+        throw new Error('Prompt not found');
+      }
+
+      await PromptGroup.findByIdAndUpdate(
+        prompt.groupId,
+        { productionId: prompt._id },
+        { new: true },
+      )
+        .lean()
+        .exec();
+
+      return {
+        message: 'Prompt production made successfully',
+      };
+    } catch (error) {
+      logger.error('Error making prompt production', error);
+      return { message: 'Error making prompt production' };
+    }
+  },
+  updatePromptLabels: async (_id, labels) => {
+    try {
+      const response = await Prompt.updateOne({ _id }, { $set: { labels } });
+      if (response.matchedCount === 0) {
+        return { message: 'Prompt not found' };
+      }
+      return { message: 'Prompt labels updated successfully' };
+    } catch (error) {
+      logger.error('Error updating prompt labels', error);
+      return { message: 'Error updating prompt labels' };
+    }
+  },
+  deletePromptGroup: async (_id) => {
+    try {
+      const response = await PromptGroup.deleteOne({ _id });
+
+      if (response.deletedCount === 0) {
+        return { promptGroup: 'Prompt group not found' };
+      }
+
+      await Prompt.deleteMany({ groupId: new ObjectId(_id) });
+      await removeGroupFromAllProjects(_id);
+      return { promptGroup: 'Prompt group deleted successfully' };
+    } catch (error) {
+      logger.error('Error deleting prompt group', error);
+      return { message: 'Error deleting prompt group' };
     }
   },
 };

api/models/Role.js

@@ -0,0 +1,86 @@
+const { SystemRoles, CacheKeys, roleDefaults } = require('librechat-data-provider');
+const getLogStores = require('~/cache/getLogStores');
+const Role = require('~/models/schema/roleSchema');
+
+/**
+ * Retrieve a role by name and convert the found role document to a plain object.
+ * If the role with the given name doesn't exist and the name is a system defined role, create it and return the lean version.
+ *
+ * @param {string} roleName - The name of the role to find or create.
+ * @param {string|string[]} [fieldsToSelect] - The fields to include or exclude in the returned document.
+ * @returns {Promise<Object>} A plain object representing the role document.
+ */
+const getRoleByName = async function (roleName, fieldsToSelect = null) {
+  try {
+    const cache = getLogStores(CacheKeys.ROLES);
+    const cachedRole = await cache.get(roleName);
+    if (cachedRole) {
+      return cachedRole;
+    }
+    let query = Role.findOne({ name: roleName });
+    if (fieldsToSelect) {
+      query = query.select(fieldsToSelect);
+    }
+    let role = await query.lean().exec();
+
+    if (!role && SystemRoles[roleName]) {
+      role = roleDefaults[roleName];
+      role = await new Role(role).save();
+      await cache.set(roleName, role);
+      return role.toObject();
+    }
+    await cache.set(roleName, role);
+    return role;
+  } catch (error) {
+    throw new Error(`Failed to retrieve or create role: ${error.message}`);
+  }
+};
+
+/**
+ * Update role values by name.
+ *
+ * @param {string} roleName - The name of the role to update.
+ * @param {Partial<TRole>} updates - The fields to update.
+ * @returns {Promise<TRole>} Updated role document.
+ */
+const updateRoleByName = async function (roleName, updates) {
+  try {
+    const cache = getLogStores(CacheKeys.ROLES);
+    const role = await Role.findOneAndUpdate(
+      { name: roleName },
+      { $set: updates },
+      { new: true, lean: true },
+    )
+      .select('-__v')
+      .lean()
+      .exec();
+    await cache.set(roleName, role);
+    return role;
+  } catch (error) {
+    throw new Error(`Failed to update role: ${error.message}`);
+  }
+};
+
+/**
+ * Initialize default roles in the system.
+ * Creates the default roles (ADMIN, USER) if they don't exist in the database.
+ *
+ * @returns {Promise<void>}
+ */
+const initializeRoles = async function () {
+  const defaultRoles = [SystemRoles.ADMIN, SystemRoles.USER];
+
+  for (const roleName of defaultRoles) {
+    let role = await Role.findOne({ name: roleName }).select('name').lean();
+    if (!role) {
+      role = new Role(roleDefaults[roleName]);
+      await role.save();
+    }
+  }
+};
+
+module.exports = {
+  getRoleByName,
+  initializeRoles,
+  updateRoleByName,
+};

api/models/Share.js

@@ -0,0 +1,117 @@
+const crypto = require('crypto');
+const { getMessages } = require('./Message');
+const SharedLink = require('./schema/shareSchema');
+const logger = require('~/config/winston');
+
+module.exports = {
+  SharedLink,
+  getSharedMessages: async (shareId) => {
+    try {
+      const share = await SharedLink.findOne({ shareId })
+        .populate({
+          path: 'messages',
+          select: '-_id -__v -user',
+        })
+        .select('-_id -__v -user')
+        .lean();
+
+      if (!share || !share.conversationId || !share.isPublic) {
+        return null;
+      }
+
+      return share;
+    } catch (error) {
+      logger.error('[getShare] Error getting share link', error);
+      throw new Error('Error getting share link');
+    }
+  },
+
+  getSharedLinks: async (user, pageNumber = 1, pageSize = 25, isPublic = true) => {
+    const query = { user, isPublic };
+    try {
+      const totalConvos = (await SharedLink.countDocuments(query)) || 1;
+      const totalPages = Math.ceil(totalConvos / pageSize);
+      const shares = await SharedLink.find(query)
+        .sort({ updatedAt: -1 })
+        .skip((pageNumber - 1) * pageSize)
+        .limit(pageSize)
+        .select('-_id -__v -user')
+        .lean();
+
+      return { sharedLinks: shares, pages: totalPages, pageNumber, pageSize };
+    } catch (error) {
+      logger.error('[getShareByPage] Error getting shares', error);
+      throw new Error('Error getting shares');
+    }
+  },
+
+  createSharedLink: async (user, { conversationId, ...shareData }) => {
+    try {
+      const share = await SharedLink.findOne({ conversationId }).select('-_id -__v -user').lean();
+      if (share) {
+        return share;
+      }
+
+      const shareId = crypto.randomUUID();
+      const messages = await getMessages({ conversationId });
+      const update = { ...shareData, shareId, messages, user };
+      return await SharedLink.findOneAndUpdate({ conversationId: conversationId, user }, update, {
+        new: true,
+        upsert: true,
+      });
+    } catch (error) {
+      logger.error('[createSharedLink] Error creating shared link', error);
+      throw new Error('Error creating shared link');
+    }
+  },
+
+  updateSharedLink: async (user, { conversationId, ...shareData }) => {
+    try {
+      const share = await SharedLink.findOne({ conversationId }).select('-_id -__v -user').lean();
+      if (!share) {
+        return { message: 'Share not found' };
+      }
+
+      // update messages to the latest
+      const messages = await getMessages({ conversationId });
+      const update = { ...shareData, messages, user };
+      return await SharedLink.findOneAndUpdate({ conversationId: conversationId, user }, update, {
+        new: true,
+        upsert: false,
+      });
+    } catch (error) {
+      logger.error('[updateSharedLink] Error updating shared link', error);
+      throw new Error('Error updating shared link');
+    }
+  },
+
+  deleteSharedLink: async (user, { shareId }) => {
+    try {
+      const share = await SharedLink.findOne({ shareId, user });
+      if (!share) {
+        return { message: 'Share not found' };
+      }
+      return await SharedLink.findOneAndDelete({ shareId, user });
+    } catch (error) {
+      logger.error('[deleteSharedLink] Error deleting shared link', error);
+      throw new Error('Error deleting shared link');
+    }
+  },
+  /**
+   * Deletes all shared links for a specific user.
+   * @param {string} user - The user ID.
+   * @returns {Promise<{ message: string, deletedCount?: number }>} A result object indicating success or error message.
+   */
+  deleteAllSharedLinks: async (user) => {
+    try {
+      const result = await SharedLink.deleteMany({ user });
+      return {
+        message: 'All shared links have been deleted successfully',
+        deletedCount: result.deletedCount,
+      };
+    } catch (error) {
+      logger.error('[deleteAllSharedLinks] Error deleting shared links', error);
+      throw new Error('Error deleting shared links');
+    }
+  },
+};

api/models/Transaction.js

@@ -2,6 +2,7 @@ const mongoose = require('mongoose');
 const { isEnabled } = require('../server/utils/handleText');
 const transactionSchema = require('./schema/transaction');
 const { getMultiplier } = require('./tx');
+const { logger } = require('~/config');
 const Balance = require('./Balance');
 const cancelRate = 1.15;
 
@@ -10,8 +11,8 @@ transactionSchema.methods.calculateTokenValue = function () {
   if (!this.valueKey || !this.tokenType) {
     this.tokenValue = this.rawAmount;
   }
-  const { valueKey, tokenType, model } = this;
-  const multiplier = getMultiplier({ valueKey, tokenType, model });
+  const { valueKey, tokenType, model, endpointTokenConfig } = this;
+  const multiplier = Math.abs(getMultiplier({ valueKey, tokenType, model, endpointTokenConfig }));
   this.rate = multiplier;
   this.tokenValue = this.rawAmount * multiplier;
   if (this.context && this.tokenType === 'completion' && this.context === 'incomplete') {
@@ -25,6 +26,7 @@ transactionSchema.statics.create = async function (transactionData) {
   const Transaction = this;
 
   const transaction = new Transaction(transactionData);
+  transaction.endpointTokenConfig = transactionData.endpointTokenConfig;
   transaction.calculateTokenValue();
 
   // Save the transaction
@@ -34,12 +36,44 @@ transactionSchema.statics.create = async function (transactionData) {
     return;
   }
 
-  // Adjust the user's balance
-  return await Balance.findOneAndUpdate(
+  let balance = await Balance.findOne({ user: transaction.user }).lean();
+  let incrementValue = transaction.tokenValue;
+
+  if (balance && balance?.tokenCredits + incrementValue < 0) {
+    incrementValue = -balance.tokenCredits;
+  }
+
+  balance = await Balance.findOneAndUpdate(
     { user: transaction.user },
-    { $inc: { tokenCredits: transaction.tokenValue } },
+    { $inc: { tokenCredits: incrementValue } },
     { upsert: true, new: true },
   ).lean();
+
+  return {
+    rate: transaction.rate,
+    user: transaction.user.toString(),
+    balance: balance.tokenCredits,
+    [transaction.tokenType]: incrementValue,
+  };
 };
 
-module.exports = mongoose.model('Transaction', transactionSchema);
+const Transaction = mongoose.model('Transaction', transactionSchema);
+
+/**
+ * Queries and retrieves transactions based on a given filter.
+ * @async
+ * @function getTransactions
+ * @param {Object} filter - MongoDB filter object to apply when querying transactions.
+ * @returns {Promise<Array>} A promise that resolves to an array of matched transactions.
+ * @throws {Error} Throws an error if querying the database fails.
+ */
+async function getTransactions(filter) {
+  try {
+    return await Transaction.find(filter).lean();
+  } catch (error) {
+    logger.error('Error querying transactions:', error);
+    throw error;
+  }
+}
+
+module.exports = { Transaction, getTransactions };

api/models/User.js

@@ -1,61 +1,5 @@
 const mongoose = require('mongoose');
-const bcrypt = require('bcryptjs');
-const signPayload = require('../server/services/signPayload');
-const userSchema = require('./schema/userSchema.js');
-const { SESSION_EXPIRY } = process.env ?? {};
-const expires = eval(SESSION_EXPIRY) ?? 1000 * 60 * 15;
-
-userSchema.methods.toJSON = function () {
-  return {
-    id: this._id,
-    provider: this.provider,
-    email: this.email,
-    name: this.name,
-    username: this.username,
-    avatar: this.avatar,
-    role: this.role,
-    emailVerified: this.emailVerified,
-    plugins: this.plugins,
-    createdAt: this.createdAt,
-    updatedAt: this.updatedAt,
-  };
-};
-
-userSchema.methods.generateToken = async function () {
-  return await signPayload({
-    payload: {
-      id: this._id,
-      username: this.username,
-      provider: this.provider,
-      email: this.email,
-    },
-    secret: process.env.JWT_SECRET,
-    expirationTime: expires / 1000,
-  });
-};
-
-userSchema.methods.comparePassword = function (candidatePassword, callback) {
-  bcrypt.compare(candidatePassword, this.password, (err, isMatch) => {
-    if (err) {
-      return callback(err);
-    }
-    callback(null, isMatch);
-  });
-};
-
-module.exports.hashPassword = async (password) => {
-  const hashedPassword = await new Promise((resolve, reject) => {
-    bcrypt.hash(password, 10, function (err, hash) {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(hash);
-      }
-    });
-  });
-
-  return hashedPassword;
-};
+const userSchema = require('~/models/schema/userSchema');
 
 const User = mongoose.model('User', userSchema);
 

api/models/checkBalance.js

@@ -1,5 +1,6 @@
+const { ViolationTypes } = require('librechat-data-provider');
+const { logViolation } = require('~/cache');
 const Balance = require('./Balance');
-const { logViolation } = require('../cache');
 /**
  * Checks the balance for a user and determines if they can spend a certain amount.
  * If the user cannot spend the amount, it logs a violation and denies the request.
@@ -14,6 +15,7 @@ const { logViolation } = require('../cache');
  * @param {('prompt' | 'completion')} params.txData.tokenType - The type of token.
  * @param {number} params.txData.amount - The amount of tokens.
  * @param {string} params.txData.model - The model name or identifier.
+ * @param {string} [params.txData.endpointTokenConfig] - The token configuration for the endpoint.
  * @returns {Promise<boolean>} Returns true if the user can spend the amount, otherwise denies the request.
  * @throws {Error} Throws an error if there's an issue with the balance check.
  */
@@ -24,7 +26,7 @@ const checkBalance = async ({ req, res, txData }) => {
     return true;
   }
 
-  const type = 'token_balance';
+  const type = ViolationTypes.TOKEN_BALANCE;
   const errorMessage = {
     type,
     balance,

api/models/index.js

@@ -1,10 +1,21 @@
 const {
   getMessages,
   saveMessage,
+  recordMessage,
   updateMessage,
   deleteMessagesSince,
   deleteMessages,
 } = require('./Message');
+const {
+  comparePassword,
+  deleteUserById,
+  generateToken,
+  getUserById,
+  updateUser,
+  createUser,
+  countUsers,
+  findUser,
+} = require('./userMethods');
 const { getConvoTitle, getConvo, saveConvo, deleteConvos } = require('./Conversation');
 const { getPreset, getPresets, savePreset, deletePresets } = require('./Preset');
 const {
@@ -20,17 +31,25 @@ const Key = require('./Key');
 const User = require('./User');
 const Session = require('./Session');
 const Balance = require('./Balance');
-const Transaction = require('./Transaction');
 
 module.exports = {
   User,
   Key,
   Session,
   Balance,
-  Transaction,
+
+  comparePassword,
+  deleteUserById,
+  generateToken,
+  getUserById,
+  countUsers,
+  createUser,
+  updateUser,
+  findUser,
 
   getMessages,
   saveMessage,
+  recordMessage,
   updateMessage,
   deleteMessagesSince,
   deleteMessages,

api/models/plugins/mongoMeili.js

@@ -155,7 +155,7 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
             function (results, value, key) {
               return { ...results, [key]: 1 };
             },
-            { _id: 1 },
+            { _id: 1, __v: 1 },
           ),
         ).lean();
 
@@ -183,6 +183,15 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
       if (object.conversationId && object.conversationId.includes('|')) {
         object.conversationId = object.conversationId.replace(/\|/g, '--');
       }
+
+      if (object.content && Array.isArray(object.content)) {
+        object.text = object.content
+          .filter((item) => item.type === 'text' && item.text && item.text.value)
+          .map((item) => item.text.value)
+          .join(' ');
+        delete object.content;
+      }
+
       return object;
     }
 
@@ -339,7 +348,7 @@ module.exports = function mongoMeili(schema, options) {
       try {
         meiliDoc = await client.index('convos').getDocument(doc.conversationId);
       } catch (error) {
-        logger.error(
+        logger.debug(
           '[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' +
             doc.conversationId,
           error,

api/models/schema/action.js

@@ -0,0 +1,59 @@
+const mongoose = require('mongoose');
+
+const { Schema } = mongoose;
+
+const AuthSchema = new Schema(
+  {
+    authorization_type: String,
+    custom_auth_header: String,
+    type: {
+      type: String,
+      enum: ['service_http', 'oauth', 'none'],
+    },
+    authorization_content_type: String,
+    authorization_url: String,
+    client_url: String,
+    scope: String,
+    token_exchange_method: {
+      type: String,
+      enum: ['default_post', 'basic_auth_header', null],
+    },
+  },
+  { _id: false },
+);
+
+const actionSchema = new Schema({
+  user: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'User',
+    index: true,
+    required: true,
+  },
+  action_id: {
+    type: String,
+    index: true,
+    required: true,
+  },
+  type: {
+    type: String,
+    default: 'action_prototype',
+  },
+  settings: Schema.Types.Mixed,
+  assistant_id: String,
+  metadata: {
+    api_key: String, // private, encrypted
+    auth: AuthSchema,
+    domain: {
+      type: String,
+      required: true,
+    },
+    // json_schema: Schema.Types.Mixed,
+    privacy_policy_url: String,
+    raw_spec: String,
+    oauth_client_id: String, // private, encrypted
+    oauth_client_secret: String, // private, encrypted
+  },
+});
+// }, { minimize: false }); // Prevent removal of empty objects
+
+module.exports = actionSchema;

api/models/schema/assistant.js

@@ -0,0 +1,33 @@
+const mongoose = require('mongoose');
+
+const assistantSchema = mongoose.Schema(
+  {
+    user: {
+      type: mongoose.Schema.Types.ObjectId,
+      ref: 'User',
+      required: true,
+    },
+    assistant_id: {
+      type: String,
+      index: true,
+      required: true,
+    },
+    avatar: {
+      type: {
+        filepath: String,
+        source: String,
+      },
+      default: undefined,
+    },
+    access_level: {
+      type: Number,
+    },
+    file_ids: { type: [String], default: undefined },
+    actions: { type: [String], default: undefined },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+module.exports = assistantSchema;

api/models/schema/categories.js

@@ -0,0 +1,19 @@
+const mongoose = require('mongoose');
+const Schema = mongoose.Schema;
+
+const categoriesSchema = new Schema({
+  label: {
+    type: String,
+    required: true,
+    unique: true,
+  },
+  value: {
+    type: String,
+    required: true,
+    unique: true,
+  },
+});
+
+const categories = mongoose.model('categories', categoriesSchema);
+
+module.exports = { Categories: categories };

api/models/schema/conversationTagSchema.js

@@ -0,0 +1,31 @@
+const mongoose = require('mongoose');
+
+const conversationTagSchema = mongoose.Schema(
+  {
+    tag: {
+      type: String,
+      index: true,
+    },
+    user: {
+      type: String,
+      index: true,
+    },
+    description: {
+      type: String,
+      index: true,
+    },
+    count: {
+      type: Number,
+      default: 0,
+    },
+    position: {
+      type: Number,
+      default: 0,
+    },
+  },
+  { timestamps: true },
+);
+
+conversationTagSchema.index({ tag: 1, user: 1 }, { unique: true });
+
+module.exports = mongoose.model('ConversationTag', conversationTagSchema);