✨ v0.7.0 (#2273)

* chore: remove version comment from pre-commit shell script

* chore: Dockerfile update

.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM node:18-bullseye
+
+RUN useradd -m -s /bin/bash vscode
+RUN mkdir -p /workspaces && chown -R vscode:vscode /workspaces
+WORKDIR /workspaces

.devcontainer/devcontainer.json

@@ -13,5 +13,6 @@
     }
   },
   "postCreateCommand": "",
-  "features": { "ghcr.io/devcontainers/features/git:1": {} }
+  "features": { "ghcr.io/devcontainers/features/git:1": {} },
+  "remoteUser": "vscode"
 }

.devcontainer/docker-compose.yml

@@ -2,7 +2,9 @@ version: "3.8"
 
 services:
   app:
-    image: node:19-bullseye
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
     # restart: always
     links:
       - mongodb
@@ -30,8 +32,8 @@ services:
     # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. 
     # (Adding the "ports" property to this file will not forward from a Codespace.)
 
-    # Uncomment the next line to use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
-    # user: vscode
+    # Use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details.
+    user: vscode
 
     # Overrides default command so things don't shut down after the process ends.
     command: /bin/sh -c "while sleep 1000; do :; done"  
@@ -57,7 +59,6 @@ services:
     # ports: 
     #   - 7700:7700 # if exposing these ports, make sure your master key is not the default value
     environment:
-      - MEILI_HTTP_ADDR=meilisearch:7700
       - MEILI_NO_ANALYTICS=true
       - MEILI_MASTER_KEY=5c71cf56d672d009e36070b5bc5e47b743535ae55c818ae3b735bb6ebfb4ba63
     volumes:

.dockerignore

@@ -1,5 +1,17 @@
+**/.circleci
+**/.editorconfig
+**/.dockerignore
+**/.git
+**/.DS_Store
+**/.vscode
 **/node_modules
-client/dist/images
+
+# Specific patterns to ignore
 data-node
-.env
-**/.env
+meili_data*
+librechat*
+Dockerfile*
+docs
+
+# Ignore all hidden files
+.*

.env.example

@@ -1,23 +1,18 @@
-#=============================================================#
-#                   LibreChat Configuration                   #
-#=============================================================#
-# Please refer to the reference documentation for assistance  #
-# with configuring your LibreChat environment. The guide is   #
-# available both online and within your local LibreChat       #
-# directory:                                                  #
-# Online: https://docs.librechat.ai/install/dotenv.html       #
-# Locally: ./docs/install/dotenv.md                           #
-#=============================================================#
+#=====================================================================#
+#                       LibreChat Configuration                       #
+#=====================================================================#
+# Please refer to the reference documentation for assistance          #
+# with configuring your LibreChat environment. The guide is           #
+# available both online and within your local LibreChat               #
+# directory:                                                          #
+# Online: https://docs.librechat.ai/install/configuration/dotenv.html #
+# Locally: ./docs/install/configuration/dotenv.md                     #
+#=====================================================================#
 
 #==================================================#
 #               Server Configuration               #
 #==================================================#
 
-APP_TITLE=LibreChat
-# CUSTOM_FOOTER="My custom footer"
-
-DEBUG_LOGGING=true
-
 HOST=localhost
 PORT=3080
 
@@ -26,6 +21,22 @@ MONGO_URI=mongodb://127.0.0.1:27017/LibreChat
 DOMAIN_CLIENT=http://localhost:3080
 DOMAIN_SERVER=http://localhost:3080
 
+NO_INDEX=true
+
+#===============#
+# JSON Logging  #
+#===============#
+
+# Use when process console logs in cloud deployment like GCP/AWS
+CONSOLE_JSON=false
+
+#===============#
+# Debug Logging #
+#===============#
+
+DEBUG_LOGGING=true
+DEBUG_CONSOLE=false
+
 #=============#
 # Permissions #
 #=============#
@@ -33,38 +44,62 @@ DOMAIN_SERVER=http://localhost:3080
 # UID=1000
 # GID=1000
 
+#===============#
+# Configuration #
+#===============#
+# Use an absolute path, a relative path, or a URL
+
+# CONFIG_PATH="/alternative/path/to/librechat.yaml"
+
 #===================================================#
 #                     Endpoints                     #
 #===================================================#
 
-# ENDPOINTS=openAI,azureOpenAI,bingAI,chatGPTBrowser,google,gptPlugins,anthropic
+# ENDPOINTS=openAI,assistants,azureOpenAI,bingAI,google,gptPlugins,anthropic
 
 PROXY=
 
+#===================================#
+# Known Endpoints - librechat.yaml  #
+#===================================#
+# https://docs.librechat.ai/install/configuration/ai_endpoints.html
+
+# GROQ_API_KEY=
+# SHUTTLEAI_KEY=
+# OPENROUTER_KEY=
+# MISTRAL_API_KEY=
+# ANYSCALE_API_KEY=
+# FIREWORKS_API_KEY=
+# PERPLEXITY_API_KEY=
+# TOGETHERAI_API_KEY=
+
 #============#
 # Anthropic  #
 #============#
 
 ANTHROPIC_API_KEY=user_provided
-ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2
+# ANTHROPIC_MODELS=claude-3-opus-20240229,claude-3-sonnet-20240229,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
 # ANTHROPIC_REVERSE_PROXY=
 
 #============#
 # Azure      #
 #============#
 
-# AZURE_API_KEY=
-AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4
-# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo
-# PLUGINS_USE_AZURE="true"
 
-AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
+# Note: these variables are DEPRECATED
+# Use the `librechat.yaml` configuration for `azureOpenAI` instead
+# You may also continue to use them if you opt out of using the `librechat.yaml` configuration
 
-# AZURE_OPENAI_API_INSTANCE_NAME=
-# AZURE_OPENAI_API_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_VERSION=
-# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME=
-# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=
+# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated
+# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 # Deprecated
+# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE # Deprecated
+# AZURE_API_KEY= # Deprecated
+# AZURE_OPENAI_API_INSTANCE_NAME= # Deprecated
+# AZURE_OPENAI_API_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_VERSION= # Deprecated
+# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated
+# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated
+# PLUGINS_USE_AZURE="true" # Deprecated
 
 #============#
 # BingAI     #
@@ -73,19 +108,12 @@ AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
 BINGAI_TOKEN=user_provided
 # BINGAI_HOST=https://cn.bing.com
 
-#============#
-# ChatGPT    #
-#============#
-
-CHATGPT_TOKEN=
-CHATGPT_MODELS=text-davinci-002-render-sha
-# CHATGPT_REVERSE_PROXY=<YOUR REVERSE PROXY>
-
 #============#
 # Google     #
 #============#
 
 GOOGLE_KEY=user_provided
+# GOOGLE_MODELS=gemini-pro,gemini-pro-vision,chat-bison,chat-bison-32k,codechat-bison,codechat-bison-32k,text-bison,text-bison-32k,text-unicorn,code-gecko,code-bison,code-bison-32k
 # GOOGLE_REVERSE_PROXY=
 
 #============#
@@ -93,7 +121,7 @@ GOOGLE_KEY=user_provided
 #============#
 
 OPENAI_API_KEY=user_provided
-# OPENAI_MODELS=gpt-3.5-turbo-1106,gpt-4-1106-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,text-davinci-003,gpt-4,gpt-4-0314,gpt-4-0613
+# OPENAI_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k
 
 DEBUG_OPENAI=false
 
@@ -107,6 +135,16 @@ DEBUG_OPENAI=false
 
 # OPENAI_REVERSE_PROXY=
 
+# OPENAI_ORGANIZATION=
+
+#====================#
+#   Assistants API   #
+#====================#
+
+ASSISTANTS_API_KEY=user_provided
+# ASSISTANTS_BASE_URL=
+# ASSISTANTS_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-16k,gpt-3.5-turbo,gpt-4,gpt-4-0314,gpt-4-32k-0314,gpt-4-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-1106,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview
+
 #============#
 # OpenRouter #
 #============#
@@ -117,7 +155,7 @@ DEBUG_OPENAI=false
 # Plugins    #
 #============#
 
-# PLUGIN_MODELS=gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,gpt-4,gpt-4-0314,gpt-4-0613
+# PLUGIN_MODELS=gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613
 
 DEBUG_PLUGINS=true
 
@@ -135,11 +173,22 @@ AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE=
 AZURE_AI_SEARCH_SEARCH_OPTION_TOP=
 AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
 
-# DALL·E 3
+# DALL·E
 #----------------
 # DALLE_API_KEY=
-# DALLE3_SYSTEM_PROMPT="Your System Prompt here"
+# DALLE3_API_KEY=
+# DALLE2_API_KEY=
+# DALLE3_SYSTEM_PROMPT=
+# DALLE2_SYSTEM_PROMPT=
 # DALLE_REVERSE_PROXY=
+# DALLE3_BASEURL=
+# DALLE2_BASEURL=
+
+# DALL·E (via Azure OpenAI)
+# Note: requires some of the variables above to be set
+#----------------
+# DALLE3_AZURE_API_VERSION=
+# DALLE2_AZURE_API_VERSION=
 
 # Google
 #-----------------
@@ -154,6 +203,14 @@ SERPAPI_API_KEY=
 #-----------------
 SD_WEBUI_URL=http://host.docker.internal:7860
 
+# Tavily
+#-----------------
+TAVILY_API_KEY=
+
+# Traversaal
+#-----------------
+TRAVERSAAL_API_KEY=
+
 # WolframAlpha
 #-----------------
 WOLFRAM_APP_ID=
@@ -169,7 +226,6 @@ ZAPIER_NLA_API_KEY=
 SEARCH=true
 MEILI_NO_ANALYTICS=true
 MEILI_HOST=http://0.0.0.0:7700
-MEILI_HTTP_ADDR=0.0.0.0:7700
 MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
 
 #===================================================#
@@ -180,6 +236,10 @@ MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
 # Moderation             #
 #========================#
 
+OPENAI_MODERATION=false
+OPENAI_MODERATION_API_KEY=
+# OPENAI_MODERATION_REVERSE_PROXY=
+
 BAN_VIOLATIONS=true
 BAN_DURATION=1000 * 60 * 60 * 2
 BAN_INTERVAL=20
@@ -206,6 +266,8 @@ LIMIT_MESSAGE_USER=false
 MESSAGE_USER_MAX=40
 MESSAGE_USER_WINDOW=1
 
+ILLEGAL_MODEL_REQ_SCORE=5
+
 #========================#
 # Balance                #
 #========================#
@@ -262,17 +324,38 @@ OPENID_IMAGE_URL=
 # Email Password Reset   #
 #========================#
 
-EMAIL_SERVICE=                  
-EMAIL_HOST=                     
-EMAIL_PORT=25                   
-EMAIL_ENCRYPTION=               
-EMAIL_ENCRYPTION_HOSTNAME=      
-EMAIL_ALLOW_SELFSIGNED=         
-EMAIL_USERNAME=                 
-EMAIL_PASSWORD=                 
-EMAIL_FROM_NAME=                
+EMAIL_SERVICE=
+EMAIL_HOST=
+EMAIL_PORT=25
+EMAIL_ENCRYPTION=
+EMAIL_ENCRYPTION_HOSTNAME=
+EMAIL_ALLOW_SELFSIGNED=
+EMAIL_USERNAME=
+EMAIL_PASSWORD=
+EMAIL_FROM_NAME=
 EMAIL_FROM=noreply@librechat.ai
 
+#========================#
+# Firebase CDN           #
+#========================#
+
+FIREBASE_API_KEY=
+FIREBASE_AUTH_DOMAIN=
+FIREBASE_PROJECT_ID=
+FIREBASE_STORAGE_BUCKET=
+FIREBASE_MESSAGING_SENDER_ID=
+FIREBASE_APP_ID=
+
+#===================================================#
+#                        UI                         #
+#===================================================#
+
+APP_TITLE=LibreChat
+# CUSTOM_FOOTER="My custom footer"
+HELP_AND_FAQ_URL=https://librechat.ai
+
+# SHOW_BIRTHDAY_ICON=true
+
 #==================================================#
 #                      Others                      #
 #==================================================#

.eslintrc.js

@@ -19,6 +19,7 @@ module.exports = {
     'e2e/playwright-report/**/*',
     'packages/data-provider/types/**/*',
     'packages/data-provider/dist/**/*',
+    'packages/data-provider/test_bundle/**/*',
     'data-node/**/*',
     'meili_data/**/*',
     'node_modules/**/*',
@@ -131,6 +132,12 @@ module.exports = {
         },
       ],
     },
+    {
+      files: ['./packages/data-provider/specs/**/*.ts'],
+      parserOptions: {
+        project: './packages/data-provider/tsconfig.spec.json',
+      },
+    },
   ],
   settings: {
     react: {

.github/CODE_OF_CONDUCT.md

@@ -60,7 +60,7 @@ representative at an online or offline event.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported to the community leaders responsible for enforcement here on GitHub or
-on the official [Discord Server](https://discord.gg/uDyZ5Tzhct). 
+on the official [Discord Server](https://discord.librechat.ai). 
 All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the

.github/CONTRIBUTING.md

@@ -8,7 +8,7 @@ If the feature you would like to contribute has not already received prior appro
 
 Please note that a pull request involving a feature that has not been reviewed and approved by the project maintainers may be rejected. We appreciate your understanding and cooperation.
 
-If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.gg/uDyZ5Tzhct), where you can engage with other contributors and seek guidance from the community.
+If you would like to discuss the changes you wish to make, join our [Discord community](https://discord.librechat.ai), where you can engage with other contributors and seek guidance from the community.
 
 ## Our Standards
 

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -50,7 +50,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/SECURITY.md

@@ -12,7 +12,7 @@ When reporting a security vulnerability, you have the following options to reach
 
 - **Option 2: GitHub Issues**: You can initiate first contact via GitHub Issues. However, please note that initial contact through GitHub Issues should not include any sensitive details.
 
-- **Option 3: Discord Server**: You can join our [Discord community](https://discord.gg/5rbRxn4uME) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
+- **Option 3: Discord Server**: You can join our [Discord community](https://discord.librechat.ai) and initiate first contact in the `#issues` channel. However, please ensure that initial contact through Discord does not include any sensitive details.
 
 _After the initial contact, we will establish a private communication channel for further discussion._
 
@@ -39,11 +39,11 @@ Please note that as a security-conscious community, we may not always disclose d
 
 This security policy applies to the following GitHub repository:
 
-- Repository: [LibreChat](https://github.com/danny-avila/LibreChat)
+- Repository: [LibreChat](https://github.librechat.ai)
 
 ## Contact
 
-If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.gg/NGaa9RPCft) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
+If you have any questions or concerns regarding the security of our project, please join our [Discord community](https://discord.librechat.ai) and report them in the appropriate channel. You can also reach out to us by [opening an issue](https://github.com/danny-avila/LibreChat/issues/new) on GitHub. Please note that the response time may vary depending on the nature and severity of the inquiry.
 
 ## Acknowledgments
 

.github/pull_request_template.md

@@ -1,7 +1,7 @@
 # Pull Request Template
 
 
-### ⚠️ Before Submitting a PR, read the [Contributing Docs](./CONTRIBUTING.md) in full!
+### ⚠️ Before Submitting a PR, read the [Contributing Docs](https://github.com/danny-avila/LibreChat/blob/main/.github/CONTRIBUTING.md) in full!
 
 ## Summary
 
@@ -15,7 +15,9 @@ Please delete any irrelevant options.
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] This change requires a documentation update
-- [ ] Documentation update 
+- [ ] Translation update
+- [ ] Documentation update
+
 
 ## Testing
 
@@ -25,6 +27,8 @@ Please describe your test process and include instructions so that we can reprod
 
 ## Checklist
 
+Please delete any irrelevant options.
+
 - [ ] My code adheres to this project's style guidelines
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented in any complex areas of my code
@@ -33,3 +37,4 @@ Please describe your test process and include instructions so that we can reprod
 - [ ] I have written tests demonstrating that my changes are effective or that my feature works
 - [ ] Local unit tests pass with my changes
 - [ ] Any changes dependent on mine have been merged and published in downstream modules.
+- [ ] New documents have been locally validated with mkdocs

.github/workflows/backend-review.yml

@@ -35,10 +35,28 @@ jobs:
 
       - name: Install Data Provider
         run: npm run build:data-provider
+      
+      - name: Create empty auth.json file
+        run: |
+          mkdir -p api/data
+          echo '{}' > api/data/auth.json
+
+      - name: Check for Circular dependency in rollup
+        working-directory: ./packages/data-provider
+        run: |
+          output=$(npm run rollup:api)
+          echo "$output"
+          if echo "$output" | grep -q "Circular dependency"; then
+            echo "Error: Circular dependency detected!"
+            exit 1
+          fi
 
       - name: Run unit tests
         run: cd api && npm run test:ci
 
+      - name: Run librechat-data-provider unit tests
+        run: cd packages/data-provider && npm run test:ci
+
       - name: Run linters
         uses: wearerequired/lint-action@v2
         with:

.github/workflows/container.yml

@@ -1,52 +0,0 @@
-name: Docker Compose Build on Tag
-
-# The workflow is triggered when a tag is pushed
-on:
-  push:
-    tags:
-      - "*"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Run docker-compose build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-          docker-compose build
-          docker build -f Dockerfile.multi --target api-build -t librechat-api .
-
-      # Get Tag Name
-      - name: Get Tag Name
-        id: tag_name
-        run: echo "TAG_NAME=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
-
-      # Tag it properly before push to github
-      - name: tag image and push
-        run: |
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:${{ env.TAG_NAME }}
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:${{ env.TAG_NAME }}
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:latest

.github/workflows/dev-images.yml

@@ -8,18 +8,32 @@ on:
     paths:
       - 'api/**'
       - 'client/**'
+      - 'packages/**'
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-dev-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat-dev
 
     steps:
       # Check out the repository
       - name: Checkout
         uses: actions/checkout@v4
 
-      # Set up Docker
-      - name: Set up Docker
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       # Log in to GitHub Container Registry
@@ -30,22 +44,29 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Build Docker images
-      - name: Build Docker images
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
         run: |
           cp .env.example .env
-          docker build -f Dockerfile.multi --target api-build -t librechat-dev-api .
-          docker build -f Dockerfile -t librechat-dev .
 
-      # Tag and push the images to GitHub Container Registry
-      - name: Tag and push images
-        run: |
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:${{ github.sha }}
-          docker tag librechat-dev-api:latest ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev-api:latest
-
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:${{ github.sha }}
-          docker tag librechat-dev:latest ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-dev:latest
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.sha }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.sha }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/generate_embeddings.yml

@@ -0,0 +1,20 @@
+name: 'generate_embeddings'
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: supabase/embeddings-generator@v0.0.5
+        with:
+          supabase-url: ${{ secrets.SUPABASE_URL }}
+          supabase-service-role-key: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
+          openai-key: ${{ secrets.OPENAI_DOC_EMBEDDINGS_KEY }}
+          docs-root-path: 'docs'

.github/workflows/latest-images-main.yml

@@ -1,40 +0,0 @@
-name: Docker Compose Build on Main Branch
-
-on:
-  workflow_dispatch:  # This line allows manual triggering
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Run docker-compose build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-          docker-compose build
-          docker build -f Dockerfile.multi --target api-build -t librechat-api .
-
-      # Tag and push the images with the 'latest' tag
-      - name: Tag image and push
-        run: |
-          docker tag librechat:latest ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat:latest
-          docker tag librechat-api:latest ghcr.io/${{ github.repository_owner }}/librechat-api:latest
-          docker push ghcr.io/${{ github.repository_owner }}/librechat-api:latest

.github/workflows/main-image-workflow.yml

@@ -0,0 +1,69 @@
+name: Docker Compose Build Latest Main Image Tag (Manual Dispatch)
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Fetch tags and set the latest tag
+        run: |
+          git fetch --tags
+          echo "LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)" >> $GITHUB_ENV
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/mkdocs.yaml

@@ -21,4 +21,7 @@ jobs:
           restore-keys: |
             mkdocs-material-
       - run: pip install mkdocs-material 
+      - run: pip install mkdocs-nav-weight
+      - run: pip install mkdocs-publisher
+      - run: pip install mkdocs-exclude
       - run: mkdocs gh-deploy --force

.github/workflows/tag-images.yml

@@ -0,0 +1,67 @@
+name: Docker Images Build on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      # Check out the repository
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.gitignore

@@ -48,6 +48,10 @@ bower_components/
 .floo
 .flooignore
 
+#config file
+librechat.yaml
+librechat.yml
+
 # Environment
 .npmrc
 .env*
@@ -66,10 +70,16 @@ src/style - official.css
 .DS_Store
 *.code-workspace
 .idea
+*.iml
 *.pem
 config.local.ts
 **/storageState.json
 junit.xml
+**/.venv/
+
+# docker override file
+docker-compose.override.yaml
+docker-compose.override.yml
 
 # meilisearch
 meilisearch
@@ -78,4 +88,12 @@ data.ms/*
 auth.json
 
 /packages/ux-shared/
-/images
+/images
+
+!client/src/components/Nav/SettingsTabs/Data/
+
+# User uploads
+uploads/
+
+# owner
+release/

.husky/pre-commit

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env sh 
 set -e
 . "$(dirname -- "$0")/_/husky.sh"
 [ -n "$CI" ] && exit 0

Dockerfile

@@ -1,17 +1,26 @@
-# Base node image
-FROM node:19-alpine AS node
+# v0.7.0
 
-COPY . /app
+# Base node image
+FROM node:18-alpine3.18 AS node
+
+RUN apk add g++ make py3-pip
+RUN npm install -g node-gyp
+RUN apk --no-cache add curl
+
+RUN mkdir -p /app && chown node:node /app
 WORKDIR /app
 
-# Install call deps - Install curl for health check
-RUN apk --no-cache add curl && \
-    # We want to inherit env from the container, not the file
-    # This will preserve any existing env file if it's already in source
-    # otherwise it will create a new one
-    touch .env && \
-    # Build deps in seperate 
-    npm ci
+USER node
+
+COPY --chown=node:node . .
+
+# Allow mounting of these files, which have no default
+# values.
+RUN touch .env
+RUN npm config set fetch-retry-maxtimeout 600000
+RUN npm config set fetch-retries 5
+RUN npm config set fetch-retry-mintimeout 15000
+RUN npm install --no-audit
 
 # React client build
 ENV NODE_OPTIONS="--max-old-space-size=2048"

Dockerfile.multi

@@ -1,3 +1,5 @@
+# v0.7.0
+
 # Build API, Client and Data Provider
 FROM node:20-alpine AS base
 
@@ -24,6 +26,8 @@ FROM data-provider-build AS api-build
 WORKDIR /app/api
 COPY api/package*.json ./
 COPY api/ ./
+# Copy helper scripts
+COPY config/ ./
 # Copy data-provider to API's node_modules
 RUN mkdir -p /app/api/node_modules/librechat-data-provider/
 RUN cp -R /app/packages/data-provider/* /app/api/node_modules/librechat-data-provider/

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 LibreChat
+Copyright (c) 2024 LibreChat
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,14 +1,14 @@
 <p align="center">
-  <a href="https://docs.librechat.ai">
+  <a href="https://librechat.ai">
     <img src="docs/assets/LibreChat.svg" height="256">
   </a>
-  <a href="https://docs.librechat.ai">
-    <h1 align="center">LibreChat</h1>
-  </a>
+  <h1 align="center">
+    <a href="https://librechat.ai">LibreChat</a>
+  </h1>
 </p>
 
 <p align="center">
-  <a href="https://discord.gg/NGaa9RPCft"> 
+  <a href="https://discord.librechat.ai"> 
     <img
       src="https://img.shields.io/discord/1086345563026489514?label=&logo=discord&style=for-the-badge&logoWidth=20&logoColor=white&labelColor=000000&color=blueviolet">
   </a>
@@ -26,31 +26,49 @@
   </a>
 </p>
 
-# Features
- - 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and 11-2023 updates
- - 💬 Multimodal Chat:
-     - Upload and analyze images with GPT-4 and Gemini Vision 📸
-     - More filetypes and Assistants API integration in Active Development 🚧 
- - 🌎 Multilingual UI:
-     - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
-     - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands
- - 🤖 AI model selection: OpenAI API, Azure, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins
- - 💾 Create, Save, & Share Custom Presets
- - 🔄 Edit, Resubmit, and Continue messages with conversation branching
- - 📤 Export conversations as screenshots, markdown, text, json.
- - 🔍 Search all messages/conversations
- - 🔌 Plugins, including web access, image generation with DALL-E-3 and more
- - 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
- - ⚙️ Configure Proxy, Reverse Proxy, Docker, many Deployment options, and completely Open-Source
+<p align="center">
+<a href="https://railway.app/template/b5k2mn?referralCode=HI9hWz">
+  <img src="https://railway.app/button.svg" alt="Deploy on Railway" height="30">
+</a>
+<a href="https://zeabur.com/templates/0X2ZY8">
+  <img src="https://zeabur.com/button.svg" alt="Deploy on Zeabur" height="30"/>
+</a>
+<a href="https://template.cloud.sealos.io/deploy?templateName=librechat">
+  <img src="https://raw.githubusercontent.com/labring-actions/templates/main/Deploy-on-Sealos.svg" alt="Deploy on Sealos" height="30">
+</a>
+</p>
+
+# 📃 Features
+
+- 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and latest updates
+- 💬 Multimodal Chat:
+    - Upload and analyze images with Claude 3, GPT-4, and Gemini Vision 📸
+    - Chat with Files using Custom Endpoints, OpenAI, Azure, Anthropic, & Google. 🗃️
+    - Advanced Agents with Files, Code Interpreter, Tools, and API Actions 🔦
+      - Available through the [OpenAI Assistants API](https://platform.openai.com/docs/assistants/overview) 🌤️
+      - Non-OpenAI Agents in Active Development 🚧
+- 🌎 Multilingual UI:
+  - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
+  - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands, עברית
+- 🤖 AI model selection: OpenAI, Azure OpenAI, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins, Assistants API (including Azure Assistants)
+- 💾 Create, Save, & Share Custom Presets
+- 🔄 Edit, Resubmit, and Continue messages with conversation branching
+- 📤 Export conversations as screenshots, markdown, text, json.
+- 🔍 Search all messages/conversations
+- 🔌 Plugins, including web access, image generation with DALL-E-3 and more
+- 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
+- ⚙️ Configure Proxy, Reverse Proxy, Docker, & many Deployment options
+- 📖 Completely Open-Source & Built in Public
+- 🧑‍🤝‍🧑 Community-driven development, support, and feedback
 
 [For a thorough review of our features, see our docs here](https://docs.librechat.ai/features/plugins/introduction.html) 📚
 
+## 🪶 All-In-One AI Conversations with LibreChat
 
-## All-In-One AI Conversations with LibreChat
 LibreChat brings together the future of assistant AIs with the revolutionary technology of OpenAI's ChatGPT. Celebrating the original styling, LibreChat gives you the ability to integrate multiple AI models. It also integrates and enhances original client features such as conversation and message search, prompt templates and plugins.
 
 With LibreChat, you no longer need to opt for ChatGPT Plus and can instead use free or pay-per-call APIs. We welcome contributions, cloning, and forking to enhance the capabilities of this advanced chatbot platform.
-  
+
 <!-- https://github.com/danny-avila/LibreChat/assets/110412045/c1eb0c0f-41f6-4335-b982-84b278b53d59 -->
 
 [![Watch the video](https://img.youtube.com/vi/pNIOs1ovsXw/maxresdefault.jpg)](https://youtu.be/pNIOs1ovsXw)
@@ -58,97 +76,26 @@ Click on the thumbnail to open the video☝️
 
 ---
 
-## ⚠️ [Breaking Changes](docs/general_info/breaking_changes.md) ⚠️
+## 📚 Documentation
 
-**Please read this before updating from a previous version**
+For more information on how to use our advanced features, install and configure our software, and access our guidelines and tutorials, please check out our documentation at [docs.librechat.ai](https://docs.librechat.ai)
 
 ---
 
-## Changelog 
+## 📝 Changelog
+
 Keep up with the latest updates by visiting the releases page - [Releases](https://github.com/danny-avila/LibreChat/releases)
 
----
-
-<h1>Table of Contents</h1>
-
-<details open>
-  <summary><strong>Getting Started</strong></summary>
-
-  * Installation
-    * [Docker Compose Install🐳](docs/install/docker_compose_install.md)
-    * [Linux Install🐧](docs/install/linux_install.md)
-    * [Mac Install🍎](docs/install/mac_install.md)
-    * [Windows Install💙](docs/install/windows_install.md)
-  * Configuration
-    * [.env Configuration](./docs/install/dotenv.md)
-    * [AI Setup](docs/install/ai_setup.md)
-    * [User Auth System](docs/install/user_auth_system.md)
-    * [Online MongoDB Database](docs/install/mongodb.md)
-    * [Default Language](docs/install/default_language.md)
-    * [LiteLLM Proxy: Load Balance LLMs + Spend Tracking](docs/install/litellm.md)
-</details>
-
-<details>
-  <summary><strong>General Information</strong></summary>
-
-  * [Code of Conduct](.github/CODE_OF_CONDUCT.md)
-  * [Project Origin](docs/general_info/project_origin.md)
-  * [Multilingual Information](docs/general_info/multilingual_information.md)
-  * [Tech Stack](docs/general_info/tech_stack.md)   
-</details>
-
-<details>
-  <summary><strong>Features</strong></summary>
-
-  * **Plugins**
-    * [Introduction](docs/features/plugins/introduction.md)
-    * [Google](docs/features/plugins/google_search.md)
-    * [Stable Diffusion](docs/features/plugins/stable_diffusion.md)
-    * [Wolfram](docs/features/plugins/wolfram.md)
-    * [Make Your Own Plugin](docs/features/plugins/make_your_own.md)
-    * [Using official ChatGPT Plugins](docs/features/plugins/chatgpt_plugins_openapi.md)
-
-  
-  * [Automated Moderation](docs/features/mod_system.md)
-  * [Token Usage](docs/features/token_usage.md)
-  * [Manage Your Database](docs/features/manage_your_database.md)
-  * [PandoraNext Deployment Guide](docs/features/pandoranext.md)
-  * [Third-Party Tools](docs/features/third_party.md)
-  * [Proxy](docs/features/proxy.md)
-  * [Bing Jailbreak](docs/features/bing_jailbreak.md)
-
-</details>
-
-<details>
-  <summary><strong>Cloud Deployment</strong></summary>
-
-  * [DigitalOcean](docs/deployment/digitalocean.md)
-  * [Azure](docs/deployment/azure-terraform.md)
-  * [Linode](docs/deployment/linode.md)
-  * [Cloudflare](docs/deployment/cloudflare.md)
-  * [Ngrok](docs/deployment/ngrok.md)
-  * [HuggingFace](docs/deployment/huggingface.md)
-  * [Render](docs/deployment/render.md)
-  * [Meilisearch in Render](docs/deployment/meilisearch_in_render.md)
-  * [Hetzner](docs/deployment/hetzner_ubuntu.md)
-  * [Heroku](docs/deployment/heroku.md)
-</details>
-
-<details>
-  <summary><strong>Contributions</strong></summary>
-  
-  * [Contributor Guidelines](.github/CONTRIBUTING.md)
-  * [Documentation Guidelines](docs/contributions/documentation_guidelines.md)
-  * [Contribute a Translation](docs/contributions/translation_contribution.md)
-  * [Code Standards and Conventions](docs/contributions/coding_conventions.md)
-  * [Testing](docs/contributions/testing.md)
-  * [Security](.github/SECURITY.md)
-  * [Project Roadmap](https://github.com/users/danny-avila/projects/2)
-</details>
+**⚠️ [Breaking Changes](docs/general_info/breaking_changes.md)**
+Please consult the breaking changes before updating.
 
 ---
 
-## Star History
+## ⭐ Star History
+
+<p align="center">
+<a href="https://trendshift.io/repositories/4685" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4685" alt="danny-avila%2FLibreChat | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</p>
 
 <a href="https://star-history.com/#danny-avila/LibreChat&Date">
   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
@@ -156,16 +103,16 @@ Keep up with the latest updates by visiting the releases page - [Releases](https
 
 ---
 
-## Contributors
-Contributions and suggestions bug reports and fixes are welcome!
-Please read the documentation before you do!
+## ✨ Contributions
 
-For new features, components, or extensions, please open an issue and discuss before sending a PR. 
+Contributions, suggestions, bug reports and fixes are welcome!
 
-- Join the [Discord community](https://discord.gg/uDyZ5Tzhct)
+For new features, components, or extensions, please open an issue and discuss before sending a PR.
 
-This project exists in its current state thanks to all the people who contribute
 ---
+
+## 💖 This project exists in its current state thanks to all the people who contribute
+
 <a href="https://github.com/danny-avila/LibreChat/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=danny-avila/LibreChat" />
 </a>

api/app/chatgpt-browser.js

@@ -1,5 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
+const { Constants } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('../server/services/UserService');
 
 const browserClient = async ({
@@ -48,7 +49,7 @@ const browserClient = async ({
     options = { ...options, parentMessageId, conversationId };
   }
 
-  if (parentMessageId === '00000000-0000-0000-0000-000000000000') {
+  if (parentMessageId === Constants.NO_PARENT) {
     delete options.conversationId;
   }
 

api/app/clients/AnthropicClient.js

@@ -1,6 +1,19 @@
 const Anthropic = require('@anthropic-ai/sdk');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
-const { getResponseSender, EModelEndpoint } = require('librechat-data-provider');
+const {
+  getResponseSender,
+  EModelEndpoint,
+  validateVisionModel,
+} = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images/encode');
+const {
+  titleFunctionPrompt,
+  parseTitleFromPrompt,
+  truncateText,
+  formatMessage,
+  createContextHandlers,
+} = require('./prompts');
+const spendTokens = require('~/models/spendTokens');
 const { getModelMaxTokens } = require('~/utils');
 const BaseClient = require('./BaseClient');
 const { logger } = require('~/config');
@@ -10,12 +23,20 @@ const AI_PROMPT = '\n\nAssistant:';
 
 const tokenizersCache = {};
 
+/** Helper function to introduce a delay before retrying */
+function delayBeforeRetry(attempts, baseDelay = 1000) {
+  return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts));
+}
+
 class AnthropicClient extends BaseClient {
   constructor(apiKey, options = {}) {
     super(apiKey, options);
     this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY;
     this.userLabel = HUMAN_PROMPT;
     this.assistantLabel = AI_PROMPT;
+    this.contextStrategy = options.contextStrategy
+      ? options.contextStrategy.toLowerCase()
+      : 'discard';
     this.setOptions(options);
   }
 
@@ -47,6 +68,12 @@ class AnthropicClient extends BaseClient {
       stop: modelOptions.stop, // no stop method for now
     };
 
+    this.isClaude3 = this.modelOptions.model.includes('claude-3');
+    this.useMessages = this.isClaude3 || !!this.options.attachments;
+
+    this.defaultVisionModel = this.options.visionModel ?? 'claude-3-sonnet-20240229';
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
+
     this.maxContextTokens =
       getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ?? 100000;
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500;
@@ -87,7 +114,12 @@ class AnthropicClient extends BaseClient {
     return this;
   }
 
+  /**
+   * Get the initialized Anthropic client.
+   * @returns {Anthropic} The Anthropic client instance.
+   */
   getClient() {
+    /** @type {Anthropic.default.RequestOptions} */
     const options = {
       apiKey: this.apiKey,
     };
@@ -99,6 +131,75 @@ class AnthropicClient extends BaseClient {
     return new Anthropic(options);
   }
 
+  getTokenCountForResponse(response) {
+    return this.getTokenCountForMessage({
+      role: 'assistant',
+      content: response.text,
+    });
+  }
+
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
+   * - Sets `this.isVisionModel` to `true` if vision request.
+   * - Deletes `this.modelOptions.stop` if vision request.
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.anthropic];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    const visionModelAvailable = availableModels?.includes(this.defaultVisionModel);
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      visionModelAvailable &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+  }
+
+  /**
+   * Calculate the token cost in tokens for an image based on its dimensions and detail level.
+   *
+   * For reference, see: https://docs.anthropic.com/claude/docs/vision#image-costs
+   *
+   * @param {Object} image - The image object.
+   * @param {number} image.width - The width of the image.
+   * @param {number} image.height - The height of the image.
+   * @returns {number} The calculated token cost measured by tokens.
+   *
+   */
+  calculateImageTokenCost({ width, height }) {
+    return Math.ceil((width * height) / 750);
+  }
+
+  async addImageURLs(message, attachments) {
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      EModelEndpoint.anthropic,
+    );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
+  async recordTokenUsage({ promptTokens, completionTokens, model, context = 'message' }) {
+    await spendTokens(
+      {
+        context,
+        user: this.user,
+        conversationId: this.conversationId,
+        model: model ?? this.modelOptions.model,
+        endpointTokenConfig: this.options.endpointTokenConfig,
+      },
+      { promptTokens, completionTokens },
+    );
+  }
+
   async buildMessages(messages, parentMessageId) {
     const orderedMessages = this.constructor.getMessagesForConversation({
       messages,
@@ -107,28 +208,145 @@ class AnthropicClient extends BaseClient {
 
     logger.debug('[AnthropicClient] orderedMessages', { orderedMessages, parentMessageId });
 
-    const formattedMessages = orderedMessages.map((message) => ({
-      author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
-      content: message?.content ?? message.text,
-    }));
+    if (this.options.attachments) {
+      const attachments = await this.options.attachments;
+      const images = attachments.filter((file) => file.type.includes('image'));
+
+      if (images.length && !this.isVisionModel) {
+        throw new Error('Images are only supported with the Claude 3 family of models');
+      }
+
+      const latestMessage = orderedMessages[orderedMessages.length - 1];
+
+      if (this.message_file_map) {
+        this.message_file_map[latestMessage.messageId] = attachments;
+      } else {
+        this.message_file_map = {
+          [latestMessage.messageId]: attachments,
+        };
+      }
+
+      const files = await this.addImageURLs(latestMessage, attachments);
+
+      this.options.attachments = files;
+    }
+
+    if (this.message_file_map) {
+      this.contextHandlers = createContextHandlers(
+        this.options.req,
+        orderedMessages[orderedMessages.length - 1].text,
+      );
+    }
+
+    const formattedMessages = orderedMessages.map((message, i) => {
+      const formattedMessage = this.useMessages
+        ? formatMessage({
+          message,
+          endpoint: EModelEndpoint.anthropic,
+        })
+        : {
+          author: message.isCreatedByUser ? this.userLabel : this.assistantLabel,
+          content: message?.content ?? message.text,
+        };
+
+      const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+      /* If tokens were never counted, or, is a Vision request and the message has files, count again */
+      if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {
+        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
+      }
+
+      /* If message has files, calculate image token cost */
+      if (this.message_file_map && this.message_file_map[message.messageId]) {
+        const attachments = this.message_file_map[message.messageId];
+        for (const file of attachments) {
+          if (file.embedded) {
+            this.contextHandlers?.processFile(file);
+            continue;
+          }
+
+          orderedMessages[i].tokenCount += this.calculateImageTokenCost({
+            width: file.width,
+            height: file.height,
+          });
+        }
+      }
+
+      formattedMessage.tokenCount = orderedMessages[i].tokenCount;
+      return formattedMessage;
+    });
+
+    if (this.contextHandlers) {
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + (this.options.promptPrefix ?? '');
+    }
+
+    let { context: messagesInWindow, remainingContextTokens } =
+      await this.getMessagesWithinTokenLimit(formattedMessages);
+
+    const tokenCountMap = orderedMessages
+      .slice(orderedMessages.length - messagesInWindow.length)
+      .reduce((map, message, index) => {
+        const { messageId } = message;
+        if (!messageId) {
+          return map;
+        }
+
+        map[messageId] = orderedMessages[index].tokenCount;
+        return map;
+      }, {});
+
+    logger.debug('[AnthropicClient]', {
+      messagesInWindow: messagesInWindow.length,
+      remainingContextTokens,
+    });
 
     let lastAuthor = '';
     let groupedMessages = [];
 
-    for (let message of formattedMessages) {
+    for (let i = 0; i < messagesInWindow.length; i++) {
+      const message = messagesInWindow[i];
+      const author = message.role ?? message.author;
       // If last author is not same as current author, add to new group
-      if (lastAuthor !== message.author) {
-        groupedMessages.push({
-          author: message.author,
+      if (lastAuthor !== author) {
+        const newMessage = {
           content: [message.content],
-        });
-        lastAuthor = message.author;
+        };
+
+        if (message.role) {
+          newMessage.role = message.role;
+        } else {
+          newMessage.author = message.author;
+        }
+
+        groupedMessages.push(newMessage);
+        lastAuthor = author;
         // If same author, append content to the last group
       } else {
         groupedMessages[groupedMessages.length - 1].content.push(message.content);
       }
     }
 
+    groupedMessages = groupedMessages.map((msg, i) => {
+      const isLast = i === groupedMessages.length - 1;
+      if (msg.content.length === 1) {
+        const content = msg.content[0];
+        return {
+          ...msg,
+          // reason: final assistant content cannot end with trailing whitespace
+          content:
+            isLast && this.useMessages && msg.role === 'assistant' && typeof content === 'string'
+              ? content?.trim()
+              : content,
+        };
+      }
+
+      if (!this.useMessages && msg.tokenCount) {
+        delete msg.tokenCount;
+      }
+
+      return msg;
+    });
+
     let identityPrefix = '';
     if (this.options.userLabel) {
       identityPrefix = `\nHuman's name: ${this.options.userLabel}`;
@@ -154,9 +372,10 @@ class AnthropicClient extends BaseClient {
     // Prompt AI to respond, empty if last message was from AI
     let isEdited = lastAuthor === this.assistantLabel;
     const promptSuffix = isEdited ? '' : `${promptPrefix}${this.assistantLabel}\n`;
-    let currentTokenCount = isEdited
-      ? this.getTokenCount(promptPrefix)
-      : this.getTokenCount(promptSuffix);
+    let currentTokenCount =
+      isEdited || this.useMessages
+        ? this.getTokenCount(promptPrefix)
+        : this.getTokenCount(promptSuffix);
 
     let promptBody = '';
     const maxTokenCount = this.maxPromptTokens;
@@ -224,7 +443,69 @@ class AnthropicClient extends BaseClient {
       return true;
     };
 
-    await buildPromptBody();
+    const messagesPayload = [];
+    const buildMessagesPayload = async () => {
+      let canContinue = true;
+
+      if (promptPrefix) {
+        this.systemMessage = promptPrefix;
+      }
+
+      while (currentTokenCount < maxTokenCount && groupedMessages.length > 0 && canContinue) {
+        const message = groupedMessages.pop();
+
+        let tokenCountForMessage = message.tokenCount ?? this.getTokenCountForMessage(message);
+
+        const newTokenCount = currentTokenCount + tokenCountForMessage;
+        const exceededMaxCount = newTokenCount > maxTokenCount;
+
+        if (exceededMaxCount && messagesPayload.length === 0) {
+          throw new Error(
+            `Prompt is too long. Max token count is ${maxTokenCount}, but prompt is ${newTokenCount} tokens long.`,
+          );
+        } else if (exceededMaxCount) {
+          canContinue = false;
+          break;
+        }
+
+        delete message.tokenCount;
+        messagesPayload.unshift(message);
+        currentTokenCount = newTokenCount;
+
+        // Switch off isEdited after using it once
+        if (isEdited && message.role === 'assistant') {
+          isEdited = false;
+        }
+
+        // Wait for next tick to avoid blocking the event loop
+        await new Promise((resolve) => setImmediate(resolve));
+      }
+    };
+
+    const processTokens = () => {
+      // Add 2 tokens for metadata after all messages have been counted.
+      currentTokenCount += 2;
+
+      // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
+      this.modelOptions.maxOutputTokens = Math.min(
+        this.maxContextTokens - currentTokenCount,
+        this.maxResponseTokens,
+      );
+    };
+
+    if (this.modelOptions.model.startsWith('claude-3')) {
+      await buildMessagesPayload();
+      processTokens();
+      return {
+        prompt: messagesPayload,
+        context: messagesInWindow,
+        promptTokens: currentTokenCount,
+        tokenCountMap,
+      };
+    } else {
+      await buildPromptBody();
+      processTokens();
+    }
 
     if (nextMessage.remove) {
       promptBody = promptBody.replace(nextMessage.messageString, '');
@@ -234,22 +515,26 @@ class AnthropicClient extends BaseClient {
 
     let prompt = `${promptBody}${promptSuffix}`;
 
-    // Add 2 tokens for metadata after all messages have been counted.
-    currentTokenCount += 2;
-
-    // Use up to `this.maxContextTokens` tokens (prompt + response), but try to leave `this.maxTokens` tokens for the response.
-    this.modelOptions.maxOutputTokens = Math.min(
-      this.maxContextTokens - currentTokenCount,
-      this.maxResponseTokens,
-    );
-
-    return { prompt, context };
+    return { prompt, context, promptTokens: currentTokenCount, tokenCountMap };
   }
 
   getCompletion() {
     logger.debug('AnthropicClient doesn\'t use getCompletion (all handled in sendCompletion)');
   }
 
+  /**
+   * Creates a message or completion response using the Anthropic client.
+   * @param {Anthropic} client - The Anthropic client instance.
+   * @param {Anthropic.default.MessageCreateParams | Anthropic.default.CompletionCreateParams} options - The options for the message or completion.
+   * @param {boolean} useMessages - Whether to use messages or completions. Defaults to `this.useMessages`.
+   * @returns {Promise<Anthropic.default.Message | Anthropic.default.Completion>} The response from the Anthropic client.
+   */
+  async createResponse(client, options, useMessages) {
+    return useMessages ?? this.useMessages
+      ? await client.messages.create(options)
+      : await client.completions.create(options);
+  }
+
   async sendCompletion(payload, { onProgress, abortController }) {
     if (!abortController) {
       abortController = new AbortController();
@@ -279,36 +564,88 @@ class AnthropicClient extends BaseClient {
       topP: top_p,
       topK: top_k,
     } = this.modelOptions;
+
     const requestOptions = {
-      prompt: payload,
       model,
       stream: stream || true,
-      max_tokens_to_sample: maxOutputTokens || 1500,
       stop_sequences,
       temperature,
       metadata,
       top_p,
       top_k,
     };
-    logger.debug('[AnthropicClient]', { ...requestOptions });
-    const response = await client.completions.create(requestOptions);
 
-    signal.addEventListener('abort', () => {
-      logger.debug('[AnthropicClient] message aborted!');
-      response.controller.abort();
-    });
-
-    for await (const completion of response) {
-      // Uncomment to debug message stream
-      // logger.debug(completion);
-      text += completion.completion;
-      onProgress(completion.completion);
+    if (this.useMessages) {
+      requestOptions.messages = payload;
+      requestOptions.max_tokens = maxOutputTokens || 1500;
+    } else {
+      requestOptions.prompt = payload;
+      requestOptions.max_tokens_to_sample = maxOutputTokens || 1500;
     }
 
-    signal.removeEventListener('abort', () => {
-      logger.debug('[AnthropicClient] message aborted!');
-      response.controller.abort();
-    });
+    if (this.systemMessage) {
+      requestOptions.system = this.systemMessage;
+    }
+
+    logger.debug('[AnthropicClient]', { ...requestOptions });
+
+    const handleChunk = (currentChunk) => {
+      if (currentChunk) {
+        text += currentChunk;
+        onProgress(currentChunk);
+      }
+    };
+
+    const maxRetries = 3;
+    async function processResponse() {
+      let attempts = 0;
+
+      while (attempts < maxRetries) {
+        let response;
+        try {
+          response = await this.createResponse(client, requestOptions);
+
+          signal.addEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+
+          for await (const completion of response) {
+            // Handle each completion as before
+            if (completion?.delta?.text) {
+              handleChunk(completion.delta.text);
+            } else if (completion.completion) {
+              handleChunk(completion.completion);
+            }
+          }
+
+          // Successful processing, exit loop
+          break;
+        } catch (error) {
+          attempts += 1;
+          logger.warn(
+            `User: ${this.user} | Anthropic Request ${attempts} failed: ${error.message}`,
+          );
+
+          if (attempts < maxRetries) {
+            await delayBeforeRetry(attempts, 350);
+          } else {
+            throw new Error(`Operation failed after ${maxRetries} attempts: ${error.message}`);
+          }
+        } finally {
+          signal.removeEventListener('abort', () => {
+            logger.debug('[AnthropicClient] message aborted!');
+            if (response.controller?.abort) {
+              response.controller.abort();
+            }
+          });
+        }
+      }
+    }
+
+    await processResponse.bind(this)();
 
     return text.trim();
   }
@@ -317,6 +654,7 @@ class AnthropicClient extends BaseClient {
     return {
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
+      resendFiles: this.options.resendFiles,
       ...this.modelOptions,
     };
   }
@@ -342,6 +680,78 @@ class AnthropicClient extends BaseClient {
   getTokenCount(text) {
     return this.gptEncoder.encode(text, 'all').length;
   }
+
+  /**
+   * Generates a concise title for a conversation based on the user's input text and response.
+   * Involves sending a chat completion request with specific instructions for title generation.
+   *
+   * This function capitlizes on [Anthropic's function calling training](https://docs.anthropic.com/claude/docs/functions-external-tools).
+   *
+   * @param {Object} params - The parameters for the conversation title generation.
+   * @param {string} params.text - The user's input.
+   * @param {string} [params.responseText=''] - The AI's immediate response to the user.
+   *
+   * @returns {Promise<string | 'New Chat'>} A promise that resolves to the generated conversation title.
+   *                            In case of failure, it will return the default title, "New Chat".
+   */
+  async titleConvo({ text, responseText = '' }) {
+    let title = 'New Chat';
+    const convo = `<initial_message>
+  ${truncateText(text)}
+  </initial_message>
+  <response>
+  ${JSON.stringify(truncateText(responseText))}
+  </response>`;
+
+    const { ANTHROPIC_TITLE_MODEL } = process.env ?? {};
+    const model = this.options.titleModel ?? ANTHROPIC_TITLE_MODEL ?? 'claude-3-haiku-20240307';
+    const system = titleFunctionPrompt;
+
+    const titleChatCompletion = async () => {
+      const content = `<conversation_context>
+  ${convo}
+  </conversation_context>
+  
+  Please generate a title for this conversation.`;
+
+      const titleMessage = { role: 'user', content };
+      const requestOptions = {
+        model,
+        temperature: 0.3,
+        max_tokens: 1024,
+        system,
+        stop_sequences: ['\n\nHuman:', '\n\nAssistant', '</function_calls>'],
+        messages: [titleMessage],
+      };
+
+      try {
+        const response = await this.createResponse(this.getClient(), requestOptions, true);
+        let promptTokens = response?.usage?.input_tokens;
+        let completionTokens = response?.usage?.output_tokens;
+        if (!promptTokens) {
+          promptTokens = this.getTokenCountForMessage(titleMessage);
+          promptTokens += this.getTokenCountForMessage({ role: 'system', content: system });
+        }
+        if (!completionTokens) {
+          completionTokens = this.getTokenCountForMessage(response.content[0]);
+        }
+        await this.recordTokenUsage({
+          model,
+          promptTokens,
+          completionTokens,
+          context: 'title',
+        });
+        const text = response.content[0].text;
+        title = parseTitleFromPrompt(text);
+      } catch (e) {
+        logger.error('[AnthropicClient] There was an issue generating the title', e);
+      }
+    };
+
+    await titleChatCompletion();
+    logger.debug('[AnthropicClient] Convo Title: ' + title);
+    return title;
+  }
 }
 
 module.exports = AnthropicClient;

api/app/clients/BaseClient.js

@@ -1,8 +1,9 @@
 const crypto = require('crypto');
-const { supportsBalanceCheck } = require('librechat-data-provider');
+const { supportsBalanceCheck, Constants } = require('librechat-data-provider');
 const { getConvo, getMessages, saveMessage, updateMessage, saveConvo } = require('~/models');
 const { addSpaceIfNeeded, isEnabled } = require('~/server/utils');
 const checkBalance = require('~/models/checkBalance');
+const { getFiles } = require('~/models/File');
 const TextStream = require('./TextStream');
 const { logger } = require('~/config');
 
@@ -73,7 +74,7 @@ class BaseClient {
     const saveOptions = this.getSaveOptions();
     this.abortController = opts.abortController ?? new AbortController();
     const conversationId = opts.conversationId ?? crypto.randomUUID();
-    const parentMessageId = opts.parentMessageId ?? '00000000-0000-0000-0000-000000000000';
+    const parentMessageId = opts.parentMessageId ?? Constants.NO_PARENT;
     const userMessageId = opts.overrideParentMessageId ?? crypto.randomUUID();
     let responseMessageId = opts.responseMessageId ?? crypto.randomUUID();
     let head = isEdited ? responseMessageId : parentMessageId;
@@ -424,7 +425,10 @@ class BaseClient {
       await this.saveMessageToDatabase(userMessage, saveOptions, user);
     }
 
-    if (isEnabled(process.env.CHECK_BALANCE) && supportsBalanceCheck[this.options.endpoint]) {
+    if (
+      isEnabled(process.env.CHECK_BALANCE) &&
+      supportsBalanceCheck[this.options.endpointType ?? this.options.endpoint]
+    ) {
       await checkBalance({
         req: this.options.req,
         res: this.options.res,
@@ -434,11 +438,14 @@ class BaseClient {
           amount: promptTokens,
           model: this.modelOptions.model,
           endpoint: this.options.endpoint,
+          endpointTokenConfig: this.options.endpointTokenConfig,
         },
       });
     }
 
     const completion = await this.sendCompletion(payload, opts);
+    this.abortController.requestCompleted = true;
+
     const responseMessage = {
       messageId: responseMessageId,
       conversationId,
@@ -449,6 +456,7 @@ class BaseClient {
       sender: this.sender,
       text: addSpaceIfNeeded(generation) + completion,
       promptTokens,
+      ...(this.metadata ?? {}),
     };
 
     if (
@@ -484,20 +492,22 @@ class BaseClient {
       mapMethod = this.getMessageMapMethod();
     }
 
-    const orderedMessages = this.constructor.getMessagesForConversation({
+    let _messages = this.constructor.getMessagesForConversation({
       messages,
       parentMessageId,
       mapMethod,
     });
 
+    _messages = await this.addPreviousAttachments(_messages);
+
     if (!this.shouldSummarize) {
-      return orderedMessages;
+      return _messages;
     }
 
     // Find the latest message with a 'summary' property
-    for (let i = orderedMessages.length - 1; i >= 0; i--) {
-      if (orderedMessages[i]?.summary) {
-        this.previous_summary = orderedMessages[i];
+    for (let i = _messages.length - 1; i >= 0; i--) {
+      if (_messages[i]?.summary) {
+        this.previous_summary = _messages[i];
         break;
       }
     }
@@ -512,14 +522,15 @@ class BaseClient {
       });
     }
 
-    return orderedMessages;
+    return _messages;
   }
 
   async saveMessageToDatabase(message, endpointOptions, user = null) {
-    await saveMessage({ ...message, user, unfinished: false, cancelled: false });
+    await saveMessage({ ...message, endpoint: this.options.endpoint, user, unfinished: false });
     await saveConvo(user, {
       conversationId: message.conversationId,
       endpoint: this.options.endpoint,
+      endpointType: this.options.endpointType,
       ...endpointOptions,
     });
   }
@@ -541,7 +552,7 @@ class BaseClient {
    *
    * Each message object should have an 'id' or 'messageId' property and may have a 'parentMessageId' property.
    * The 'parentMessageId' is the ID of the message that the current message is a reply to.
-   * If 'parentMessageId' is not present, null, or is '00000000-0000-0000-0000-000000000000',
+   * If 'parentMessageId' is not present, null, or is Constants.NO_PARENT,
    * the message is considered a root message.
    *
    * @param {Object} options - The options for the function.
@@ -596,9 +607,7 @@ class BaseClient {
       }
 
       currentMessageId =
-        message.parentMessageId === '00000000-0000-0000-0000-000000000000'
-          ? null
-          : message.parentMessageId;
+        message.parentMessageId === Constants.NO_PARENT ? null : message.parentMessageId;
     }
 
     orderedMessages.reverse();
@@ -617,6 +626,11 @@ class BaseClient {
    * An additional 3 tokens need to be added for assistant label priming after all messages have been counted.
    * In our implementation, this is accounted for in the getMessagesWithinTokenLimit method.
    *
+   * The content parts example was adapted from the following example:
+   * https://github.com/openai/openai-cookbook/pull/881/files
+   *
+   * Note: image token calculation is to be done elsewhere where we have access to the image metadata
+   *
    * @param {Object} message
    */
   getTokenCountForMessage(message) {
@@ -630,11 +644,18 @@ class BaseClient {
     }
 
     const processValue = (value) => {
-      if (typeof value === 'object' && value !== null) {
-        for (let [nestedKey, nestedValue] of Object.entries(value)) {
-          if (nestedKey === 'image_url' || nestedValue === 'image_url') {
+      if (Array.isArray(value)) {
+        for (let item of value) {
+          if (!item || !item.type || item.type === 'image_url') {
             continue;
           }
+
+          const nestedValue = item[item.type];
+
+          if (!nestedValue) {
+            continue;
+          }
+
           processValue(nestedValue);
         }
       } else {
@@ -660,6 +681,54 @@ class BaseClient {
 
     return await this.sendCompletion(payload, opts);
   }
+
+  /**
+   *
+   * @param {TMessage[]} _messages
+   * @returns {Promise<TMessage[]>}
+   */
+  async addPreviousAttachments(_messages) {
+    if (!this.options.resendFiles) {
+      return _messages;
+    }
+
+    /**
+     *
+     * @param {TMessage} message
+     */
+    const processMessage = async (message) => {
+      if (!this.message_file_map) {
+        /** @type {Record<string, MongoFile[]> */
+        this.message_file_map = {};
+      }
+
+      const fileIds = message.files.map((file) => file.file_id);
+      const files = await getFiles({
+        file_id: { $in: fileIds },
+      });
+
+      await this.addImageURLs(message, files);
+
+      this.message_file_map[message.messageId] = files;
+      return message;
+    };
+
+    const promises = [];
+
+    for (const message of _messages) {
+      if (!message.files) {
+        promises.push(message);
+        continue;
+      }
+
+      promises.push(processMessage(message));
+    }
+
+    const messages = await Promise.all(promises);
+
+    this.checkVisionRequest(Object.values(this.message_file_map ?? {}).flat());
+    return messages;
+  }
 }
 
 module.exports = BaseClient;

api/app/clients/ChatGPTClient.js

@@ -1,9 +1,16 @@
-const crypto = require('crypto');
 const Keyv = require('keyv');
+const crypto = require('crypto');
+const {
+  EModelEndpoint,
+  resolveHeaders,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
 const { Agent, ProxyAgent } = require('undici');
 const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
+const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils');
 
 const CHATGPT_MODEL = 'gpt-3.5-turbo';
 const tokenizersCache = {};
@@ -144,7 +151,8 @@ class ChatGPTClient extends BaseClient {
     if (!abortController) {
       abortController = new AbortController();
     }
-    const modelOptions = { ...this.modelOptions };
+
+    let modelOptions = { ...this.modelOptions };
     if (typeof onProgress === 'function') {
       modelOptions.stream = true;
     }
@@ -159,56 +167,171 @@ class ChatGPTClient extends BaseClient {
     }
 
     const { debug } = this.options;
-    const url = this.completionsUrl;
+    let baseURL = this.completionsUrl;
     if (debug) {
       console.debug();
-      console.debug(url);
+      console.debug(baseURL);
       console.debug(modelOptions);
       console.debug();
     }
 
-    if (this.azure || this.options.azure) {
-      // Azure does not accept `model` in the body, so we need to remove it.
-      delete modelOptions.model;
-    }
-
     const opts = {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify(modelOptions),
       dispatcher: new Agent({
         bodyTimeout: 0,
         headersTimeout: 0,
       }),
     };
 
-    if (this.apiKey && this.options.azure) {
-      opts.headers['api-key'] = this.apiKey;
+    if (this.isVisionModel) {
+      modelOptions.max_tokens = 4000;
+    }
+
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const isAzure = this.azure || this.options.azure;
+    if (
+      (isAzure && this.isVisionModel && azureConfig) ||
+      (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    ) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+      opts.headers = resolveHeaders(headers);
+      this.langchainProxy = extractBaseURL(baseURL);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+      this.azure = !serverless && azureOptions;
+      this.azureEndpoint =
+        !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+    }
+
+    if (this.options.headers) {
+      opts.headers = { ...opts.headers, ...this.options.headers };
+    }
+
+    if (isAzure) {
+      // Azure does not accept `model` in the body, so we need to remove it.
+      delete modelOptions.model;
+
+      baseURL = this.langchainProxy
+        ? constructAzureURL({
+          baseURL: this.langchainProxy,
+          azureOptions: this.azure,
+        })
+        : this.azureEndpoint.split(/(?<!\/)\/(chat|completion)\//)[0];
+
+      if (this.options.forcePrompt) {
+        baseURL += '/completions';
+      } else {
+        baseURL += '/chat/completions';
+      }
+
+      opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
+      opts.headers = { ...opts.headers, 'api-key': this.apiKey };
     } else if (this.apiKey) {
       opts.headers.Authorization = `Bearer ${this.apiKey}`;
     }
 
+    if (process.env.OPENAI_ORGANIZATION) {
+      opts.headers['OpenAI-Organization'] = process.env.OPENAI_ORGANIZATION;
+    }
+
     if (this.useOpenRouter) {
       opts.headers['HTTP-Referer'] = 'https://librechat.ai';
       opts.headers['X-Title'] = 'LibreChat';
     }
 
-    if (this.options.headers) {
-      opts.headers = { ...opts.headers, ...this.options.headers };
-    }
-
     if (this.options.proxy) {
       opts.dispatcher = new ProxyAgent(this.options.proxy);
     }
 
+    /* hacky fixes for Mistral AI API:
+      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
+      - If there is only one message and it's a system message, change the role to user
+      */
+    if (baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+      const { messages } = modelOptions;
+
+      const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+      if (systemMessageIndex > 0) {
+        const [systemMessage] = messages.splice(systemMessageIndex, 1);
+        messages.unshift(systemMessage);
+      }
+
+      modelOptions.messages = messages;
+
+      if (messages.length === 1 && messages[0].role === 'system') {
+        modelOptions.messages[0].role = 'user';
+      }
+    }
+
+    if (this.options.addParams && typeof this.options.addParams === 'object') {
+      modelOptions = {
+        ...modelOptions,
+        ...this.options.addParams,
+      };
+      logger.debug('[ChatGPTClient] chatCompletion: added params', {
+        addParams: this.options.addParams,
+        modelOptions,
+      });
+    }
+
+    if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
+      this.options.dropParams.forEach((param) => {
+        delete modelOptions[param];
+      });
+      logger.debug('[ChatGPTClient] chatCompletion: dropped params', {
+        dropParams: this.options.dropParams,
+        modelOptions,
+      });
+    }
+
+    if (baseURL.includes('v1') && !baseURL.includes('/completions') && !this.isChatCompletion) {
+      baseURL = baseURL.split('v1')[0] + 'v1/completions';
+    } else if (
+      baseURL.includes('v1') &&
+      !baseURL.includes('/chat/completions') &&
+      this.isChatCompletion
+    ) {
+      baseURL = baseURL.split('v1')[0] + 'v1/chat/completions';
+    }
+
+    const BASE_URL = new URL(baseURL);
+    if (opts.defaultQuery) {
+      Object.entries(opts.defaultQuery).forEach(([key, value]) => {
+        BASE_URL.searchParams.append(key, value);
+      });
+      delete opts.defaultQuery;
+    }
+
+    const completionsURL = BASE_URL.toString();
+    opts.body = JSON.stringify(modelOptions);
+
     if (modelOptions.stream) {
       // eslint-disable-next-line no-async-promise-executor
       return new Promise(async (resolve, reject) => {
         try {
           let done = false;
-          await fetchEventSource(url, {
+          await fetchEventSource(completionsURL, {
             ...opts,
             signal: abortController.signal,
             async onopen(response) {
@@ -236,7 +359,6 @@ class ChatGPTClient extends BaseClient {
               // workaround for private API not sending [DONE] event
               if (!done) {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
               }
             },
@@ -249,14 +371,13 @@ class ChatGPTClient extends BaseClient {
             },
             onmessage(message) {
               if (debug) {
-                // console.debug(message);
+                console.debug(message);
               }
               if (!message.data || message.event === 'ping') {
                 return;
               }
               if (message.data === '[DONE]') {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
                 done = true;
                 return;
@@ -269,7 +390,7 @@ class ChatGPTClient extends BaseClient {
         }
       });
     }
-    const response = await fetch(url, {
+    const response = await fetch(completionsURL, {
       ...opts,
       signal: abortController.signal,
     });

api/app/clients/GoogleClient.js

@@ -4,16 +4,17 @@ const { GoogleVertexAI } = require('langchain/llms/googlevertexai');
 const { ChatGoogleGenerativeAI } = require('@langchain/google-genai');
 const { ChatGoogleVertexAI } = require('langchain/chat_models/googlevertexai');
 const { AIMessage, HumanMessage, SystemMessage } = require('langchain/schema');
-const { encodeAndFormat, validateVisionModel } = require('~/server/services/Files/images');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const {
+  validateVisionModel,
   getResponseSender,
-  EModelEndpoint,
   endpointSettings,
+  EModelEndpoint,
   AuthKeys,
 } = require('librechat-data-provider');
+const { encodeAndFormat } = require('~/server/services/Files/images');
+const { formatMessage, createContextHandlers } = require('./prompts');
 const { getModelMaxTokens } = require('~/utils');
-const { formatMessage } = require('./prompts');
 const BaseClient = require('./BaseClient');
 const { logger } = require('~/config');
 
@@ -123,18 +124,11 @@ class GoogleClient extends BaseClient {
       // stop: modelOptions.stop // no stop method for now
     };
 
-    if (this.options.attachments) {
-      this.modelOptions.model = 'gemini-pro-vision';
-    }
+    this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
 
     // TODO: as of 12/14/23, only gemini models are "Generative AI" models provided by Google
     this.isGenerativeModel = this.modelOptions.model.includes('gemini');
-    this.isVisionModel = validateVisionModel(this.modelOptions.model);
     const { isGenerativeModel } = this;
-    if (this.isVisionModel && !this.options.attachments) {
-      this.modelOptions.model = 'gemini-pro';
-      this.isVisionModel = false;
-    }
     this.isChatModel = !isGenerativeModel && this.modelOptions.model.includes('chat');
     const { isChatModel } = this;
     this.isTextModel =
@@ -219,6 +213,33 @@ class GoogleClient extends BaseClient {
     return this;
   }
 
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    /* Validation vision request */
+    this.defaultVisionModel = this.options.visionModel ?? 'gemini-pro-vision';
+    const availableModels = this.options.modelsConfig?.[EModelEndpoint.google];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      availableModels?.includes(this.defaultVisionModel) &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+
+    if (this.isVisionModel && !attachments) {
+      this.modelOptions.model = 'gemini-pro';
+      this.isVisionModel = false;
+    }
+  }
+
   formatMessages() {
     return ((message) => ({
       author: message?.author ?? (message.isCreatedByUser ? this.userLabel : this.modelLabel),
@@ -226,18 +247,45 @@ class GoogleClient extends BaseClient {
     })).bind(this);
   }
 
-  async buildVisionMessages(messages = [], parentMessageId) {
-    const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId);
-    const attachments = await this.options.attachments;
+  /**
+   *
+   * Adds image URLs to the message object and returns the files
+   *
+   * @param {TMessage[]} messages
+   * @param {MongoFile[]} files
+   * @returns {Promise<MongoFile[]>}
+   */
+  async addImageURLs(message, attachments) {
     const { files, image_urls } = await encodeAndFormat(
       this.options.req,
-      attachments.filter((file) => file.type.includes('image')),
+      attachments,
       EModelEndpoint.google,
     );
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
 
+  async buildVisionMessages(messages = [], parentMessageId) {
+    const attachments = await this.options.attachments;
     const latestMessage = { ...messages[messages.length - 1] };
+    this.contextHandlers = createContextHandlers(this.options.req, latestMessage.text);
+
+    if (this.contextHandlers) {
+      for (const file of attachments) {
+        if (file.embedded) {
+          this.contextHandlers?.processFile(file);
+          continue;
+        }
+      }
+
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      this.options.promptPrefix = this.augmentedPrompt + this.options.promptPrefix;
+    }
+
+    const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId);
+
+    const files = await this.addImageURLs(latestMessage, attachments);
 
-    latestMessage.image_urls = image_urls;
     this.options.attachments = files;
 
     latestMessage.text = prompt;
@@ -264,7 +312,7 @@ class GoogleClient extends BaseClient {
       );
     }
 
-    if (this.options.attachments) {
+    if (this.options.attachments && this.isGenerativeModel) {
       return this.buildVisionMessages(messages, parentMessageId);
     }
 

api/app/clients/OpenAIClient.js

@@ -1,15 +1,28 @@
 const OpenAI = require('openai');
 const { HttpsProxyAgent } = require('https-proxy-agent');
-const { getResponseSender, EModelEndpoint } = require('librechat-data-provider');
+const {
+  ImageDetail,
+  EModelEndpoint,
+  resolveHeaders,
+  ImageDetailCost,
+  getResponseSender,
+  validateVisionModel,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
-const { encodeAndFormat, validateVisionModel } = require('~/server/services/Files/images');
-const { getModelMaxTokens, genAzureChatCompletion, extractBaseURL } = require('~/utils');
-const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts');
+const {
+  extractBaseURL,
+  constructAzureURL,
+  getModelMaxTokens,
+  genAzureChatCompletion,
+} = require('~/utils');
+const { truncateText, formatMessage, createContextHandlers, CUT_OFF_PROMPT } = require('./prompts');
+const { encodeAndFormat } = require('~/server/services/Files/images/encode');
 const { handleOpenAIErrors } = require('./tools/util');
 const spendTokens = require('~/models/spendTokens');
 const { createLLM, RunManager } = require('./llm');
-const { isEnabled } = require('~/server/utils');
 const ChatGPTClient = require('./ChatGPTClient');
+const { isEnabled } = require('~/server/utils');
 const { summaryBuffer } = require('./memory');
 const { runTitleChain } = require('./chains');
 const { tokenSplit } = require('./document');
@@ -31,8 +44,10 @@ class OpenAIClient extends BaseClient {
       ? options.contextStrategy.toLowerCase()
       : 'discard';
     this.shouldSummarize = this.contextStrategy === 'summarize';
+    /** @type {AzureOptions} */
     this.azure = options.azure || false;
     this.setOptions(options);
+    this.metadata = {};
   }
 
   // TODO: PluginsClient calls this 3x, unneeded
@@ -76,15 +91,11 @@ class OpenAIClient extends BaseClient {
       };
     }
 
-    this.isVisionModel = validateVisionModel(this.modelOptions.model);
-
-    if (this.options.attachments && !this.isVisionModel) {
-      this.modelOptions.model = 'gpt-4-vision-preview';
-      this.isVisionModel = true;
-    }
-
-    if (this.isVisionModel) {
-      delete this.modelOptions.stop;
+    this.defaultVisionModel = this.options.visionModel ?? 'gpt-4-vision-preview';
+    if (typeof this.options.attachments?.then === 'function') {
+      this.options.attachments.then((attachments) => this.checkVisionRequest(attachments));
+    } else {
+      this.checkVisionRequest(this.options.attachments);
     }
 
     const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {};
@@ -94,15 +105,28 @@ class OpenAIClient extends BaseClient {
     }
 
     const { reverseProxyUrl: reverseProxy } = this.options;
+
+    if (
+      !this.useOpenRouter &&
+      reverseProxy &&
+      reverseProxy.includes('https://openrouter.ai/api/v1')
+    ) {
+      this.useOpenRouter = true;
+    }
+
     this.FORCE_PROMPT =
       isEnabled(OPENAI_FORCE_PROMPT) ||
       (reverseProxy && reverseProxy.includes('completions') && !reverseProxy.includes('chat'));
 
+    if (typeof this.options.forcePrompt === 'boolean') {
+      this.FORCE_PROMPT = this.options.forcePrompt;
+    }
+
     if (this.azure && process.env.AZURE_OPENAI_DEFAULT_MODEL) {
-      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model);
+      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model, this);
       this.modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL;
     } else if (this.azure) {
-      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model);
+      this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model, this);
     }
 
     const { model } = this.modelOptions;
@@ -120,7 +144,13 @@ class OpenAIClient extends BaseClient {
     const { isChatGptModel } = this;
     this.isUnofficialChatGptModel =
       model.startsWith('text-chat') || model.startsWith('text-davinci-002-render');
-    this.maxContextTokens = getModelMaxTokens(model) ?? 4095; // 1 less than maximum
+
+    this.maxContextTokens =
+      getModelMaxTokens(
+        model,
+        this.options.endpointType ?? this.options.endpoint,
+        this.options.endpointTokenConfig,
+      ) ?? 4095; // 1 less than maximum
 
     if (this.shouldSummarize) {
       this.maxContextTokens = Math.floor(this.maxContextTokens / 2);
@@ -146,8 +176,10 @@ class OpenAIClient extends BaseClient {
       this.options.sender ??
       getResponseSender({
         model: this.modelOptions.model,
-        endpoint: EModelEndpoint.openAI,
+        endpoint: this.options.endpoint,
+        endpointType: this.options.endpointType,
         chatGptLabel: this.options.chatGptLabel,
+        modelDisplayLabel: this.options.modelDisplayLabel,
       });
 
     this.userLabel = this.options.userLabel || 'User';
@@ -189,6 +221,34 @@ class OpenAIClient extends BaseClient {
     return this;
   }
 
+  /**
+   *
+   * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+   * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
+   * - Sets `this.isVisionModel` to `true` if vision request.
+   * - Deletes `this.modelOptions.stop` if vision request.
+   * @param {MongoFile[]} attachments
+   */
+  checkVisionRequest(attachments) {
+    const availableModels = this.options.modelsConfig?.[this.options.endpoint];
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
+
+    const visionModelAvailable = availableModels?.includes(this.defaultVisionModel);
+    if (
+      attachments &&
+      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
+      visionModelAvailable &&
+      !this.isVisionModel
+    ) {
+      this.modelOptions.model = this.defaultVisionModel;
+      this.isVisionModel = true;
+    }
+
+    if (this.isVisionModel) {
+      delete this.modelOptions.stop;
+    }
+  }
+
   setupTokens() {
     if (this.isChatCompletion) {
       this.startToken = '||>';
@@ -273,7 +333,11 @@ class OpenAIClient extends BaseClient {
     tokenizerCallsCount++;
   }
 
-  // Returns the token count of a given text. It also checks and resets the tokenizers if necessary.
+  /**
+   * Returns the token count of a given text. It also checks and resets the tokenizers if necessary.
+   * @param {string} text - The text to get the token count for.
+   * @returns {number} The token count of the given text.
+   */
   getTokenCount(text) {
     this.resetTokenizersIfNecessary();
     try {
@@ -286,10 +350,33 @@ class OpenAIClient extends BaseClient {
     }
   }
 
+  /**
+   * Calculate the token cost for an image based on its dimensions and detail level.
+   *
+   * @param {Object} image - The image object.
+   * @param {number} image.width - The width of the image.
+   * @param {number} image.height - The height of the image.
+   * @param {'low'|'high'|string|undefined} [image.detail] - The detail level ('low', 'high', or other).
+   * @returns {number} The calculated token cost.
+   */
+  calculateImageTokenCost({ width, height, detail }) {
+    if (detail === 'low') {
+      return ImageDetailCost.LOW;
+    }
+
+    // Calculate the number of 512px squares
+    const numSquares = Math.ceil(width / 512) * Math.ceil(height / 512);
+
+    // Default to high detail cost calculation
+    return numSquares * ImageDetailCost.HIGH + ImageDetailCost.ADDITIONAL;
+  }
+
   getSaveOptions() {
     return {
       chatGptLabel: this.options.chatGptLabel,
       promptPrefix: this.options.promptPrefix,
+      resendFiles: this.options.resendFiles,
+      imageDetail: this.options.imageDetail,
       ...this.modelOptions,
     };
   }
@@ -302,6 +389,20 @@ class OpenAIClient extends BaseClient {
     };
   }
 
+  /**
+   *
+   * Adds image URLs to the message object and returns the files
+   *
+   * @param {TMessage[]} messages
+   * @param {MongoFile[]} files
+   * @returns {Promise<MongoFile[]>}
+   */
+  async addImageURLs(message, attachments) {
+    const { files, image_urls } = await encodeAndFormat(this.options.req, attachments);
+    message.image_urls = image_urls.length ? image_urls : undefined;
+    return files;
+  }
+
   async buildMessages(
     messages,
     parentMessageId,
@@ -326,8 +427,74 @@ class OpenAIClient extends BaseClient {
     let promptTokens;
 
     promptPrefix = (promptPrefix || this.options.promptPrefix || '').trim();
+
+    if (this.options.attachments) {
+      const attachments = await this.options.attachments;
+
+      if (this.message_file_map) {
+        this.message_file_map[orderedMessages[orderedMessages.length - 1].messageId] = attachments;
+      } else {
+        this.message_file_map = {
+          [orderedMessages[orderedMessages.length - 1].messageId]: attachments,
+        };
+      }
+
+      const files = await this.addImageURLs(
+        orderedMessages[orderedMessages.length - 1],
+        attachments,
+      );
+
+      this.options.attachments = files;
+    }
+
+    if (this.message_file_map) {
+      this.contextHandlers = createContextHandlers(
+        this.options.req,
+        orderedMessages[orderedMessages.length - 1].text,
+      );
+    }
+
+    const formattedMessages = orderedMessages.map((message, i) => {
+      const formattedMessage = formatMessage({
+        message,
+        userName: this.options?.name,
+        assistantName: this.options?.chatGptLabel,
+      });
+
+      const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+
+      /* If tokens were never counted, or, is a Vision request and the message has files, count again */
+      if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {
+        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
+      }
+
+      /* If message has files, calculate image token cost */
+      if (this.message_file_map && this.message_file_map[message.messageId]) {
+        const attachments = this.message_file_map[message.messageId];
+        for (const file of attachments) {
+          if (file.embedded) {
+            this.contextHandlers?.processFile(file);
+            continue;
+          }
+
+          orderedMessages[i].tokenCount += this.calculateImageTokenCost({
+            width: file.width,
+            height: file.height,
+            detail: this.options.imageDetail ?? ImageDetail.auto,
+          });
+        }
+      }
+
+      return formattedMessage;
+    });
+
+    if (this.contextHandlers) {
+      this.augmentedPrompt = await this.contextHandlers.createContext();
+      promptPrefix = this.augmentedPrompt + promptPrefix;
+    }
+
     if (promptPrefix) {
-      promptPrefix = `Instructions:\n${promptPrefix}`;
+      promptPrefix = `Instructions:\n${promptPrefix.trim()}`;
       instructions = {
         role: 'system',
         name: 'instructions',
@@ -339,31 +506,6 @@ class OpenAIClient extends BaseClient {
       }
     }
 
-    if (this.options.attachments) {
-      const attachments = await this.options.attachments;
-      const { files, image_urls } = await encodeAndFormat(
-        this.options.req,
-        attachments.filter((file) => file.type.includes('image')),
-      );
-
-      orderedMessages[orderedMessages.length - 1].image_urls = image_urls;
-      this.options.attachments = files;
-    }
-
-    const formattedMessages = orderedMessages.map((message, i) => {
-      const formattedMessage = formatMessage({
-        message,
-        userName: this.options?.name,
-        assistantName: this.options?.chatGptLabel,
-      });
-
-      if (this.contextStrategy && !orderedMessages[i].tokenCount) {
-        orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
-      }
-
-      return formattedMessage;
-    });
-
     // TODO: need to handle interleaving instructions better
     if (this.contextStrategy) {
       ({ payload, tokenCountMap, promptTokens, messages } = await this.handleContextStrategy({
@@ -397,7 +539,7 @@ class OpenAIClient extends BaseClient {
     let streamResult = null;
     this.modelOptions.user = this.user;
     const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null;
-    const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion);
+    const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion || typeof Bun !== 'undefined');
     if (typeof opts.onProgress === 'function' && useOldMethod) {
       await this.getCompletion(
         payload,
@@ -434,10 +576,9 @@ class OpenAIClient extends BaseClient {
         },
         opts.abortController || new AbortController(),
       );
-    } else if (typeof opts.onProgress === 'function') {
+    } else if (typeof opts.onProgress === 'function' || this.options.useChatCompletion) {
       reply = await this.chatCompletion({
         payload,
-        clientOptions: opts,
         onProgress: opts.onProgress,
         abortController: opts.abortController,
       });
@@ -457,11 +598,11 @@ class OpenAIClient extends BaseClient {
       }
     }
 
-    if (streamResult && typeof opts.addMetadata === 'function') {
+    if (streamResult) {
       const { finish_reason } = streamResult.choices[0];
-      opts.addMetadata({ finish_reason });
+      this.metadata = { finish_reason };
     }
-    return reply.trim();
+    return (reply ?? '').trim();
   }
 
   initializeLLM({
@@ -475,6 +616,7 @@ class OpenAIClient extends BaseClient {
     context,
     tokenBuffer,
     initialMessageCount,
+    conversationId,
   }) {
     const modelOptions = {
       modelName: modelName ?? model,
@@ -504,6 +646,16 @@ class OpenAIClient extends BaseClient {
       };
     }
 
+    const { headers } = this.options;
+    if (headers && typeof headers === 'object' && !Array.isArray(headers)) {
+      configOptions.baseOptions = {
+        headers: resolveHeaders({
+          ...headers,
+          ...configOptions?.baseOptions?.headers,
+        }),
+      };
+    }
+
     if (this.options.proxy) {
       configOptions.httpAgent = new HttpsProxyAgent(this.options.proxy);
       configOptions.httpsAgent = new HttpsProxyAgent(this.options.proxy);
@@ -522,7 +674,7 @@ class OpenAIClient extends BaseClient {
       callbacks: runManager.createCallbacks({
         context,
         tokenBuffer,
-        conversationId: this.conversationId,
+        conversationId: this.conversationId ?? conversationId,
         initialMessageCount,
       }),
     });
@@ -530,7 +682,21 @@ class OpenAIClient extends BaseClient {
     return llm;
   }
 
-  async titleConvo({ text, responseText = '' }) {
+  /**
+   * Generates a concise title for a conversation based on the user's input text and response.
+   * Uses either specified method or starts with the OpenAI `functions` method (using LangChain).
+   * If the `functions` method fails, it falls back to the `completion` method,
+   * which involves sending a chat completion request with specific instructions for title generation.
+   *
+   * @param {Object} params - The parameters for the conversation title generation.
+   * @param {string} params.text - The user's input.
+   * @param {string} [params.conversationId] - The current conversationId, if not already defined on client initialization.
+   * @param {string} [params.responseText=''] - The AI's immediate response to the user.
+   *
+   * @returns {Promise<string | 'New Chat'>} A promise that resolves to the generated conversation title.
+   *                            In case of failure, it will return the default title, "New Chat".
+   */
+  async titleConvo({ text, conversationId, responseText = '' }) {
     let title = 'New Chat';
     const convo = `||>User:
 "${truncateText(text)}"
@@ -539,32 +705,58 @@ class OpenAIClient extends BaseClient {
 
     const { OPENAI_TITLE_MODEL } = process.env ?? {};
 
+    const model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+
     const modelOptions = {
-      model: OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo',
+      // TODO: remove the gpt fallback and make it specific to endpoint
+      model,
       temperature: 0.2,
       presence_penalty: 0,
       frequency_penalty: 0,
       max_tokens: 16,
     };
 
-    try {
-      this.abortController = new AbortController();
-      const llm = this.initializeLLM({ ...modelOptions, context: 'title', tokenBuffer: 150 });
-      title = await runTitleChain({ llm, text, convo, signal: this.abortController.signal });
-    } catch (e) {
-      if (e?.message?.toLowerCase()?.includes('abort')) {
-        logger.debug('[OpenAIClient] Aborted title generation');
-        return;
-      }
-      logger.error(
-        '[OpenAIClient] There was an issue generating title with LangChain, trying the old method...',
-        e,
-      );
-      modelOptions.model = OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const resetTitleOptions = !!(
+      (this.azure && azureConfig) ||
+      (azureConfig && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    );
+
+    if (resetTitleOptions) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+
+      this.options.headers = resolveHeaders(headers);
+      this.options.reverseProxyUrl = baseURL ?? null;
+      this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      this.options.forcePrompt = azureConfig.groupMap[groupName].forcePrompt;
+      this.azure = !serverless && azureOptions;
+    }
+
+    const titleChatCompletion = async () => {
+      modelOptions.model = model;
+
       if (this.azure) {
         modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL ?? modelOptions.model;
-        this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model);
+        this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this);
       }
+
       const instructionsPayload = [
         {
           role: 'system',
@@ -578,10 +770,43 @@ ${convo}
       ];
 
       try {
-        title = (await this.sendPayload(instructionsPayload, { modelOptions })).replaceAll('"', '');
+        title = (
+          await this.sendPayload(instructionsPayload, { modelOptions, useChatCompletion: true })
+        ).replaceAll('"', '');
       } catch (e) {
-        logger.error('[OpenAIClient] There was another issue generating the title', e);
+        logger.error(
+          '[OpenAIClient] There was an issue generating the title with the completion method',
+          e,
+        );
       }
+    };
+
+    if (this.options.titleMethod === 'completion') {
+      await titleChatCompletion();
+      logger.debug('[OpenAIClient] Convo Title: ' + title);
+      return title;
+    }
+
+    try {
+      this.abortController = new AbortController();
+      const llm = this.initializeLLM({
+        ...modelOptions,
+        conversationId,
+        context: 'title',
+        tokenBuffer: 150,
+      });
+      title = await runTitleChain({ llm, text, convo, signal: this.abortController.signal });
+    } catch (e) {
+      if (e?.message?.toLowerCase()?.includes('abort')) {
+        logger.debug('[OpenAIClient] Aborted title generation');
+        return;
+      }
+      logger.error(
+        '[OpenAIClient] There was an issue generating title with LangChain, trying completion method...',
+        e,
+      );
+
+      await titleChatCompletion();
     }
 
     logger.debug('[OpenAIClient] Convo Title: ' + title);
@@ -593,8 +818,16 @@ ${convo}
     let context = messagesToRefine;
     let prompt;
 
+    // TODO: remove the gpt fallback and make it specific to endpoint
     const { OPENAI_SUMMARY_MODEL = 'gpt-3.5-turbo' } = process.env ?? {};
-    const maxContextTokens = getModelMaxTokens(OPENAI_SUMMARY_MODEL) ?? 4095;
+    const model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL;
+    const maxContextTokens =
+      getModelMaxTokens(
+        model,
+        this.options.endpointType ?? this.options.endpoint,
+        this.options.endpointTokenConfig,
+      ) ?? 4095; // 1 less than maximum
+
     // 3 tokens for the assistant label, and 98 for the summarizer prompt (101)
     let promptBuffer = 101;
 
@@ -644,7 +877,7 @@ ${convo}
     logger.debug('[OpenAIClient] initialPromptTokens', initialPromptTokens);
 
     const llm = this.initializeLLM({
-      model: OPENAI_SUMMARY_MODEL,
+      model,
       temperature: 0.2,
       context: 'summary',
       tokenBuffer: initialPromptTokens,
@@ -692,13 +925,13 @@ ${convo}
   }
 
   async recordTokenUsage({ promptTokens, completionTokens }) {
-    logger.debug('[OpenAIClient] recordTokenUsage:', { promptTokens, completionTokens });
     await spendTokens(
       {
         user: this.user,
         model: this.modelOptions.model,
         context: 'message',
         conversationId: this.conversationId,
+        endpointTokenConfig: this.options.endpointTokenConfig,
       },
       { promptTokens, completionTokens },
     );
@@ -711,7 +944,7 @@ ${convo}
     });
   }
 
-  async chatCompletion({ payload, onProgress, clientOptions, abortController = null }) {
+  async chatCompletion({ payload, onProgress, abortController = null }) {
     let error = null;
     const errorCallback = (err) => (error = err);
     let intermediateReply = '';
@@ -719,27 +952,19 @@ ${convo}
       if (!abortController) {
         abortController = new AbortController();
       }
-      const modelOptions = { ...this.modelOptions };
+
+      let modelOptions = { ...this.modelOptions };
+
       if (typeof onProgress === 'function') {
         modelOptions.stream = true;
       }
       if (this.isChatCompletion) {
         modelOptions.messages = payload;
       } else {
-        // TODO: unreachable code. Need to implement completions call for non-chat models
         modelOptions.prompt = payload;
       }
 
       const baseURL = extractBaseURL(this.completionsUrl);
-      // let { messages: _msgsToLog, ...modelOptionsToLog } = modelOptions;
-      // if (modelOptionsToLog.messages) {
-      //   _msgsToLog = modelOptionsToLog.messages.map((msg) => {
-      //     let { content, ...rest } = msg;
-
-      //     if (content)
-      //     return { ...rest, content: truncateText(content) };
-      //   });
-      // }
       logger.debug('[OpenAIClient] chatCompletion', { baseURL, modelOptions });
       const opts = {
         baseURL,
@@ -764,21 +989,106 @@ ${convo}
         modelOptions.max_tokens = 4000;
       }
 
+      /** @type {TAzureConfig | undefined} */
+      const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+      if (
+        (this.azure && this.isVisionModel && azureConfig) ||
+        (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+      ) {
+        const { modelGroupMap, groupMap } = azureConfig;
+        const {
+          azureOptions,
+          baseURL,
+          headers = {},
+          serverless,
+        } = mapModelToAzureConfig({
+          modelName: modelOptions.model,
+          modelGroupMap,
+          groupMap,
+        });
+        opts.defaultHeaders = resolveHeaders(headers);
+        this.langchainProxy = extractBaseURL(baseURL);
+        this.apiKey = azureOptions.azureOpenAIApiKey;
+
+        const groupName = modelGroupMap[modelOptions.model].group;
+        this.options.addParams = azureConfig.groupMap[groupName].addParams;
+        this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+        // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+        this.azure = !serverless && azureOptions;
+        this.azureEndpoint =
+          !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+      }
+
       if (this.azure || this.options.azure) {
         // Azure does not accept `model` in the body, so we need to remove it.
         delete modelOptions.model;
 
-        opts.baseURL = this.azureEndpoint.split('/chat')[0];
+        opts.baseURL = this.langchainProxy
+          ? constructAzureURL({
+            baseURL: this.langchainProxy,
+            azureOptions: this.azure,
+          })
+          : this.azureEndpoint.split(/(?<!\/)\/(chat|completion)\//)[0];
+
         opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
         opts.defaultHeaders = { ...opts.defaultHeaders, 'api-key': this.apiKey };
       }
 
+      if (process.env.OPENAI_ORGANIZATION) {
+        opts.organization = process.env.OPENAI_ORGANIZATION;
+      }
+
       let chatCompletion;
+      /** @type {OpenAI} */
       const openai = new OpenAI({
         apiKey: this.apiKey,
         ...opts,
       });
 
+      /* hacky fixes for Mistral AI API:
+      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
+      - If there is only one message and it's a system message, change the role to user
+      */
+      if (opts.baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+        const { messages } = modelOptions;
+
+        const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+        if (systemMessageIndex > 0) {
+          const [systemMessage] = messages.splice(systemMessageIndex, 1);
+          messages.unshift(systemMessage);
+        }
+
+        modelOptions.messages = messages;
+
+        if (messages.length === 1 && messages[0].role === 'system') {
+          modelOptions.messages[0].role = 'user';
+        }
+      }
+
+      if (this.options.addParams && typeof this.options.addParams === 'object') {
+        modelOptions = {
+          ...modelOptions,
+          ...this.options.addParams,
+        };
+        logger.debug('[OpenAIClient] chatCompletion: added params', {
+          addParams: this.options.addParams,
+          modelOptions,
+        });
+      }
+
+      if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
+        this.options.dropParams.forEach((param) => {
+          delete modelOptions[param];
+        });
+        logger.debug('[OpenAIClient] chatCompletion: dropped params', {
+          dropParams: this.options.dropParams,
+          modelOptions,
+        });
+      }
+
       let UnexpectedRoleError = false;
       if (modelOptions.stream) {
         const stream = await openai.beta.chat.completions
@@ -792,6 +1102,16 @@ ${convo}
           .on('error', (err) => {
             handleOpenAIErrors(err, errorCallback, 'stream');
           })
+          .on('finalChatCompletion', (finalChatCompletion) => {
+            const finalMessage = finalChatCompletion?.choices?.[0]?.message;
+            if (finalMessage && finalMessage?.role !== 'assistant') {
+              finalChatCompletion.choices[0].message.role = 'assistant';
+            }
+
+            if (finalMessage && !finalMessage?.content?.trim()) {
+              finalChatCompletion.choices[0].message.content = intermediateReply;
+            }
+          })
           .on('finalMessage', (message) => {
             if (message?.role !== 'assistant') {
               stream.messages.push({ role: 'assistant', content: intermediateReply });
@@ -837,8 +1157,18 @@ ${convo}
       }
 
       const { message, finish_reason } = chatCompletion.choices[0];
-      if (chatCompletion && typeof clientOptions.addMetadata === 'function') {
-        clientOptions.addMetadata({ finish_reason });
+      if (chatCompletion) {
+        this.metadata = { finish_reason };
+      }
+
+      logger.debug('[OpenAIClient] chatCompletion response', chatCompletion);
+
+      if (!message?.content?.trim() && intermediateReply.length) {
+        logger.debug(
+          '[OpenAIClient] chatCompletion: using intermediateReply due to empty message.content',
+          { intermediateReply },
+        );
+        return intermediateReply;
       }
 
       return message.content;
@@ -847,19 +1177,21 @@ ${convo}
         err?.message?.includes('abort') ||
         (err instanceof OpenAI.APIError && err?.message?.includes('abort'))
       ) {
-        return '';
+        return intermediateReply;
       }
       if (
         err?.message?.includes(
           'OpenAI error: Invalid final message: OpenAI expects final message to include role=assistant',
         ) ||
+        err?.message?.includes(
+          'stream ended without producing a ChatCompletionMessage with role=assistant',
+        ) ||
         err?.message?.includes('The server had an error processing your request') ||
         err?.message?.includes('missing finish_reason') ||
         err?.message?.includes('missing role') ||
         (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason'))
       ) {
         logger.error('[OpenAIClient] Known OpenAI error:', err);
-        await abortController.abortCompletion();
         return intermediateReply;
       } else if (err instanceof OpenAI.APIError) {
         if (intermediateReply) {

api/app/clients/PluginsClient.js

@@ -3,6 +3,7 @@ const { CallbackManager } = require('langchain/callbacks');
 const { BufferMemory, ChatMessageHistory } = require('langchain/memory');
 const { initializeCustomAgent, initializeFunctionsAgent } = require('./agents');
 const { addImages, buildErrorInput, buildPromptPrefix } = require('./output_parsers');
+const { processFileURL } = require('~/server/services/Files/process');
 const { EModelEndpoint } = require('librechat-data-provider');
 const { formatLangChainMessages } = require('./prompts');
 const checkBalance = require('~/models/checkBalance');
@@ -30,10 +31,6 @@ class PluginsClient extends OpenAIClient {
 
     super.setOptions(options);
 
-    if (this.functionsAgent && this.agentOptions.model && !this.useOpenRouter) {
-      this.agentOptions.model = this.getFunctionModelName(this.agentOptions.model);
-    }
-
     this.isGpt3 = this.modelOptions?.model?.includes('gpt-3');
 
     if (this.options.reverseProxyUrl) {
@@ -112,7 +109,8 @@ class PluginsClient extends OpenAIClient {
         signal: this.abortController.signal,
         openAIApiKey: this.openAIApiKey,
         conversationId: this.conversationId,
-        debug: this.options?.debug,
+        fileStrategy: this.options.req.app.locals.fileStrategy,
+        processFileURL,
         message,
       },
     });

api/app/clients/llm/createLLM.js

@@ -1,38 +1,6 @@
 const { ChatOpenAI } = require('langchain/chat_models/openai');
-const { sanitizeModelName } = require('../../../utils');
-const { isEnabled } = require('../../../server/utils');
-
-/**
- * @typedef {Object} ModelOptions
- * @property {string} modelName - The name of the model.
- * @property {number} [temperature] - The temperature setting for the model.
- * @property {number} [presence_penalty] - The presence penalty setting.
- * @property {number} [frequency_penalty] - The frequency penalty setting.
- * @property {number} [max_tokens] - The maximum number of tokens to generate.
- */
-
-/**
- * @typedef {Object} ConfigOptions
- * @property {string} [basePath] - The base path for the API requests.
- * @property {Object} [baseOptions] - Base options for the API requests, including headers.
- * @property {Object} [httpAgent] - The HTTP agent for the request.
- * @property {Object} [httpsAgent] - The HTTPS agent for the request.
- */
-
-/**
- * @typedef {Object} Callbacks
- * @property {Function} [handleChatModelStart] - A callback function for handleChatModelStart
- * @property {Function} [handleLLMEnd] - A callback function for handleLLMEnd
- * @property {Function} [handleLLMError] - A callback function for handleLLMError
- */
-
-/**
- * @typedef {Object} AzureOptions
- * @property {string} [azureOpenAIApiKey] - The Azure OpenAI API key.
- * @property {string} [azureOpenAIApiInstanceName] - The Azure OpenAI API instance name.
- * @property {string} [azureOpenAIApiDeploymentName] - The Azure OpenAI API deployment name.
- * @property {string} [azureOpenAIApiVersion] - The Azure OpenAI API version.
- */
+const { sanitizeModelName, constructAzureURL } = require('~/utils');
+const { isEnabled } = require('~/server/utils');
 
 /**
  * Creates a new instance of a language model (LLM) for chat interactions.
@@ -68,6 +36,7 @@ function createLLM({
     apiKey: openAIApiKey,
   };
 
+  /**  @type {AzureOptions} */
   let azureOptions = {};
   if (azure) {
     const useModelName = isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME);
@@ -85,17 +54,24 @@ function createLLM({
     modelOptions.modelName = process.env.AZURE_OPENAI_DEFAULT_MODEL;
   }
 
-  // console.debug('createLLM: configOptions');
-  // console.debug(configOptions);
+  if (azure && configOptions.basePath) {
+    const azureURL = constructAzureURL({
+      baseURL: configOptions.basePath,
+      azureOptions,
+    });
+    azureOptions.azureOpenAIBasePath = azureURL.split(
+      `/${azureOptions.azureOpenAIApiDeploymentName}`,
+    )[0];
+  }
 
   return new ChatOpenAI(
     {
       streaming,
-      verbose: true,
       credentials,
       configuration,
       ...azureOptions,
       ...modelOptions,
+      ...credentials,
       callbacks,
     },
     configOptions,

api/app/clients/output_parsers/addImages.js

@@ -60,12 +60,10 @@ function addImages(intermediateSteps, responseMessage) {
     if (!observation || !observation.includes('![')) {
       return;
     }
-    const observedImagePath = observation.match(/\(\/images\/.*\.\w*\)/g);
+    const observedImagePath = observation.match(/!\[.*\]\([^)]*\)/g);
     if (observedImagePath && !responseMessage.text.includes(observedImagePath[0])) {
       responseMessage.text += '\n' + observation;
-      if (process.env.DEBUG_PLUGINS) {
-        logger.debug('[addImages] added image from intermediateSteps:', observation);
-      }
+      logger.debug('[addImages] added image from intermediateSteps:', observation);
     }
   });
 }

api/app/clients/prompts/createContextHandlers.js

@@ -0,0 +1,159 @@
+const axios = require('axios');
+const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
+
+const footer = `Use the context as your learned knowledge to better answer the user.
+
+In your response, remember to follow these guidelines:
+- If you don't know the answer, simply say that you don't know.
+- If you are unsure how to answer, ask for clarification.
+- Avoid mentioning that you obtained the information from the context.
+
+Answer appropriately in the user's language.
+`;
+
+function createContextHandlers(req, userMessageContent) {
+  if (!process.env.RAG_API_URL) {
+    return;
+  }
+
+  const queryPromises = [];
+  const processedFiles = [];
+  const processedIds = new Set();
+  const jwtToken = req.headers.authorization.split(' ')[1];
+  const useFullContext = isEnabled(process.env.RAG_USE_FULL_CONTEXT);
+
+  const query = async (file) => {
+    if (useFullContext) {
+      return axios.get(`${process.env.RAG_API_URL}/documents/${file.file_id}/context`, {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+        },
+      });
+    }
+
+    return axios.post(
+      `${process.env.RAG_API_URL}/query`,
+      {
+        file_id: file.file_id,
+        query: userMessageContent,
+        k: 4,
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${jwtToken}`,
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+  };
+
+  const processFile = async (file) => {
+    if (file.embedded && !processedIds.has(file.file_id)) {
+      try {
+        const promise = query(file);
+        queryPromises.push(promise);
+        processedFiles.push(file);
+        processedIds.add(file.file_id);
+      } catch (error) {
+        logger.error(`Error processing file ${file.filename}:`, error);
+      }
+    }
+  };
+
+  const createContext = async () => {
+    try {
+      if (!queryPromises.length || !processedFiles.length) {
+        return '';
+      }
+
+      const oneFile = processedFiles.length === 1;
+      const header = `The user has attached ${oneFile ? 'a' : processedFiles.length} file${
+        !oneFile ? 's' : ''
+      } to the conversation:`;
+
+      const files = `${
+        oneFile
+          ? ''
+          : `
+      <files>`
+      }${processedFiles
+        .map(
+          (file) => `
+              <file>
+                <filename>${file.filename}</filename>
+                <type>${file.type}</type>
+              </file>`,
+        )
+        .join('')}${
+        oneFile
+          ? ''
+          : `
+        </files>`
+      }`;
+
+      const resolvedQueries = await Promise.all(queryPromises);
+
+      const context = resolvedQueries
+        .map((queryResult, index) => {
+          const file = processedFiles[index];
+          let contextItems = queryResult.data;
+
+          const generateContext = (currentContext) =>
+            `
+          <file>
+            <filename>${file.filename}</filename>
+            <context>${currentContext}
+            </context>
+          </file>`;
+
+          if (useFullContext) {
+            return generateContext(`\n${contextItems}`);
+          }
+
+          contextItems = queryResult.data
+            .map((item) => {
+              const pageContent = item[0].page_content;
+              return `
+            <contextItem>
+              <![CDATA[${pageContent?.trim()}]]>
+            </contextItem>`;
+            })
+            .join('');
+
+          return generateContext(contextItems);
+        })
+        .join('');
+
+      if (useFullContext) {
+        const prompt = `${header}
+          ${context}
+          ${footer}`;
+
+        return prompt;
+      }
+
+      const prompt = `${header}
+        ${files}
+
+        A semantic search was executed with the user's message as the query, retrieving the following context inside <context></context> XML tags.
+
+        <context>${context}
+        </context>
+
+        ${footer}`;
+
+      return prompt;
+    } catch (error) {
+      logger.error('Error creating context:', error);
+      throw error;
+    }
+  };
+
+  return {
+    processFile,
+    createContext,
+  };
+}
+
+module.exports = createContextHandlers;

api/app/clients/prompts/createVisionPrompt.js

@@ -0,0 +1,34 @@
+/**
+ * Generates a prompt instructing the user to describe an image in detail, tailored to different types of visual content.
+ * @param {boolean} pluralized - Whether to pluralize the prompt for multiple images.
+ * @returns {string} - The generated vision prompt.
+ */
+const createVisionPrompt = (pluralized = false) => {
+  return `Please describe the image${
+    pluralized ? 's' : ''
+  } in detail, covering relevant aspects such as:
+
+  For photographs, illustrations, or artwork:
+  - The main subject(s) and their appearance, positioning, and actions
+  - The setting, background, and any notable objects or elements
+  - Colors, lighting, and overall mood or atmosphere
+  - Any interesting details, textures, or patterns
+  - The style, technique, or medium used (if discernible)
+  
+  For screenshots or images containing text:
+  - The content and purpose of the text
+  - The layout, formatting, and organization of the information
+  - Any notable visual elements, such as logos, icons, or graphics
+  - The overall context or message conveyed by the screenshot
+  
+  For graphs, charts, or data visualizations:
+  - The type of graph or chart (e.g., bar graph, line chart, pie chart)
+  - The variables being compared or analyzed
+  - Any trends, patterns, or outliers in the data
+  - The axis labels, scales, and units of measurement
+  - The title, legend, and any additional context provided
+  
+  Be as specific and descriptive as possible while maintaining clarity and concision.`;
+};
+
+module.exports = createVisionPrompt;

api/app/clients/prompts/formatMessages.js

@@ -1,3 +1,4 @@
+const { EModelEndpoint } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
 
 /**
@@ -7,10 +8,16 @@ const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
  * @param {Object} params.message - The message object to format.
  * @param {string} [params.message.role] - The role of the message sender (must be 'user').
  * @param {string} [params.message.content] - The text content of the message.
+ * @param {EModelEndpoint} [params.endpoint] - Identifier for specific endpoint handling
  * @param {Array<string>} [params.image_urls] - The image_urls to attach to the message.
  * @returns {(Object)} - The formatted message.
  */
-const formatVisionMessage = ({ message, image_urls }) => {
+const formatVisionMessage = ({ message, image_urls, endpoint }) => {
+  if (endpoint === EModelEndpoint.anthropic) {
+    message.content = [...image_urls, { type: 'text', text: message.content }];
+    return message;
+  }
+
   message.content = [{ type: 'text', text: message.content }, ...image_urls];
 
   return message;
@@ -29,10 +36,11 @@ const formatVisionMessage = ({ message, image_urls }) => {
  * @param {Array<string>} [params.message.image_urls] - The image_urls attached to the message for Vision API.
  * @param {string} [params.userName] - The name of the user.
  * @param {string} [params.assistantName] - The name of the assistant.
+ * @param {string} [params.endpoint] - Identifier for specific endpoint handling
  * @param {boolean} [params.langChain=false] - Whether to return a LangChain message object.
  * @returns {(Object|HumanMessage|AIMessage|SystemMessage)} - The formatted message.
  */
-const formatMessage = ({ message, userName, assistantName, langChain = false }) => {
+const formatMessage = ({ message, userName, assistantName, endpoint, langChain = false }) => {
   let { role: _role, _name, sender, text, content: _content, lc_id } = message;
   if (lc_id && lc_id[2] && !langChain) {
     const roleMapping = {
@@ -51,7 +59,11 @@ const formatMessage = ({ message, userName, assistantName, langChain = false })
 
   const { image_urls } = message;
   if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
-    return formatVisionMessage({ message: formattedMessage, image_urls: message.image_urls });
+    return formatVisionMessage({
+      message: formattedMessage,
+      image_urls: message.image_urls,
+      endpoint,
+    });
   }
 
   if (_name) {

api/app/clients/prompts/formatMessages.spec.js

@@ -1,5 +1,6 @@
-const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
+const { Constants } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
+const { formatMessage, formatLangChainMessages, formatFromLangChain } = require('./formatMessages');
 
 describe('formatMessage', () => {
   it('formats user message', () => {
@@ -54,7 +55,6 @@ describe('formatMessage', () => {
         _id: '6512cdfb92cbf69fea615331',
         messageId: 'b620bf73-c5c3-4a38-b724-76886aac24c4',
         __v: 0,
-        cancelled: false,
         conversationId: '5c23d24f-941f-4aab-85df-127b596c8aa5',
         createdAt: Date.now(),
         error: false,
@@ -62,7 +62,7 @@ describe('formatMessage', () => {
         isCreatedByUser: true,
         isEdited: false,
         model: null,
-        parentMessageId: '00000000-0000-0000-0000-000000000000',
+        parentMessageId: Constants.NO_PARENT,
         sender: 'User',
         text: 'hi',
         tokenCount: 5,

api/app/clients/prompts/index.js

@@ -4,6 +4,8 @@ const handleInputs = require('./handleInputs');
 const instructions = require('./instructions');
 const titlePrompts = require('./titlePrompts');
 const truncateText = require('./truncateText');
+const createVisionPrompt = require('./createVisionPrompt');
+const createContextHandlers = require('./createContextHandlers');
 
 module.exports = {
   ...formatMessages,
@@ -12,4 +14,6 @@ module.exports = {
   ...instructions,
   ...titlePrompts,
   truncateText,
+  createVisionPrompt,
+  createContextHandlers,
 };

api/app/clients/prompts/titlePrompts.js

@@ -27,7 +27,60 @@ ${convo}`,
   return titlePrompt;
 };
 
+const titleFunctionPrompt = `In this environment you have access to a set of tools you can use to generate the conversation title.
+  
+You may call them like this:
+<function_calls>
+<invoke>
+<tool_name>$TOOL_NAME</tool_name>
+<parameters>
+<$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME>
+...
+</parameters>
+</invoke>
+</function_calls>
+
+Here are the tools available:
+<tools>
+<tool_description>
+<tool_name>submit_title</tool_name>
+<description>
+Submit a brief title in the conversation's language, following the parameter description closely.
+</description>
+<parameters>
+<parameter>
+<name>title</name>
+<type>string</type>
+<description>A concise, 5-word-or-less title for the conversation, using its same language, with no punctuation. Apply title case conventions appropriate for the language. For English, use AP Stylebook Title Case. Never directly mention the language name or the word "title"</description>
+</parameter>
+</parameters>
+</tool_description>
+</tools>`;
+
+/**
+ * Parses titles from title functions based on the provided prompt.
+ * @param {string} prompt - The prompt containing the title function.
+ * @returns {string} The parsed title. "New Chat" if no title is found.
+ */
+function parseTitleFromPrompt(prompt) {
+  const titleRegex = /<title>(.+?)<\/title>/;
+  const titleMatch = prompt.match(titleRegex);
+
+  if (titleMatch && titleMatch[1]) {
+    const title = titleMatch[1].trim();
+
+    // // Capitalize the first letter of each word; Note: unnecessary due to title case prompting
+    // const capitalizedTitle = title.replace(/\b\w/g, (char) => char.toUpperCase());
+
+    return title;
+  }
+
+  return 'New Chat';
+}
+
 module.exports = {
   langPrompt,
   createTitlePrompt,
+  titleFunctionPrompt,
+  parseTitleFromPrompt,
 };

api/app/clients/specs/BaseClient.test.js

@@ -1,3 +1,4 @@
+const { Constants } = require('librechat-data-provider');
 const { initializeFakeClient } = require('./FakeClient');
 
 jest.mock('../../../lib/db/connectDb');
@@ -307,7 +308,7 @@ describe('BaseClient', () => {
     const unorderedMessages = [
       { id: '3', parentMessageId: '2', text: 'Message 3' },
       { id: '2', parentMessageId: '1', text: 'Message 2' },
-      { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+      { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
     ];
 
     it('should return ordered messages based on parentMessageId', () => {
@@ -316,7 +317,7 @@ describe('BaseClient', () => {
         parentMessageId: '3',
       });
       expect(result).toEqual([
-        { id: '1', parentMessageId: '00000000-0000-0000-0000-000000000000', text: 'Message 1' },
+        { id: '1', parentMessageId: Constants.NO_PARENT, text: 'Message 1' },
         { id: '2', parentMessageId: '1', text: 'Message 2' },
         { id: '3', parentMessageId: '2', text: 'Message 3' },
       ]);

api/app/clients/specs/OpenAIClient.test.js

@@ -546,6 +546,39 @@ describe('OpenAIClient', () => {
         expect(totalTokens).toBe(testCase.expected);
       });
     });
+
+    const vision_request = [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'describe what is in this image?',
+          },
+          {
+            type: 'image_url',
+            image_url: {
+              url: 'https://venturebeat.com/wp-content/uploads/2019/03/openai-1.png',
+              detail: 'high',
+            },
+          },
+        ],
+      },
+    ];
+
+    const expectedTokens = 14;
+    const visionModel = 'gpt-4-vision-preview';
+
+    it(`should return ${expectedTokens} tokens for model ${visionModel} (Vision Request)`, () => {
+      client.modelOptions.model = visionModel;
+      client.selectTokenizer();
+      // 3 tokens for assistant label
+      let totalTokens = 3;
+      for (let message of vision_request) {
+        totalTokens += client.getTokenCountForMessage(message);
+      }
+      expect(totalTokens).toBe(expectedTokens);
+    });
   });
 
   describe('sendMessage/getCompletion/chatCompletion', () => {

api/app/clients/specs/PluginsClient.test.js

@@ -1,9 +1,10 @@
+const crypto = require('crypto');
+const { Constants } = require('librechat-data-provider');
 const { HumanChatMessage, AIChatMessage } = require('langchain/schema');
 const PluginsClient = require('../PluginsClient');
-const crypto = require('crypto');
 
-jest.mock('../../../lib/db/connectDb');
-jest.mock('../../../models/Conversation', () => {
+jest.mock('~/lib/db/connectDb');
+jest.mock('~/models/Conversation', () => {
   return function () {
     return {
       save: jest.fn(),
@@ -12,6 +13,12 @@ jest.mock('../../../models/Conversation', () => {
   };
 });
 
+const defaultAzureOptions = {
+  azureOpenAIApiInstanceName: 'your-instance-name',
+  azureOpenAIApiDeploymentName: 'your-deployment-name',
+  azureOpenAIApiVersion: '2020-07-01-preview',
+};
+
 describe('PluginsClient', () => {
   let TestAgent;
   let options = {
@@ -60,7 +67,7 @@ describe('PluginsClient', () => {
         TestAgent.setOptions(opts);
       }
       const conversationId = opts.conversationId || crypto.randomUUID();
-      const parentMessageId = opts.parentMessageId || '00000000-0000-0000-0000-000000000000';
+      const parentMessageId = opts.parentMessageId || Constants.NO_PARENT;
       const userMessageId = opts.overrideParentMessageId || crypto.randomUUID();
       this.pastMessages = await TestAgent.loadHistory(
         conversationId,
@@ -187,4 +194,30 @@ describe('PluginsClient', () => {
       expect(client.getFunctionModelName('')).toBe('gpt-3.5-turbo');
     });
   });
+  describe('Azure OpenAI tests specific to Plugins', () => {
+    // TODO: add more tests for Azure OpenAI integration with Plugins
+    // let client;
+    // beforeEach(() => {
+    //   client = new PluginsClient('dummy_api_key');
+    // });
+
+    test('should not call getFunctionModelName when azure options are set', () => {
+      const spy = jest.spyOn(PluginsClient.prototype, 'getFunctionModelName');
+      const model = 'gpt-4-turbo';
+
+      // note, without the azure change in PR #1766, `getFunctionModelName` is called twice
+      const testClient = new PluginsClient('dummy_api_key', {
+        agentOptions: {
+          model,
+          agent: 'functions',
+        },
+        azure: defaultAzureOptions,
+      });
+
+      expect(spy).not.toHaveBeenCalled();
+      expect(testClient.agentOptions.model).toBe(model);
+
+      spy.mockRestore();
+    });
+  });
 });

api/app/clients/tools/AzureAiSearch.js

@@ -16,6 +16,9 @@ class AzureAISearch extends StructuredTool {
 
   constructor(fields = {}) {
     super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
 
     // Initialize properties using helper function
     this.serviceEndpoint = this._initializeField(
@@ -68,15 +71,6 @@ class AzureAISearch extends StructuredTool {
     });
   }
 
-  // Simplified getter methods
-  get name() {
-    return 'azure-ai-search';
-  }
-
-  get description() {
-    return 'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
-  }
-
   // Improved error handling and logging
   async _call(data) {
     const { query } = data;

api/app/clients/tools/DALL-E.js

@@ -1,53 +1,43 @@
-// From https://platform.openai.com/docs/api-reference/images/create
-// To use this tool, you must pass in a configured OpenAIApi object.
-const fs = require('fs');
 const path = require('path');
 const OpenAI = require('openai');
-// const { genAzureEndpoint } = require('~/utils/genAzureEndpoints');
+const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
+const { FileContext } = require('librechat-data-provider');
+const { getImageBasename } = require('~/server/services/Files/images');
 const extractBaseURL = require('~/utils/extractBaseURL');
-const saveImageFromUrl = require('./saveImageFromUrl');
 const { logger } = require('~/config');
 
-const { DALLE_REVERSE_PROXY, PROXY } = process.env;
 class OpenAICreateImage extends Tool {
   constructor(fields = {}) {
     super();
 
-    let apiKey = fields.DALLE_API_KEY || this.getApiKey();
+    this.userId = fields.userId;
+    this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
+    let apiKey = fields.DALLE2_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
+
     const config = { apiKey };
-    if (DALLE_REVERSE_PROXY) {
-      config.baseURL = extractBaseURL(DALLE_REVERSE_PROXY);
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
     }
 
-    if (PROXY) {
-      config.httpAgent = new HttpsProxyAgent(PROXY);
+    if (process.env.DALLE2_AZURE_API_VERSION && process.env.DALLE2_BASEURL) {
+      config.baseURL = process.env.DALLE2_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE2_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE2_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE2_API_KEY;
     }
 
-    // let azureKey = fields.AZURE_API_KEY || process.env.AZURE_API_KEY;
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
+    }
 
-    // if (azureKey) {
-    //   apiKey = azureKey;
-    //   const azureConfig = {
-    //     apiKey,
-    //     azureOpenAIApiInstanceName: process.env.AZURE_OPENAI_API_INSTANCE_NAME || fields.azureOpenAIApiInstanceName,
-    //     azureOpenAIApiDeploymentName: process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME || fields.azureOpenAIApiDeploymentName,
-    //     azureOpenAIApiVersion: process.env.AZURE_OPENAI_API_VERSION || fields.azureOpenAIApiVersion
-    //   };
-    //   config = {
-    //     apiKey,
-    //     basePath: genAzureEndpoint({
-    //       ...azureConfig,
-    //     }),
-    //     baseOptions: {
-    //       headers: { 'api-key': apiKey },
-    //       params: {
-    //         'api-version': azureConfig.azureOpenAIApiVersion // this might change. I got the current value from the sample code at https://oai.azure.com/portal/chat
-    //       }
-    //     }
-    //   };
-    // }
     this.openai = new OpenAI(config);
     this.name = 'dall-e';
     this.description = `You can generate images with 'dall-e'. This tool is exclusively for visual content.
@@ -57,10 +47,24 @@ Guidelines:
 - It's best to follow this format for image creation. Come up with the optional inputs yourself if none are given:
 "Subject: [subject], Style: [style], Color: [color], Details: [details], Emotion: [emotion]"
 - Generate images only once per human query unless explicitly requested by the user`;
+    this.description_for_model =
+      process.env.DALLE2_SYSTEM_PROMPT ??
+      `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
+// 1. Prompts must be in English. Translate to English if needed.
+// 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
+// 3. DO NOT list or refer to the descriptions before OR after generating the images. They should ONLY ever be written out ONCE, in the \`"prompts"\` field of the request. You do not need to ask for permission to generate, just do it!
+// 4. Always mention the image type (photo, oil painting, watercolor painting, illustration, cartoon, drawing, vector, render, etc.) at the beginning of the caption. Unless the captions suggests otherwise, make one of the images a photo.
+// 5. Diversify depictions of ALL images with people to always include always DESCENT and GENDER for EACH person using direct terms. Adjust only human descriptions.
+// - EXPLICITLY specify these attributes, not abstractly reference them. The attributes should be specified in a minimal way and should directly describe their physical form.
+// - Your choices should be grounded in reality. For example, all of a given OCCUPATION should not be the same gender or race. Additionally, focus on creating diverse, inclusive, and exploratory scenes via the properties you choose during rewrites.  Make choices that may be insightful or unique sometimes.
+// - Use "various" or "diverse" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
+// - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
+// The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
+// All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.`;
   }
 
   getApiKey() {
-    const apiKey = process.env.DALLE_API_KEY || '';
+    const apiKey = process.env.DALLE2_API_KEY ?? process.env.DALLE_API_KEY ?? '';
     if (!apiKey) {
       throw new Error('Missing DALLE_API_KEY environment variable.');
     }
@@ -74,22 +78,26 @@ Guidelines:
       .trim();
   }
 
-  getMarkdownImageUrl(imageName) {
-    const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
-      .replace(/\\/g, '/')
-      .replace('public/', '');
-    return `![generated image](/${imageUrl})`;
+  wrapInMarkdown(imageUrl) {
+    return `![generated image](${imageUrl})`;
   }
 
   async _call(input) {
-    const resp = await this.openai.images.generate({
-      prompt: this.replaceUnwantedChars(input),
-      // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
-      n: 1,
-      // size: '1024x1024'
-      size: '512x512',
-    });
+    let resp;
+
+    try {
+      resp = await this.openai.images.generate({
+        prompt: this.replaceUnwantedChars(input),
+        // TODO: Future idea -- could we ask an LLM to extract these arguments from an input that might contain them?
+        n: 1,
+        // size: '1024x1024'
+        size: '512x512',
+      });
+    } catch (error) {
+      logger.error('[DALL-E] Problem generating the image:', error);
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
+Error Message: ${error.message}`;
+    }
 
     const theImageUrl = resp.data[0].url;
 
@@ -97,35 +105,35 @@ Guidelines:
       throw new Error('No image URL returned from OpenAI API.');
     }
 
-    const regex = /img-[\w\d]+.png/;
-    const match = theImageUrl.match(regex);
-    let imageName = '1.png';
+    const imageBasename = getImageBasename(theImageUrl);
+    const imageExt = path.extname(imageBasename);
 
-    if (match) {
-      imageName = match[0];
-      logger.debug('[DALL-E]', { imageName }); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
-    } else {
-      logger.debug('[DALL-E] No image name found in the string.', {
-        theImageUrl,
-        data: resp.data[0],
-      });
-    }
+    const extension = imageExt.startsWith('.') ? imageExt.slice(1) : imageExt;
+    const imageName = `img-${uuidv4()}.${extension}`;
 
-    this.outputPath = path.resolve(__dirname, '..', '..', '..', '..', 'client', 'public', 'images');
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
-
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
-    }
+    logger.debug('[DALL-E-2]', {
+      imageName,
+      imageBasename,
+      imageExt,
+      extension,
+      theImageUrl,
+      data: resp.data[0],
+    });
 
     try {
-      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
-      this.result = this.getMarkdownImageUrl(imageName);
+      const result = await this.processFileURL({
+        fileStrategy: this.fileStrategy,
+        userId: this.userId,
+        URL: theImageUrl,
+        fileName: imageName,
+        basePath: 'images',
+        context: FileContext.image_generation,
+      });
+
+      this.result = this.wrapInMarkdown(result.filepath);
     } catch (error) {
-      logger.error('Error while saving the DALL-E image:', error);
-      this.result = theImageUrl;
+      logger.error('Error while saving the image:', error);
+      this.result = `Failed to save the image locally. ${error.message}`;
     }
 
     return this.result;

api/app/clients/tools/GoogleSearch.js

@@ -1,121 +0,0 @@
-const { google } = require('googleapis');
-const { Tool } = require('langchain/tools');
-const { logger } = require('~/config');
-
-/**
- * Represents a tool that allows an agent to use the Google Custom Search API.
- * @extends Tool
- */
-class GoogleSearchAPI extends Tool {
-  constructor(fields = {}) {
-    super();
-    this.cx = fields.GOOGLE_CSE_ID || this.getCx();
-    this.apiKey = fields.GOOGLE_API_KEY || this.getApiKey();
-    this.customSearch = undefined;
-  }
-
-  /**
-   * The name of the tool.
-   * @type {string}
-   */
-  name = 'google';
-
-  /**
-   * A description for the agent to use
-   * @type {string}
-   */
-  description =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-  description_for_model =
-    'Use the \'google\' tool to retrieve internet search results relevant to your input. The results will return links and snippets of text from the webpages';
-
-  getCx() {
-    const cx = process.env.GOOGLE_CSE_ID || '';
-    if (!cx) {
-      throw new Error('Missing GOOGLE_CSE_ID environment variable.');
-    }
-    return cx;
-  }
-
-  getApiKey() {
-    const apiKey = process.env.GOOGLE_API_KEY || '';
-    if (!apiKey) {
-      throw new Error('Missing GOOGLE_API_KEY environment variable.');
-    }
-    return apiKey;
-  }
-
-  getCustomSearch() {
-    if (!this.customSearch) {
-      const version = 'v1';
-      this.customSearch = google.customsearch(version);
-    }
-    return this.customSearch;
-  }
-
-  resultsToReadableFormat(results) {
-    let output = 'Results:\n';
-
-    results.forEach((resultObj, index) => {
-      output += `Title: ${resultObj.title}\n`;
-      output += `Link: ${resultObj.link}\n`;
-      if (resultObj.snippet) {
-        output += `Snippet: ${resultObj.snippet}\n`;
-      }
-
-      if (index < results.length - 1) {
-        output += '\n';
-      }
-    });
-
-    return output;
-  }
-
-  /**
-   * Calls the tool with the provided input and returns a promise that resolves with a response from the Google Custom Search API.
-   * @param {string} input - The input to provide to the API.
-   * @returns {Promise<String>} A promise that resolves with a response from the Google Custom Search API.
-   */
-  async _call(input) {
-    try {
-      const metadataResults = [];
-      const response = await this.getCustomSearch().cse.list({
-        q: input,
-        cx: this.cx,
-        auth: this.apiKey,
-        num: 5, // Limit the number of results to 5
-      });
-
-      // return response.data;
-      // logger.debug(response.data);
-
-      if (!response.data.items || response.data.items.length === 0) {
-        return this.resultsToReadableFormat([
-          { title: 'No good Google Search Result was found', link: '' },
-        ]);
-      }
-
-      // const results = response.items.slice(0, numResults);
-      const results = response.data.items;
-
-      for (const result of results) {
-        const metadataResult = {
-          title: result.title || '',
-          link: result.link || '',
-        };
-        if (result.snippet) {
-          metadataResult.snippet = result.snippet;
-        }
-        metadataResults.push(metadataResult);
-      }
-
-      return this.resultsToReadableFormat(metadataResults);
-    } catch (error) {
-      logger.error('[GoogleSearchAPI]', error);
-      // throw error;
-      return 'There was an error searching Google.';
-    }
-  }
-}
-
-module.exports = GoogleSearchAPI;

api/app/clients/tools/index.js

@@ -1,35 +1,44 @@
-const GoogleSearchAPI = require('./GoogleSearch');
-const OpenAICreateImage = require('./DALL-E');
-const DALLE3 = require('./structured/DALLE3');
-const StructuredSD = require('./structured/StableDiffusion');
-const StableDiffusionAPI = require('./StableDiffusion');
+const availableTools = require('./manifest.json');
+// Basic Tools
+const CodeBrew = require('./CodeBrew');
 const WolframAlphaAPI = require('./Wolfram');
-const StructuredWolfram = require('./structured/Wolfram');
-const SelfReflectionTool = require('./SelfReflection');
 const AzureAiSearch = require('./AzureAiSearch');
-const StructuredACS = require('./structured/AzureAISearch');
+const OpenAICreateImage = require('./DALL-E');
+const StableDiffusionAPI = require('./StableDiffusion');
+const SelfReflectionTool = require('./SelfReflection');
+
+// Structured Tools
+const DALLE3 = require('./structured/DALLE3');
 const ChatTool = require('./structured/ChatTool');
 const E2BTools = require('./structured/E2BTools');
 const CodeSherpa = require('./structured/CodeSherpa');
+const StructuredSD = require('./structured/StableDiffusion');
+const StructuredACS = require('./structured/AzureAISearch');
 const CodeSherpaTools = require('./structured/CodeSherpaTools');
-const availableTools = require('./manifest.json');
-const CodeBrew = require('./CodeBrew');
+const GoogleSearchAPI = require('./structured/GoogleSearch');
+const StructuredWolfram = require('./structured/Wolfram');
+const TavilySearchResults = require('./structured/TavilySearchResults');
+const TraversaalSearch = require('./structured/TraversaalSearch');
 
 module.exports = {
   availableTools,
-  GoogleSearchAPI,
-  OpenAICreateImage,
-  DALLE3,
-  StableDiffusionAPI,
-  StructuredSD,
-  WolframAlphaAPI,
-  StructuredWolfram,
-  SelfReflectionTool,
-  AzureAiSearch,
-  StructuredACS,
-  E2BTools,
-  ChatTool,
-  CodeSherpa,
-  CodeSherpaTools,
+  // Basic Tools
   CodeBrew,
+  AzureAiSearch,
+  GoogleSearchAPI,
+  WolframAlphaAPI,
+  OpenAICreateImage,
+  StableDiffusionAPI,
+  SelfReflectionTool,
+  // Structured Tools
+  DALLE3,
+  ChatTool,
+  E2BTools,
+  CodeSherpa,
+  StructuredSD,
+  StructuredACS,
+  CodeSherpaTools,
+  StructuredWolfram,
+  TavilySearchResults,
+  TraversaalSearch,
 };

api/app/clients/tools/manifest.json

@@ -1,4 +1,17 @@
 [
+  {
+    "name": "Traversaal",
+    "pluginKey": "traversaal_search",
+    "description": "Traversaal is a robust search API tailored for LLM Agents. Get an API key here: https://api.traversaal.ai",
+    "icon": "https://traversaal.ai/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TRAVERSAAL_API_KEY",
+        "label": "Traversaal API Key",
+        "description": "Get your API key here: <a href=\"https://api.traversaal.ai\" target=\"_blank\">https://api.traversaal.ai</a>"
+      }
+    ]
+  },
   {
     "name": "Google",
     "pluginKey": "google",
@@ -89,7 +102,7 @@
     "icon": "https://i.imgur.com/u2TzXzH.png",
     "authConfig": [
       {
-        "authField": "DALLE_API_KEY",
+        "authField": "DALLE2_API_KEY||DALLE_API_KEY",
         "label": "OpenAI API Key",
         "description": "You can use DALL-E with your API Key from OpenAI."
       }
@@ -102,12 +115,25 @@
     "icon": "https://i.imgur.com/u2TzXzH.png",
     "authConfig": [
       {
-        "authField": "DALLE_API_KEY",
+        "authField": "DALLE3_API_KEY||DALLE_API_KEY",
         "label": "OpenAI API Key",
         "description": "You can use DALL-E with your API Key from OpenAI."
       }
     ]
   },
+  {
+    "name": "Tavily Search",
+    "pluginKey": "tavily_search_results_json",
+    "description": "Tavily Search is a robust search API tailored for LLM Agents. It seamlessly integrates with diverse data sources to ensure a superior, relevant search experience.",
+    "icon": "https://tavily.com/favicon.ico",
+    "authConfig": [
+      {
+        "authField": "TAVILY_API_KEY",
+        "label": "Tavily API Key",
+        "description": "Get your API key here: https://app.tavily.com/"
+      }
+    ]
+  },
   {
     "name": "Calculator",
     "pluginKey": "calculator",

api/app/clients/tools/saveImageFromUrl.js

@@ -1,40 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const axios = require('axios');
-const { logger } = require('~/config');
-
-async function saveImageFromUrl(url, outputPath, outputFilename) {
-  try {
-    // Fetch the image from the URL
-    const response = await axios({
-      url,
-      responseType: 'stream',
-    });
-
-    // Check if the output directory exists, if not, create it
-    if (!fs.existsSync(outputPath)) {
-      fs.mkdirSync(outputPath, { recursive: true });
-    }
-
-    // Ensure the output filename has a '.png' extension
-    const filenameWithPngExt = outputFilename.endsWith('.png')
-      ? outputFilename
-      : `${outputFilename}.png`;
-
-    // Create a writable stream for the output path
-    const outputFilePath = path.join(outputPath, filenameWithPngExt);
-    const writer = fs.createWriteStream(outputFilePath);
-
-    // Pipe the response data to the output file
-    response.data.pipe(writer);
-
-    return new Promise((resolve, reject) => {
-      writer.on('finish', resolve);
-      writer.on('error', reject);
-    });
-  } catch (error) {
-    logger.error('[saveImageFromUrl] Error while saving the image:', error);
-  }
-}
-
-module.exports = saveImageFromUrl;

api/app/clients/tools/structured/AzureAISearch.js

@@ -16,6 +16,16 @@ class AzureAISearch extends StructuredTool {
 
   constructor(fields = {}) {
     super();
+    this.name = 'azure-ai-search';
+    this.description =
+      'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
+    // Define schema
+    this.schema = z.object({
+      query: z.string().describe('Search word or phrase to Azure AI Search'),
+    });
 
     // Initialize properties using helper function
     this.serviceEndpoint = this._initializeField(
@@ -48,12 +58,16 @@ class AzureAISearch extends StructuredTool {
     );
 
     // Check for required fields
-    if (!this.serviceEndpoint || !this.indexName || !this.apiKey) {
+    if (!this.override && (!this.serviceEndpoint || !this.indexName || !this.apiKey)) {
       throw new Error(
         'Missing AZURE_AI_SEARCH_SERVICE_ENDPOINT, AZURE_AI_SEARCH_INDEX_NAME, or AZURE_AI_SEARCH_API_KEY environment variable.',
       );
     }
 
+    if (this.override) {
+      return;
+    }
+
     // Create SearchClient
     this.client = new SearchClient(
       this.serviceEndpoint,
@@ -61,20 +75,6 @@ class AzureAISearch extends StructuredTool {
       new AzureKeyCredential(this.apiKey),
       { apiVersion: this.apiVersion },
     );
-
-    // Define schema
-    this.schema = z.object({
-      query: z.string().describe('Search word or phrase to Azure AI Search'),
-    });
-  }
-
-  // Simplified getter methods
-  get name() {
-    return 'azure-ai-search';
-  }
-
-  get description() {
-    return 'Use the \'azure-ai-search\' tool to retrieve search results relevant to your input';
   }
 
   // Improved error handling and logging

api/app/clients/tools/structured/DALLE3.js

@@ -1,30 +1,50 @@
-// From https://platform.openai.com/docs/guides/images/usage?context=node
-// To use this tool, you must pass in a configured OpenAIApi object.
-const fs = require('fs');
-const path = require('path');
 const { z } = require('zod');
+const path = require('path');
 const OpenAI = require('openai');
+const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
-const saveImageFromUrl = require('../saveImageFromUrl');
+const { FileContext } = require('librechat-data-provider');
+const { getImageBasename } = require('~/server/services/Files/images');
 const extractBaseURL = require('~/utils/extractBaseURL');
 const { logger } = require('~/config');
 
-const { DALLE3_SYSTEM_PROMPT, DALLE_REVERSE_PROXY, PROXY } = process.env;
 class DALLE3 extends Tool {
   constructor(fields = {}) {
     super();
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
 
-    let apiKey = fields.DALLE_API_KEY || this.getApiKey();
+    this.userId = fields.userId;
+    this.fileStrategy = fields.fileStrategy;
+    if (fields.processFileURL) {
+      /** @type {processFileURL} Necessary for output to contain all image metadata. */
+      this.processFileURL = fields.processFileURL.bind(this);
+    }
+
+    let apiKey = fields.DALLE3_API_KEY ?? fields.DALLE_API_KEY ?? this.getApiKey();
     const config = { apiKey };
-    if (DALLE_REVERSE_PROXY) {
-      config.baseURL = extractBaseURL(DALLE_REVERSE_PROXY);
+    if (process.env.DALLE_REVERSE_PROXY) {
+      config.baseURL = extractBaseURL(process.env.DALLE_REVERSE_PROXY);
     }
 
-    if (PROXY) {
-      config.httpAgent = new HttpsProxyAgent(PROXY);
+    if (process.env.DALLE3_AZURE_API_VERSION && process.env.DALLE3_BASEURL) {
+      config.baseURL = process.env.DALLE3_BASEURL;
+      config.defaultQuery = { 'api-version': process.env.DALLE3_AZURE_API_VERSION };
+      config.defaultHeaders = {
+        'api-key': process.env.DALLE3_API_KEY,
+        'Content-Type': 'application/json',
+      };
+      config.apiKey = process.env.DALLE3_API_KEY;
     }
 
+    if (process.env.PROXY) {
+      config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
+    }
+
+    /** @type {OpenAI} */
     this.openai = new OpenAI(config);
     this.name = 'dalle';
     this.description = `Use DALLE to create images from text descriptions.
@@ -32,7 +52,7 @@ class DALLE3 extends Tool {
     - Create only one image, without repeating or listing descriptions outside the "prompts" field.
     - Maintains the original intent of the description, with parameters for image style, quality, and size to tailor the output.`;
     this.description_for_model =
-      DALLE3_SYSTEM_PROMPT ??
+      process.env.DALLE3_SYSTEM_PROMPT ??
       `// Whenever a description of an image is given, generate prompts (following these rules), and use dalle to create the image. If the user does not ask for a specific number of images, default to creating 2 prompts to send to dalle that are written to be as diverse as possible. All prompts sent to dalle must abide by the following policies:
     // 1. Prompts must be in English. Translate to English if needed.
     // 2. One image per function call. Create only 1 image per request unless explicitly told to generate more than 1 image.
@@ -44,7 +64,8 @@ class DALLE3 extends Tool {
     // - Use "various" or "diverse" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
     // - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
     // The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
-    // All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.`;
+    // All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.
+    // - The "vivid" style is HIGHLY preferred, but "natural" is also supported.`;
     this.schema = z.object({
       prompt: z
         .string()
@@ -69,8 +90,8 @@ class DALLE3 extends Tool {
   }
 
   getApiKey() {
-    const apiKey = process.env.DALLE_API_KEY || '';
-    if (!apiKey) {
+    const apiKey = process.env.DALLE3_API_KEY ?? process.env.DALLE_API_KEY ?? '';
+    if (!apiKey && !this.override) {
       throw new Error('Missing DALLE_API_KEY environment variable.');
     }
     return apiKey;
@@ -83,12 +104,8 @@ class DALLE3 extends Tool {
       .trim();
   }
 
-  getMarkdownImageUrl(imageName) {
-    const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
-      .replace(/\\/g, '/')
-      .replace('public/', '');
-    return `![generated image](/${imageUrl})`;
+  wrapInMarkdown(imageUrl) {
+    return `![generated image](${imageUrl})`;
   }
 
   async _call(data) {
@@ -108,12 +125,13 @@ class DALLE3 extends Tool {
         n: 1,
       });
     } catch (error) {
-      return `Something went wrong when trying to generate the image. The DALL-E API may unavailable:
+      logger.error('[DALL-E-3] Problem generating the image:', error);
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
 Error Message: ${error.message}`;
     }
 
     if (!resp) {
-      return 'Something went wrong when trying to generate the image. The DALL-E API may unavailable';
+      return 'Something went wrong when trying to generate the image. The DALL-E API may be unavailable';
     }
 
     const theImageUrl = resp.data[0].url;
@@ -122,45 +140,39 @@ Error Message: ${error.message}`;
       return 'No image URL returned from OpenAI API. There may be a problem with the API or your configuration.';
     }
 
-    const regex = /img-[\w\d]+.png/;
-    const match = theImageUrl.match(regex);
-    let imageName = '1.png';
+    const imageBasename = getImageBasename(theImageUrl);
+    const imageExt = path.extname(imageBasename);
 
-    if (match) {
-      imageName = match[0];
-      logger.debug('[DALL-E-3]', { imageName }); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
-    } else {
-      logger.debug('[DALL-E-3] No image name found in the string.', {
-        theImageUrl,
-        data: resp.data[0],
-      });
-    }
+    const extension = imageExt.startsWith('.') ? imageExt.slice(1) : imageExt;
+    const imageName = `img-${uuidv4()}.${extension}`;
 
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
-
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
-    }
+    logger.debug('[DALL-E-3]', {
+      imageName,
+      imageBasename,
+      imageExt,
+      extension,
+      theImageUrl,
+      data: resp.data[0],
+    });
 
     try {
-      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
-      this.result = this.getMarkdownImageUrl(imageName);
+      const result = await this.processFileURL({
+        fileStrategy: this.fileStrategy,
+        userId: this.userId,
+        URL: theImageUrl,
+        fileName: imageName,
+        basePath: 'images',
+        context: FileContext.image_generation,
+      });
+
+      if (this.returnMetadata) {
+        this.result = result;
+      } else {
+        this.result = this.wrapInMarkdown(result.filepath);
+      }
     } catch (error) {
       logger.error('Error while saving the image:', error);
-      this.result = theImageUrl;
+      this.result = `Failed to save the image locally. ${error.message}`;
     }
 
     return this.result;

api/app/clients/tools/structured/GoogleSearch.js

@@ -0,0 +1,65 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class GoogleSearchResults extends Tool {
+  static lc_name() {
+    return 'GoogleSearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVarApiKey = 'GOOGLE_API_KEY';
+    this.envVarSearchEngineId = 'GOOGLE_CSE_ID';
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? getEnvironmentVariable(this.envVarApiKey);
+    this.searchEngineId =
+      fields.searchEngineId ?? getEnvironmentVariable(this.envVarSearchEngineId);
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'google';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 10.'),
+      // Note: Google API has its own parameters for search customization, adjust as needed.
+    });
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, max_results = 5 } = validationResult.data;
+
+    const response = await fetch(
+      `https://www.googleapis.com/customsearch/v1?key=${this.apiKey}&cx=${
+        this.searchEngineId
+      }&q=${encodeURIComponent(query)}&num=${max_results}`,
+      {
+        method: 'GET',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error.message}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = GoogleSearchResults;

api/app/clients/tools/structured/StableDiffusion.js

@@ -4,12 +4,28 @@ const { z } = require('zod');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { v4: uuidv4 } = require('uuid');
 const { StructuredTool } = require('langchain/tools');
+const { FileContext } = require('librechat-data-provider');
+const paths = require('~/config/paths');
 const { logger } = require('~/config');
 
 class StableDiffusionAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /** @type {string} User ID */
+    this.userId = fields.userId;
+    /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
+    this.req = fields.req;
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
+    if (fields.uploadImageBuffer) {
+      /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */
+      this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);
+    }
+
     this.name = 'stable-diffusion';
     this.url = fields.SD_WEBUI_URL || this.getServerURL();
     this.description_for_model = `// Generate images and visuals using text.
@@ -44,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getMarkdownImageUrl(imageName) {
     const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
+      .join(this.relativePath, this.userId, imageName)
       .replace(/\\/g, '/')
       .replace('public/', '');
     return `![generated image](/${imageUrl})`;
@@ -52,7 +68,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getServerURL() {
     const url = process.env.SD_WEBUI_URL || '';
-    if (!url) {
+    if (!url && !this.override) {
       throw new Error('Missing SD_WEBUI_URL environment variable.');
     }
     return url;
@@ -70,46 +86,67 @@ class StableDiffusionAPI extends StructuredTool {
       width: 1024,
       height: 1024,
     };
-    const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
-    const image = response.data.images[0];
-    const pngPayload = { image: `data:image/png;base64,${image}` };
-    const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);
-    const info = response2.data.info;
+    const generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
+    const image = generationResponse.data.images[0];
 
-    // Generate unique name
-    const imageName = `${Date.now()}.png`;
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+    /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */
+    let info = {};
+    try {
+      info = JSON.parse(generationResponse.data.info);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while getting image metadata:', error);
+    }
 
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
+    const file_id = uuidv4();
+    const imageName = `${file_id}.png`;
+    const { imageOutput: imageOutputPath, clientPath } = paths;
+    const filepath = path.join(imageOutputPath, this.userId, imageName);
+    this.relativePath = path.relative(clientPath, imageOutputPath);
+
+    if (!fs.existsSync(path.join(imageOutputPath, this.userId))) {
+      fs.mkdirSync(path.join(imageOutputPath, this.userId), { recursive: true });
     }
 
     try {
       const buffer = Buffer.from(image.split(',', 1)[0], 'base64');
+      if (this.returnMetadata && this.uploadImageBuffer && this.req) {
+        const file = await this.uploadImageBuffer({
+          req: this.req,
+          context: FileContext.image_generation,
+          resize: false,
+          metadata: {
+            buffer,
+            height: info.height,
+            width: info.width,
+            bytes: Buffer.byteLength(buffer),
+            filename: imageName,
+            type: 'image/png',
+            file_id,
+          },
+        });
+
+        const generationInfo = info.infotexts[0].split('\n').pop();
+        return {
+          ...file,
+          prompt,
+          metadata: {
+            negative_prompt,
+            seed: info.seed,
+            info: generationInfo,
+          },
+        };
+      }
+
       await sharp(buffer)
         .withMetadata({
           iptcpng: {
-            parameters: info,
+            parameters: info.infotexts[0],
           },
         })
-        .toFile(this.outputPath + '/' + imageName);
+        .toFile(filepath);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
       logger.error('[StableDiffusion] Error while saving the image:', error);
-      // this.result = theImageUrl;
     }
 
     return this.result;

api/app/clients/tools/structured/TavilySearchResults.js

@@ -0,0 +1,92 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+class TavilySearchResults extends Tool {
+  static lc_name() {
+    return 'TavilySearchResults';
+  }
+
+  constructor(fields = {}) {
+    super(fields);
+    this.envVar = 'TAVILY_API_KEY';
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+    this.apiKey = fields.apiKey ?? this.getApiKey();
+
+    this.kwargs = fields?.kwargs ?? {};
+    this.name = 'tavily_search_results_json';
+    this.description =
+      'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.';
+
+    this.schema = z.object({
+      query: z.string().min(1).describe('The search query string.'),
+      max_results: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('The maximum number of search results to return. Defaults to 5.'),
+      search_depth: z
+        .enum(['basic', 'advanced'])
+        .optional()
+        .describe(
+          'The depth of the search, affecting result quality and response time (`basic` or `advanced`). Default is basic for quick results and advanced for indepth high quality results but longer response time. Advanced calls equals 2 requests.',
+        ),
+      include_images: z
+        .boolean()
+        .optional()
+        .describe(
+          'Whether to include a list of query-related images in the response. Default is False.',
+        ),
+      include_answer: z
+        .boolean()
+        .optional()
+        .describe('Whether to include answers in the search results. Default is False.'),
+      // include_raw_content: z.boolean().optional().describe('Whether to include raw content in the search results. Default is False.'),
+      // include_domains: z.array(z.string()).optional().describe('A list of domains to specifically include in the search results.'),
+      // exclude_domains: z.array(z.string()).optional().describe('A list of domains to specifically exclude from the search results.'),
+    });
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable(this.envVar);
+    if (!apiKey && !this.override) {
+      throw new Error(`Missing ${this.envVar} environment variable.`);
+    }
+    return apiKey;
+  }
+
+  async _call(input) {
+    const validationResult = this.schema.safeParse(input);
+    if (!validationResult.success) {
+      throw new Error(`Validation failed: ${JSON.stringify(validationResult.error.issues)}`);
+    }
+
+    const { query, ...rest } = validationResult.data;
+
+    const requestBody = {
+      api_key: this.apiKey,
+      query,
+      ...rest,
+      ...this.kwargs,
+    };
+
+    const response = await fetch('https://api.tavily.com/search', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(requestBody),
+    });
+
+    const json = await response.json();
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${json.error}`);
+    }
+
+    return JSON.stringify(json);
+  }
+}
+
+module.exports = TavilySearchResults;

api/app/clients/tools/structured/TraversaalSearch.js

@@ -0,0 +1,89 @@
+const { z } = require('zod');
+const { Tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+const { logger } = require('~/config');
+
+/**
+ * Tool for the Traversaal AI search API, Ares.
+ */
+class TraversaalSearch extends Tool {
+  static lc_name() {
+    return 'TraversaalSearch';
+  }
+  constructor(fields) {
+    super(fields);
+    this.name = 'traversaal_search';
+    this.description = `An AI search engine optimized for comprehensive, accurate, and trusted results.
+    Useful for when you need to answer questions about current events. Input should be a search query.`;
+    this.description_for_model =
+      '\'Please create a specific sentence for the AI to understand and use as a query to search the web based on the user\'s request. For example, "Find information about the highest mountains in the world." or "Show me the latest news articles about climate change and its impact on polar ice caps."\'';
+    this.schema = z.object({
+      query: z
+        .string()
+        .describe(
+          'A properly written sentence to be interpreted by an AI to search the web according to the user\'s request.',
+        ),
+    });
+
+    this.apiKey = fields?.TRAVERSAAL_API_KEY ?? this.getApiKey();
+  }
+
+  getApiKey() {
+    const apiKey = getEnvironmentVariable('TRAVERSAAL_API_KEY');
+    if (!apiKey && this.override) {
+      throw new Error(
+        'No Traversaal API key found. Either set an environment variable named "TRAVERSAAL_API_KEY" or pass an API key as "apiKey".',
+      );
+    }
+    return apiKey;
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  async _call({ query }, _runManager) {
+    const body = {
+      query: [query],
+    };
+    try {
+      const response = await fetch('https://api-ares.traversaal.ai/live/predict', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-api-key': this.apiKey,
+        },
+        body: JSON.stringify({ ...body }),
+      });
+      const json = await response.json();
+      if (!response.ok) {
+        throw new Error(
+          `Request failed with status code ${response.status}: ${json.error ?? json.message}`,
+        );
+      }
+      if (!json.data) {
+        throw new Error('Could not parse Traversaal API results. Please try again.');
+      }
+
+      const baseText = json.data?.response_text ?? '';
+      const sources = json.data?.web_url;
+      const noResponse = 'No response found in Traversaal API results';
+
+      if (!baseText && !sources) {
+        return noResponse;
+      }
+
+      const sourcesText = sources?.length ? '\n\nSources:\n - ' + sources.join('\n - ') : '';
+
+      const result = baseText + sourcesText;
+
+      if (!result) {
+        return noResponse;
+      }
+
+      return result;
+    } catch (error) {
+      logger.error('Traversaal API request failed', error);
+      return `Traversaal API request failed: ${error.message}`;
+    }
+  }
+}
+
+module.exports = TraversaalSearch;

api/app/clients/tools/structured/Wolfram.js

@@ -7,6 +7,9 @@ const { logger } = require('~/config');
 class WolframAlphaAPI extends StructuredTool {
   constructor(fields) {
     super();
+    /* Used to initialize the Tool without necessary variables. */
+    this.override = fields.override ?? false;
+
     this.name = 'wolfram';
     this.apiKey = fields.WOLFRAM_APP_ID || this.getAppId();
     this.description_for_model = `// Access dynamic computation and curated data from WolframAlpha and Wolfram Cloud.
@@ -55,7 +58,7 @@ class WolframAlphaAPI extends StructuredTool {
 
   getAppId() {
     const appId = process.env.WOLFRAM_APP_ID || '';
-    if (!appId) {
+    if (!appId && !this.override) {
       throw new Error('Missing WOLFRAM_APP_ID environment variable.');
     }
     return appId;

api/app/clients/tools/structured/specs/DALLE3.spec.js

@@ -1,12 +1,31 @@
-const fs = require('fs');
-const path = require('path');
 const OpenAI = require('openai');
 const DALLE3 = require('../DALLE3');
-const saveImageFromUrl = require('../../saveImageFromUrl');
+
 const { logger } = require('~/config');
 
 jest.mock('openai');
 
+const processFileURL = jest.fn();
+
+jest.mock('~/server/services/Files/images', () => ({
+  getImageBasename: jest.fn().mockImplementation((url) => {
+    // Split the URL by '/'
+    const parts = url.split('/');
+
+    // Get the last part of the URL
+    const lastPart = parts.pop();
+
+    // Check if the last part of the URL matches the image extension regex
+    const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
+    if (imageExtensionRegex.test(lastPart)) {
+      return lastPart;
+    }
+
+    // If the regex test fails, return an empty string
+    return '';
+  }),
+}));
+
 const generate = jest.fn();
 OpenAI.mockImplementation(() => ({
   images: {
@@ -21,15 +40,14 @@ jest.mock('fs', () => {
   };
 });
 
-jest.mock('../../saveImageFromUrl', () => {
-  return jest.fn();
-});
-
 jest.mock('path', () => {
   return {
     resolve: jest.fn(),
     join: jest.fn(),
     relative: jest.fn(),
+    extname: jest.fn().mockImplementation((filename) => {
+      return filename.slice(filename.lastIndexOf('.'));
+    }),
   };
 });
 
@@ -48,7 +66,7 @@ describe('DALLE3', () => {
     jest.resetModules();
     process.env = { ...originalEnv, DALLE_API_KEY: mockApiKey };
     // Instantiate DALLE3 for tests that do not depend on DALLE3_SYSTEM_PROMPT
-    dalle = new DALLE3();
+    dalle = new DALLE3({ processFileURL });
   });
 
   afterEach(() => {
@@ -57,7 +75,8 @@ describe('DALLE3', () => {
     process.env = originalEnv;
   });
 
-  it('should throw an error if DALLE_API_KEY is missing', () => {
+  it('should throw an error if all potential API keys are missing', () => {
+    delete process.env.DALLE3_API_KEY;
     delete process.env.DALLE_API_KEY;
     expect(() => new DALLE3()).toThrow('Missing DALLE_API_KEY environment variable.');
   });
@@ -70,10 +89,8 @@ describe('DALLE3', () => {
 
   it('should generate markdown image URL correctly', () => {
     const imageName = 'test.png';
-    path.join.mockReturnValue('images/test.png');
-    path.relative.mockReturnValue('images/test.png');
-    const markdownImage = dalle.getMarkdownImageUrl(imageName);
-    expect(markdownImage).toBe('![generated image](/images/test.png)');
+    const markdownImage = dalle.wrapInMarkdown(imageName);
+    expect(markdownImage).toBe('![generated image](test.png)');
   });
 
   it('should call OpenAI API with correct parameters', async () => {
@@ -93,11 +110,9 @@ describe('DALLE3', () => {
     };
 
     generate.mockResolvedValue(mockResponse);
-    saveImageFromUrl.mockResolvedValue(true);
-    fs.existsSync.mockReturnValue(true);
-    path.resolve.mockReturnValue('/fakepath/images');
-    path.join.mockReturnValue('/fakepath/images/img-test.png');
-    path.relative.mockReturnValue('images/img-test.png');
+    processFileURL.mockResolvedValue({
+      filepath: 'http://example.com/img-test.png',
+    });
 
     const result = await dalle._call(mockData);
 
@@ -109,6 +124,7 @@ describe('DALLE3', () => {
       prompt: mockData.prompt,
       n: 1,
     });
+
     expect(result).toContain('![generated image]');
   });
 
@@ -135,7 +151,7 @@ describe('DALLE3', () => {
     await expect(dalle._call(mockData)).rejects.toThrow('Missing required field: prompt');
   });
 
-  it('should log to console if no image name is found in the URL', async () => {
+  it('should log appropriate debug values', async () => {
     const mockData = {
       prompt: 'A test prompt',
     };
@@ -149,29 +165,16 @@ describe('DALLE3', () => {
 
     generate.mockResolvedValue(mockResponse);
     await dalle._call(mockData);
-    expect(logger.debug).toHaveBeenCalledWith('[DALL-E-3] No image name found in the string.', {
+    expect(logger.debug).toHaveBeenCalledWith('[DALL-E-3]', {
       data: { url: 'http://example.com/invalid-url' },
       theImageUrl: 'http://example.com/invalid-url',
+      extension: expect.any(String),
+      imageBasename: expect.any(String),
+      imageExt: expect.any(String),
+      imageName: expect.any(String),
     });
   });
 
-  it('should create the directory if it does not exist', async () => {
-    const mockData = {
-      prompt: 'A test prompt',
-    };
-    const mockResponse = {
-      data: [
-        {
-          url: 'http://example.com/img-test.png',
-        },
-      ],
-    };
-    generate.mockResolvedValue(mockResponse);
-    fs.existsSync.mockReturnValue(false); // Simulate directory does not exist
-    await dalle._call(mockData);
-    expect(fs.mkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true });
-  });
-
   it('should log an error and return the image URL if there is an error saving the image', async () => {
     const mockData = {
       prompt: 'A test prompt',
@@ -185,9 +188,25 @@ describe('DALLE3', () => {
     };
     const error = new Error('Error while saving the image');
     generate.mockResolvedValue(mockResponse);
-    saveImageFromUrl.mockRejectedValue(error);
+    processFileURL.mockRejectedValue(error);
     const result = await dalle._call(mockData);
     expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
-    expect(result).toBe(mockResponse.data[0].url);
+    expect(result).toBe('Failed to save the image locally. Error while saving the image');
+  });
+
+  it('should handle error when saving image to Firebase Storage fails', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+    };
+    const mockImageUrl = 'http://example.com/img-test.png';
+    const mockResponse = { data: [{ url: mockImageUrl }] };
+    const error = new Error('Error while saving to Firebase');
+    generate.mockResolvedValue(mockResponse);
+    processFileURL.mockRejectedValue(error);
+
+    const result = await dalle._call(mockData);
+
+    expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
+    expect(result).toContain('Failed to save the image');
   });
 });

api/app/clients/tools/util/handleTools.js

@@ -6,19 +6,23 @@ const { OpenAIEmbeddings } = require('langchain/embeddings/openai');
 const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const {
   availableTools,
+  // Basic Tools
+  CodeBrew,
+  AzureAISearch,
   GoogleSearchAPI,
   WolframAlphaAPI,
-  StructuredWolfram,
   OpenAICreateImage,
   StableDiffusionAPI,
+  // Structured Tools
   DALLE3,
-  StructuredSD,
-  AzureAISearch,
-  StructuredACS,
   E2BTools,
   CodeSherpa,
+  StructuredSD,
+  StructuredACS,
   CodeSherpaTools,
-  CodeBrew,
+  TraversaalSearch,
+  StructuredWolfram,
+  TavilySearchResults,
 } = require('../');
 const { loadToolSuite } = require('./loadToolSuite');
 const { loadSpecs } = require('./loadSpecs');
@@ -30,6 +34,14 @@ const getOpenAIKey = async (options, user) => {
   return openAIApiKey || (await getUserPluginAuthValue(user, 'OPENAI_API_KEY'));
 };
 
+/**
+ * Validates the availability and authentication of tools for a user based on environment variables or user-specific plugin authentication values.
+ * Tools without required authentication or with valid authentication are considered valid.
+ *
+ * @param {Object} user The user object for whom to validate tool access.
+ * @param {Array<string>} tools An array of tool identifiers to validate. Defaults to an empty array.
+ * @returns {Promise<Array<string>>} A promise that resolves to an array of valid tool identifiers.
+ */
 const validateTools = async (user, tools = []) => {
   try {
     const validToolsSet = new Set(tools);
@@ -37,16 +49,34 @@ const validateTools = async (user, tools = []) => {
       validToolsSet.has(tool.pluginKey),
     );
 
+    /**
+     * Validates the credentials for a given auth field or set of alternate auth fields for a tool.
+     * If valid admin or user authentication is found, the function returns early. Otherwise, it removes the tool from the set of valid tools.
+     *
+     * @param {string} authField The authentication field or fields (separated by "||" for alternates) to validate.
+     * @param {string} toolName The identifier of the tool being validated.
+     */
     const validateCredentials = async (authField, toolName) => {
-      const adminAuth = process.env[authField];
-      if (adminAuth && adminAuth.length > 0) {
-        return;
+      const fields = authField.split('||');
+      for (const field of fields) {
+        const adminAuth = process.env[field];
+        if (adminAuth && adminAuth.length > 0) {
+          return;
+        }
+
+        let userAuth = null;
+        try {
+          userAuth = await getUserPluginAuthValue(user, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !userAuth) {
+            throw err;
+          }
+        }
+        if (userAuth && userAuth.length > 0) {
+          return;
+        }
       }
 
-      const userAuth = await getUserPluginAuthValue(user, authField);
-      if (userAuth && userAuth.length > 0) {
-        return;
-      }
       validToolsSet.delete(toolName);
     };
 
@@ -63,23 +93,58 @@ const validateTools = async (user, tools = []) => {
     return Array.from(validToolsSet.values());
   } catch (err) {
     logger.error('[validateTools] There was a problem validating tools', err);
-    throw new Error(err);
+    throw new Error('There was a problem validating tools');
   }
 };
 
-const loadToolWithAuth = async (user, authFields, ToolConstructor, options = {}) => {
+/**
+ * Initializes a tool with authentication values for the given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {string} userId The user ID for which the tool is being loaded.
+ * @param {Array<string>} authFields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+ * @param {typeof import('langchain/tools').Tool} ToolConstructor The constructor function for the tool to be initialized.
+ * @param {Object} options Optional parameters to be passed to the tool constructor alongside authentication values.
+ * @returns {Function} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication.
+ */
+const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => {
   return async function () {
     let authValues = {};
 
-    for (const authField of authFields) {
-      let authValue = process.env[authField];
-      if (!authValue) {
-        authValue = await getUserPluginAuthValue(user, authField);
+    /**
+     * Finds the first non-empty value for the given authentication field, supporting alternate fields.
+     * @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
+     * @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found.
+     */
+    const findAuthValue = async (fields) => {
+      for (const field of fields) {
+        let value = process.env[field];
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+        try {
+          value = await getUserPluginAuthValue(userId, field);
+        } catch (err) {
+          if (field === fields[fields.length - 1] && !value) {
+            throw err;
+          }
+        }
+        if (value) {
+          return { authField: field, authValue: value };
+        }
+      }
+      return null;
+    };
+
+    for (let authField of authFields) {
+      const fields = authField.split('||');
+      const result = await findAuthValue(fields);
+      if (result) {
+        authValues[result.authField] = result.authValue;
       }
-      authValues[authField] = authValue;
     }
 
-    return new ToolConstructor({ ...options, ...authValues });
+    return new ToolConstructor({ ...options, ...authValues, userId });
   };
 };
 
@@ -90,8 +155,10 @@ const loadTools = async ({
   returnMap = false,
   tools = [],
   options = {},
+  skipSpecs = false,
 }) => {
   const toolConstructors = {
+    tavily_search_results_json: TavilySearchResults,
     calculator: Calculator,
     google: GoogleSearchAPI,
     wolfram: functions ? StructuredWolfram : WolframAlphaAPI,
@@ -99,6 +166,7 @@ const loadTools = async ({
     'stable-diffusion': functions ? StructuredSD : StableDiffusionAPI,
     'azure-ai-search': functions ? StructuredACS : AzureAISearch,
     CodeBrew: CodeBrew,
+    traversaal_search: TraversaalSearch,
   };
 
   const openAIApiKey = await getOpenAIKey(options, user);
@@ -168,8 +236,19 @@ const loadTools = async ({
     toolConstructors.codesherpa = CodeSherpa;
   }
 
+  const imageGenOptions = {
+    req: options.req,
+    fileStrategy: options.fileStrategy,
+    processFileURL: options.processFileURL,
+    returnMetadata: options.returnMetadata,
+    uploadImageBuffer: options.uploadImageBuffer,
+  };
+
   const toolOptions = {
     serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' },
+    dalle: imageGenOptions,
+    'dall-e': imageGenOptions,
+    'stable-diffusion': imageGenOptions,
   };
 
   const toolAuthFields = {};
@@ -192,7 +271,7 @@ const loadTools = async ({
 
     if (toolConstructors[tool]) {
       const options = toolOptions[tool] || {};
-      const toolInstance = await loadToolWithAuth(
+      const toolInstance = loadToolWithAuth(
         user,
         toolAuthFields[tool],
         toolConstructors[tool],
@@ -208,7 +287,7 @@ const loadTools = async ({
   }
 
   let specs = null;
-  if (functions && remainingTools.length > 0) {
+  if (functions && remainingTools.length > 0 && skipSpecs !== true) {
     specs = await loadSpecs({
       llm: model,
       user,
@@ -235,6 +314,9 @@ const loadTools = async ({
   let result = [];
   for (const tool of tools) {
     const validTool = requestedTools[tool];
+    if (!validTool) {
+      continue;
+    }
     const plugin = await validTool();
 
     if (Array.isArray(plugin)) {
@@ -248,6 +330,7 @@ const loadTools = async ({
 };
 
 module.exports = {
+  loadToolWithAuth,
   validateTools,
   loadTools,
 };

api/app/clients/tools/util/handleTools.test.js

@@ -4,26 +4,33 @@ const mockUser = {
   findByIdAndDelete: jest.fn(),
 };
 
-var mockPluginService = {
+const mockPluginService = {
   updateUserPluginAuth: jest.fn(),
   deleteUserPluginAuth: jest.fn(),
   getUserPluginAuthValue: jest.fn(),
 };
 
-jest.mock('../../../../models/User', () => {
+jest.mock('~/models/User', () => {
   return function () {
     return mockUser;
   };
 });
 
-jest.mock('../../../../server/services/PluginService', () => mockPluginService);
+jest.mock('~/server/services/PluginService', () => mockPluginService);
 
-const User = require('../../../../models/User');
-const { validateTools, loadTools } = require('./');
-const PluginService = require('../../../../server/services/PluginService');
-const { BaseChatModel } = require('langchain/chat_models/openai');
 const { Calculator } = require('langchain/tools/calculator');
-const { availableTools, OpenAICreateImage, GoogleSearchAPI, StructuredSD } = require('../');
+const { BaseChatModel } = require('langchain/chat_models/openai');
+
+const User = require('~/models/User');
+const PluginService = require('~/server/services/PluginService');
+const { validateTools, loadTools, loadToolWithAuth } = require('./handleTools');
+const {
+  availableTools,
+  OpenAICreateImage,
+  GoogleSearchAPI,
+  StructuredSD,
+  WolframAlphaAPI,
+} = require('../');
 
 describe('Tool Handlers', () => {
   let fakeUser;
@@ -44,7 +51,10 @@ describe('Tool Handlers', () => {
     });
     mockPluginService.updateUserPluginAuth.mockImplementation(
       (userId, authField, _pluginKey, credential) => {
-        userAuthValues[`${userId}-${authField}`] = credential;
+        const fields = authField.split('||');
+        fields.forEach((field) => {
+          userAuthValues[`${userId}-${field}`] = credential;
+        });
       },
     );
 
@@ -53,6 +63,7 @@ describe('Tool Handlers', () => {
       username: 'fakeuser',
       email: 'fakeuser@example.com',
       emailVerified: false,
+      // file deepcode ignore NoHardcodedPasswords/test: fake value
       password: 'fakepassword123',
       avatar: '',
       provider: 'local',
@@ -133,6 +144,18 @@ describe('Tool Handlers', () => {
       loadTool2 = toolFunctions[sampleTools[1]];
       loadTool3 = toolFunctions[sampleTools[2]];
     });
+
+    let originalEnv;
+
+    beforeEach(() => {
+      originalEnv = process.env;
+      process.env = { ...originalEnv };
+    });
+
+    afterEach(() => {
+      process.env = originalEnv;
+    });
+
     it('returns the expected load functions for requested tools', async () => {
       expect(loadTool1).toBeDefined();
       expect(loadTool2).toBeDefined();
@@ -149,6 +172,86 @@ describe('Tool Handlers', () => {
       expect(authTool).toBeInstanceOf(ToolClass);
       expect(tool).toBeInstanceOf(ToolClass2);
     });
+
+    it('should initialize an authenticated tool with primary auth field', async () => {
+      process.env.DALLE2_API_KEY = 'mocked_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool with alternate auth field when primary is missing', async () => {
+      delete process.env.DALLE2_API_KEY; // Ensure the primary key is not set
+      process.env.DALLE_API_KEY = 'mocked_alternate_api_key';
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'DALLE2_API_KEY',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when env vars are missing', async () => {
+      mockPluginService.updateUserPluginAuth('userId', 'DALLE_API_KEY', 'dalle', 'mocked_api_key');
+      const initToolFunction = loadToolWithAuth(
+        'userId',
+        ['DALLE2_API_KEY||DALLE_API_KEY'],
+        ToolClass,
+      );
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(ToolClass);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(2);
+    });
+
+    it('should initialize an authenticated tool with singular auth field', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalled();
+    });
+
+    it('should initialize an authenticated tool when env var is set', async () => {
+      process.env.WOLFRAM_APP_ID = 'mocked_app_id';
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).not.toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
+    it('should fallback to getUserPluginAuthValue when singular env var is missing', async () => {
+      delete process.env.WOLFRAM_APP_ID; // Ensure the environment variable is not set
+      mockPluginService.getUserPluginAuthValue.mockResolvedValue('mocked_user_auth_value');
+      const initToolFunction = loadToolWithAuth('userId', ['WOLFRAM_APP_ID'], WolframAlphaAPI);
+      const authTool = await initToolFunction();
+
+      expect(authTool).toBeInstanceOf(WolframAlphaAPI);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledTimes(1);
+      expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith(
+        'userId',
+        'WOLFRAM_APP_ID',
+      );
+    });
+
     it('should throw an error for an unauthenticated tool', async () => {
       try {
         await loadTool2();

api/app/clients/tools/util/loadToolSuite.js

@@ -1,17 +1,49 @@
-const { getUserPluginAuthValue } = require('../../../../server/services/PluginService');
+const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const { availableTools } = require('../');
+const { logger } = require('~/config');
 
-const loadToolSuite = async ({ pluginKey, tools, user, options }) => {
+/**
+ * Loads a suite of tools with authentication values for a given user, supporting alternate authentication fields.
+ * Authentication fields can have alternates separated by "||", and the first defined variable will be used.
+ *
+ * @param {Object} params Parameters for loading the tool suite.
+ * @param {string} params.pluginKey Key identifying the plugin whose tools are to be loaded.
+ * @param {Array<Function>} params.tools Array of tool constructor functions.
+ * @param {Object} params.user User object for whom the tools are being loaded.
+ * @param {Object} [params.options={}] Optional parameters to be passed to each tool constructor.
+ * @returns {Promise<Array>} A promise that resolves to an array of instantiated tools.
+ */
+const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
   const authConfig = availableTools.find((tool) => tool.pluginKey === pluginKey).authConfig;
   const suite = [];
   const authValues = {};
 
-  for (const auth of authConfig) {
-    let authValue = process.env[auth.authField];
-    if (!authValue) {
-      authValue = await getUserPluginAuthValue(user, auth.authField);
+  const findAuthValue = async (authField) => {
+    const fields = authField.split('||');
+    for (const field of fields) {
+      let value = process.env[field];
+      if (value) {
+        return value;
+      }
+      try {
+        value = await getUserPluginAuthValue(user, field);
+        if (value) {
+          return value;
+        }
+      } catch (err) {
+        logger.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
+      }
+    }
+    return null;
+  };
+
+  for (const auth of authConfig) {
+    const authValue = await findAuthValue(auth.authField);
+    if (authValue !== null) {
+      authValues[auth.authField] = authValue;
+    } else {
+      logger.warn(`[loadToolSuite] No auth value found for ${auth.authField}`);
     }
-    authValues[auth.authField] = authValue;
   }
 
   for (const tool of tools) {

api/cache/getLogStores.js

@@ -1,9 +1,10 @@
 const Keyv = require('keyv');
-const keyvMongo = require('./keyvMongo');
-const keyvRedis = require('./keyvRedis');
-const { CacheKeys } = require('~/common/enums');
-const { math, isEnabled } = require('~/server/utils');
+const { CacheKeys, ViolationTypes } = require('librechat-data-provider');
 const { logFile, violationFile } = require('./keyvFiles');
+const { math, isEnabled } = require('~/server/utils');
+const keyvRedis = require('./keyvRedis');
+const keyvMongo = require('./keyvMongo');
+
 const { BAN_DURATION, USE_REDIS } = process.env ?? {};
 
 const duration = math(BAN_DURATION, 7200000);
@@ -20,38 +21,58 @@ const pending_req = isEnabled(USE_REDIS)
 
 const config = isEnabled(USE_REDIS)
   ? new Keyv({ store: keyvRedis })
-  : new Keyv({ namespace: CacheKeys.CONFIG });
+  : new Keyv({ namespace: CacheKeys.CONFIG_STORE });
+
+const tokenConfig = isEnabled(USE_REDIS) // ttl: 30 minutes
+  ? new Keyv({ store: keyvRedis, ttl: 1800000 })
+  : new Keyv({ namespace: CacheKeys.TOKEN_CONFIG, ttl: 1800000 });
+
+const genTitle = isEnabled(USE_REDIS) // ttl: 2 minutes
+  ? new Keyv({ store: keyvRedis, ttl: 120000 })
+  : new Keyv({ namespace: CacheKeys.GEN_TITLE, ttl: 120000 });
+
+const modelQueries = isEnabled(process.env.USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.MODEL_QUERIES });
+
+const abortKeys = isEnabled(USE_REDIS)
+  ? new Keyv({ store: keyvRedis })
+  : new Keyv({ namespace: CacheKeys.ABORT_KEYS, ttl: 600000 });
 
 const namespaces = {
-  config,
+  [CacheKeys.CONFIG_STORE]: config,
   pending_req,
   ban: new Keyv({ store: keyvMongo, namespace: 'bans', ttl: duration }),
   general: new Keyv({ store: logFile, namespace: 'violations' }),
   concurrent: createViolationInstance('concurrent'),
   non_browser: createViolationInstance('non_browser'),
   message_limit: createViolationInstance('message_limit'),
-  token_balance: createViolationInstance('token_balance'),
+  token_balance: createViolationInstance(ViolationTypes.TOKEN_BALANCE),
   registrations: createViolationInstance('registrations'),
+  [ViolationTypes.FILE_UPLOAD_LIMIT]: createViolationInstance(ViolationTypes.FILE_UPLOAD_LIMIT),
+  [ViolationTypes.ILLEGAL_MODEL_REQUEST]: createViolationInstance(
+    ViolationTypes.ILLEGAL_MODEL_REQUEST,
+  ),
   logins: createViolationInstance('logins'),
+  [CacheKeys.ABORT_KEYS]: abortKeys,
+  [CacheKeys.TOKEN_CONFIG]: tokenConfig,
+  [CacheKeys.GEN_TITLE]: genTitle,
+  [CacheKeys.MODEL_QUERIES]: modelQueries,
 };
 
 /**
  * Returns the keyv cache specified by type.
  * If an invalid type is passed, an error will be thrown.
  *
- * @module getLogStores
- * @requires keyv - a simple key-value storage that allows you to easily switch out storage adapters.
- * @requires keyvFiles - a module that includes the logFile and violationFile.
- *
- * @param {string} type - The type of violation, which can be 'concurrent', 'message_limit', 'registrations' or 'logins'.
- * @returns {Keyv} - If a valid type is passed, returns an object containing the logs for violations of the specified type.
- * @throws Will throw an error if an invalid violation type is passed.
+ * @param {string} key - The key for the namespace to access
+ * @returns {Keyv} - If a valid key is passed, returns an object containing the cache store of the specified key.
+ * @throws Will throw an error if an invalid key is passed.
  */
-const getLogStores = (type) => {
-  if (!type || !namespaces[type]) {
-    throw new Error(`Invalid store type: ${type}`);
+const getLogStores = (key) => {
+  if (!key || !namespaces[key]) {
+    throw new Error(`Invalid store key: ${key}`);
   }
-  return namespaces[type];
+  return namespaces[key];
 };
 
 module.exports = getLogStores;

api/cache/keyvRedis.js

@@ -10,10 +10,11 @@ if (REDIS_URI && isEnabled(USE_REDIS)) {
   keyvRedis = new KeyvRedis(REDIS_URI, { useRedisSets: false });
   keyvRedis.on('error', (err) => logger.error('KeyvRedis connection error:', err));
   keyvRedis.setMaxListeners(20);
-} else {
   logger.info(
-    '`REDIS_URI` not provided, or `USE_REDIS` not set. Redis module will not be initialized.',
+    '[Optional] Redis initialized. Note: Redis support is experimental. If you have issues, disable it. Cache needs to be flushed for values to refresh.',
   );
+} else {
+  logger.info('[Optional] Redis not initialized. Note: Redis support is experimental.');
 }
 
 module.exports = keyvRedis;

api/common/enums.js

@@ -1,17 +0,0 @@
-/**
- * @typedef {Object} CacheKeys
- * @property {'config'} CONFIG - Key for the config cache.
- * @property {'plugins'} PLUGINS - Key for the plugins cache.
- * @property {'modelsConfig'} MODELS_CONFIG - Key for the model config cache.
- * @property {'defaultConfig'} DEFAULT_CONFIG - Key for the default config cache.
- * @property {'overrideConfig'} OVERRIDE_CONFIG - Key for the override config cache.
- */
-const CacheKeys = {
-  CONFIG: 'config',
-  PLUGINS: 'plugins',
-  MODELS_CONFIG: 'modelsConfig',
-  DEFAULT_CONFIG: 'defaultConfig',
-  OVERRIDE_CONFIG: 'overrideConfig',
-};
-
-module.exports = { CacheKeys };

api/config/parsers.js

@@ -1,11 +1,16 @@
+const { klona } = require('klona');
 const winston = require('winston');
 const traverse = require('traverse');
-const { klona } = require('klona/full');
 
 const SPLAT_SYMBOL = Symbol.for('splat');
 const MESSAGE_SYMBOL = Symbol.for('message');
 
-const sensitiveKeys = [/^(sk-)[^\s]+/, /(Bearer )[^\s]+/, /(api-key:? )[^\s]+/, /(key=)[^\s]+/];
+const sensitiveKeys = [
+  /^(sk-)[^\s]+/, // OpenAI API key pattern
+  /(Bearer )[^\s]+/, // Header: Bearer token pattern
+  /(api-key:? )[^\s]+/, // Header: API key pattern
+  /(key=)[^\s]+/, // URL query param: sensitive key pattern (Google)
+];
 
 /**
  * Determines if a given value string is sensitive and returns matching regex patterns.
@@ -28,6 +33,10 @@ function getMatchingSensitivePatterns(valueStr) {
  * @returns {string} - The redacted console message.
  */
 function redactMessage(str) {
+  if (!str) {
+    return '';
+  }
+
   const patterns = getMatchingSensitivePatterns(str);
 
   if (patterns.length === 0) {
@@ -102,55 +111,72 @@ const condenseArray = (item) => {
  */
 const debugTraverse = winston.format.printf(({ level, message, timestamp, ...metadata }) => {
   let msg = `${timestamp} ${level}: ${truncateLongStrings(message?.trim(), 150)}`;
-
-  if (level !== 'debug') {
-    return msg;
-  }
-
-  if (!metadata) {
-    return msg;
-  }
-
-  const debugValue = metadata[SPLAT_SYMBOL]?.[0];
-
-  if (!debugValue) {
-    return msg;
-  }
-
-  if (debugValue && Array.isArray(debugValue)) {
-    msg += `\n${JSON.stringify(debugValue.map(condenseArray))}`;
-    return msg;
-  }
-
-  if (typeof debugValue !== 'object') {
-    return (msg += ` ${debugValue}`);
-  }
-
-  msg += '\n{';
-
-  const copy = klona(metadata);
-  traverse(copy).forEach(function (value) {
-    const parent = this.parent;
-    const parentKey = `${parent && parent.notRoot ? parent.key + '.' : ''}`;
-    const tabs = `${parent && parent.notRoot ? '\t\t' : '\t'}`;
-    if (this.isLeaf && typeof value === 'string') {
-      const truncatedText = truncateLongStrings(value);
-      msg += `\n${tabs}${parentKey}${this.key}: ${JSON.stringify(truncatedText)},`;
-    } else if (this.notLeaf && Array.isArray(value) && value.length > 0) {
-      const currentMessage = `\n${tabs}// ${value.length} ${this.key.replace(/s$/, '')}(s)`;
-      this.update(currentMessage, true);
-      msg += currentMessage;
-      const stringifiedArray = value.map(condenseArray);
-      msg += `\n${tabs}${parentKey}${this.key}: [${stringifiedArray}],`;
-    } else if (this.isLeaf && typeof value === 'function') {
-      msg += `\n${tabs}${parentKey}${this.key}: function,`;
-    } else if (this.isLeaf) {
-      msg += `\n${tabs}${parentKey}${this.key}: ${value},`;
+  try {
+    if (level !== 'debug') {
+      return msg;
     }
-  });
 
-  msg += '\n}';
-  return msg;
+    if (!metadata) {
+      return msg;
+    }
+
+    const debugValue = metadata[SPLAT_SYMBOL]?.[0];
+
+    if (!debugValue) {
+      return msg;
+    }
+
+    if (debugValue && Array.isArray(debugValue)) {
+      msg += `\n${JSON.stringify(debugValue.map(condenseArray))}`;
+      return msg;
+    }
+
+    if (typeof debugValue !== 'object') {
+      return (msg += ` ${debugValue}`);
+    }
+
+    msg += '\n{';
+
+    const copy = klona(metadata);
+    traverse(copy).forEach(function (value) {
+      if (typeof this?.key === 'symbol') {
+        return;
+      }
+
+      let _parentKey = '';
+      const parent = this.parent;
+
+      if (typeof parent?.key !== 'symbol' && parent?.key) {
+        _parentKey = parent.key;
+      }
+
+      const parentKey = `${parent && parent.notRoot ? _parentKey + '.' : ''}`;
+
+      const tabs = `${parent && parent.notRoot ? '    ' : '  '}`;
+
+      const currentKey = this?.key ?? 'unknown';
+
+      if (this.isLeaf && typeof value === 'string') {
+        const truncatedText = truncateLongStrings(value);
+        msg += `\n${tabs}${parentKey}${currentKey}: ${JSON.stringify(truncatedText)},`;
+      } else if (this.notLeaf && Array.isArray(value) && value.length > 0) {
+        const currentMessage = `\n${tabs}// ${value.length} ${currentKey.replace(/s$/, '')}(s)`;
+        this.update(currentMessage, true);
+        msg += currentMessage;
+        const stringifiedArray = value.map(condenseArray);
+        msg += `\n${tabs}${parentKey}${currentKey}: [${stringifiedArray}],`;
+      } else if (this.isLeaf && typeof value === 'function') {
+        msg += `\n${tabs}${parentKey}${currentKey}: function,`;
+      } else if (this.isLeaf) {
+        msg += `\n${tabs}${parentKey}${currentKey}: ${value},`;
+      }
+    });
+
+    msg += '\n}';
+    return msg;
+  } catch (e) {
+    return (msg += `\n[LOGGER PARSING ERROR] ${e.message}`);
+  }
 });
 
 module.exports = {

api/config/paths.js

@@ -1,6 +1,12 @@
 const path = require('path');
 
 module.exports = {
+  root: path.resolve(__dirname, '..', '..'),
+  uploads: path.resolve(__dirname, '..', '..', 'uploads'),
+  clientPath: path.resolve(__dirname, '..', '..', 'client'),
+  dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),
   publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),
   imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),
+  structuredTools: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'structured'),
+  pluginManifest: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'manifest.json'),
 };

api/config/winston.js

@@ -5,7 +5,15 @@ const { redactFormat, redactMessage, debugTraverse } = require('./parsers');
 
 const logDir = path.join(__dirname, '..', 'logs');
 
-const { NODE_ENV, DEBUG_LOGGING = true, DEBUG_CONSOLE = false } = process.env;
+const { NODE_ENV, DEBUG_LOGGING = true, DEBUG_CONSOLE = false, CONSOLE_JSON = false } = process.env;
+
+const useConsoleJson =
+  (typeof CONSOLE_JSON === 'string' && CONSOLE_JSON?.toLowerCase() === 'true') ||
+  CONSOLE_JSON === true;
+
+const useDebugConsole =
+  (typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE?.toLowerCase() === 'true') ||
+  DEBUG_CONSOLE === true;
 
 const levels = {
   error: 0,
@@ -33,7 +41,7 @@ const level = () => {
 
 const fileFormat = winston.format.combine(
   redactFormat(),
-  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.timestamp({ format: () => new Date().toISOString() }),
   winston.format.errors({ stack: true }),
   winston.format.splat(),
   // redactErrors(),
@@ -99,14 +107,20 @@ const consoleFormat = winston.format.combine(
   }),
 );
 
-if (
-  (typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE?.toLowerCase() === 'true') ||
-  DEBUG_CONSOLE === true
-) {
+if (useDebugConsole) {
   transports.push(
     new winston.transports.Console({
       level: 'debug',
-      format: winston.format.combine(consoleFormat, debugTraverse),
+      format: useConsoleJson
+        ? winston.format.combine(fileFormat, debugTraverse, winston.format.json())
+        : winston.format.combine(fileFormat, debugTraverse),
+    }),
+  );
+} else if (useConsoleJson) {
+  transports.push(
+    new winston.transports.Console({
+      level: 'info',
+      format: winston.format.combine(fileFormat, winston.format.json()),
     }),
   );
 } else {

api/models/Action.js

@@ -0,0 +1,68 @@
+const mongoose = require('mongoose');
+const actionSchema = require('./schema/action');
+
+const Action = mongoose.model('action', actionSchema);
+
+/**
+ * Update an action with new data without overwriting existing properties,
+ * or create a new action if it doesn't exist.
+ *
+ * @param {Object} searchParams - The search parameters to find the action to update.
+ * @param {string} searchParams.action_id - The ID of the action to update.
+ * @param {string} searchParams.user - The user ID of the action's author.
+ * @param {Object} updateData - An object containing the properties to update.
+ * @returns {Promise<Object>} The updated or newly created action document as a plain object.
+ */
+const updateAction = async (searchParams, updateData) => {
+  return await Action.findOneAndUpdate(searchParams, updateData, {
+    new: true,
+    upsert: true,
+  }).lean();
+};
+
+/**
+ * Retrieves all actions that match the given search parameters.
+ *
+ * @param {Object} searchParams - The search parameters to find matching actions.
+ * @param {boolean} includeSensitive - Flag to include sensitive data in the metadata.
+ * @returns {Promise<Array<Object>>} A promise that resolves to an array of action documents as plain objects.
+ */
+const getActions = async (searchParams, includeSensitive = false) => {
+  const actions = await Action.find(searchParams).lean();
+
+  if (!includeSensitive) {
+    for (let i = 0; i < actions.length; i++) {
+      const metadata = actions[i].metadata;
+      if (!metadata) {
+        continue;
+      }
+
+      const sensitiveFields = ['api_key', 'oauth_client_id', 'oauth_client_secret'];
+      for (let field of sensitiveFields) {
+        if (metadata[field]) {
+          delete metadata[field];
+        }
+      }
+    }
+  }
+
+  return actions;
+};
+
+/**
+ * Deletes an action by its ID.
+ *
+ * @param {Object} searchParams - The search parameters to find the action to update.
+ * @param {string} searchParams.action_id - The ID of the action to update.
+ * @param {string} searchParams.user - The user ID of the action's author.
+ * @returns {Promise<Object>} A promise that resolves to the deleted action document as a plain object, or null if no document was found.
+ */
+const deleteAction = async (searchParams) => {
+  return await Action.findOneAndDelete(searchParams).lean();
+};
+
+module.exports = {
+  updateAction,
+  getActions,
+  deleteAction,
+};

api/models/Assistant.js

@@ -0,0 +1,47 @@
+const mongoose = require('mongoose');
+const assistantSchema = require('./schema/assistant');
+
+const Assistant = mongoose.model('assistant', assistantSchema);
+
+/**
+ * Update an assistant with new data without overwriting existing properties,
+ * or create a new assistant if it doesn't exist.
+ *
+ * @param {Object} searchParams - The search parameters to find the assistant to update.
+ * @param {string} searchParams.assistant_id - The ID of the assistant to update.
+ * @param {string} searchParams.user - The user ID of the assistant's author.
+ * @param {Object} updateData - An object containing the properties to update.
+ * @returns {Promise<Object>} The updated or newly created assistant document as a plain object.
+ */
+const updateAssistant = async (searchParams, updateData) => {
+  return await Assistant.findOneAndUpdate(searchParams, updateData, {
+    new: true,
+    upsert: true,
+  }).lean();
+};
+
+/**
+ * Retrieves an assistant document based on the provided ID.
+ *
+ * @param {Object} searchParams - The search parameters to find the assistant to update.
+ * @param {string} searchParams.assistant_id - The ID of the assistant to update.
+ * @param {string} searchParams.user - The user ID of the assistant's author.
+ * @returns {Promise<Object|null>} The assistant document as a plain object, or null if not found.
+ */
+const getAssistant = async (searchParams) => await Assistant.findOne(searchParams).lean();
+
+/**
+ * Retrieves all assistants that match the given search parameters.
+ *
+ * @param {Object} searchParams - The search parameters to find matching assistants.
+ * @returns {Promise<Array<Object>>} A promise that resolves to an array of action documents as plain objects.
+ */
+const getAssistants = async (searchParams) => {
+  return await Assistant.find(searchParams).lean();
+};
+
+module.exports = {
+  updateAssistant,
+  getAssistants,
+  getAssistant,
+};

api/models/Balance.js

@@ -10,8 +10,9 @@ balanceSchema.statics.check = async function ({
   valueKey,
   tokenType,
   amount,
+  endpointTokenConfig,
 }) {
-  const multiplier = getMultiplier({ valueKey, tokenType, model, endpoint });
+  const multiplier = getMultiplier({ valueKey, tokenType, model, endpoint, endpointTokenConfig });
   const tokenCost = amount * multiplier;
   const { tokenCredits: balance } = (await this.findOne({ user }, 'tokenCredits').lean()) ?? {};
 
@@ -24,6 +25,7 @@ balanceSchema.statics.check = async function ({
     amount,
     balance,
     multiplier,
+    endpointTokenConfig: !!endpointTokenConfig,
   });
 
   if (!balance) {

api/models/Conversation.js

@@ -30,12 +30,12 @@ module.exports = {
       return { message: 'Error saving conversation' };
     }
   },
-  getConvosByPage: async (user, pageNumber = 1, pageSize = 14) => {
+  getConvosByPage: async (user, pageNumber = 1, pageSize = 25) => {
     try {
       const totalConvos = (await Conversation.countDocuments({ user })) || 1;
       const totalPages = Math.ceil(totalConvos / pageSize);
       const convos = await Conversation.find({ user })
-        .sort({ createdAt: -1 })
+        .sort({ updatedAt: -1 })
         .skip((pageNumber - 1) * pageSize)
         .limit(pageSize)
         .lean();
@@ -45,7 +45,7 @@ module.exports = {
       return { message: 'Error getting conversations' };
     }
   },
-  getConvosQueried: async (user, convoIds, pageNumber = 1, pageSize = 14) => {
+  getConvosQueried: async (user, convoIds, pageNumber = 1, pageSize = 25) => {
     try {
       if (!convoIds || convoIds.length === 0) {
         return { conversations: [], pages: 1, pageNumber, pageSize };

api/models/File.js

@@ -14,24 +14,32 @@ const findFileById = async (file_id, options = {}) => {
 };
 
 /**
- * Retrieves files matching a given filter.
+ * Retrieves files matching a given filter, sorted by the most recently updated.
  * @param {Object} filter - The filter criteria to apply.
+ * @param {Object} [_sortOptions] - Optional sort parameters.
  * @returns {Promise<Array<MongoFile>>} A promise that resolves to an array of file documents.
  */
-const getFiles = async (filter) => {
-  return await File.find(filter).lean();
+const getFiles = async (filter, _sortOptions) => {
+  const sortOptions = { updatedAt: -1, ..._sortOptions };
+  return await File.find(filter).sort(sortOptions).lean();
 };
 
 /**
  * Creates a new file with a TTL of 1 hour.
  * @param {MongoFile} data - The file data to be created, must contain file_id.
+ * @param {boolean} disableTTL - Whether to disable the TTL.
  * @returns {Promise<MongoFile>} A promise that resolves to the created file document.
  */
-const createFile = async (data) => {
+const createFile = async (data, disableTTL) => {
   const fileData = {
     ...data,
     expiresAt: new Date(Date.now() + 3600 * 1000),
   };
+
+  if (disableTTL) {
+    delete fileData.expiresAt;
+  }
+
   return await File.findOneAndUpdate({ file_id: data.file_id }, fileData, {
     new: true,
     upsert: true,
@@ -61,7 +69,7 @@ const updateFileUsage = async (data) => {
   const { file_id, inc = 1 } = data;
   const updateOperation = {
     $inc: { usage: inc },
-    $unset: { expiresAt: '' },
+    $unset: { expiresAt: '', temp_file_id: '' },
   };
   return await File.findOneAndUpdate({ file_id }, updateOperation, { new: true }).lean();
 };
@@ -75,6 +83,15 @@ const deleteFile = async (file_id) => {
   return await File.findOneAndDelete({ file_id }).lean();
 };
 
+/**
+ * Deletes a file identified by a filter.
+ * @param {object} filter - The filter criteria to apply.
+ * @returns {Promise<MongoFile>} A promise that resolves to the deleted file document or null.
+ */
+const deleteFileByFilter = async (filter) => {
+  return await File.findOneAndDelete(filter).lean();
+};
+
 /**
  * Deletes multiple files identified by an array of file_ids.
  * @param {Array<string>} file_ids - The unique identifiers of the files to delete.
@@ -93,4 +110,5 @@ module.exports = {
   updateFileUsage,
   deleteFile,
   deleteFiles,
+  deleteFileByFilter,
 };

api/models/Message.js

@@ -9,23 +9,23 @@ module.exports = {
 
   async saveMessage({
     user,
+    endpoint,
     messageId,
     newMessageId,
     conversationId,
     parentMessageId,
     sender,
     text,
-    isCreatedByUser = false,
+    isCreatedByUser,
     error,
     unfinished,
-    cancelled,
     files,
-    isEdited = false,
-    finish_reason = null,
-    tokenCount = null,
-    plugin = null,
-    plugins = null,
-    model = null,
+    isEdited,
+    finish_reason,
+    tokenCount,
+    plugin,
+    plugins,
+    model,
   }) {
     try {
       const validConvoId = idSchema.safeParse(conversationId);
@@ -35,6 +35,7 @@ module.exports = {
 
       const update = {
         user,
+        endpoint,
         messageId: newMessageId || messageId,
         conversationId,
         parentMessageId,
@@ -45,7 +46,6 @@ module.exports = {
         finish_reason,
         error,
         unfinished,
-        cancelled,
         tokenCount,
         plugin,
         plugins,
@@ -72,11 +72,49 @@ module.exports = {
       throw new Error('Failed to save message.');
     }
   },
+  /**
+   * Records a message in the database.
+   *
+   * @async
+   * @function recordMessage
+   * @param {Object} params - The message data object.
+   * @param {string} params.user - The identifier of the user.
+   * @param {string} params.endpoint - The endpoint where the message originated.
+   * @param {string} params.messageId - The unique identifier for the message.
+   * @param {string} params.conversationId - The identifier of the conversation.
+   * @param {string} [params.parentMessageId] - The identifier of the parent message, if any.
+   * @param {Partial<TMessage>} rest - Any additional properties from the TMessage typedef not explicitly listed.
+   * @returns {Promise<Object>} The updated or newly inserted message document.
+   * @throws {Error} If there is an error in saving the message.
+   */
+  async recordMessage({ user, endpoint, messageId, conversationId, parentMessageId, ...rest }) {
+    try {
+      // No parsing of convoId as may use threadId
+      const message = {
+        user,
+        endpoint,
+        messageId,
+        conversationId,
+        parentMessageId,
+        ...rest,
+      };
+
+      return await Message.findOneAndUpdate({ user, messageId }, message, {
+        upsert: true,
+        new: true,
+      });
+    } catch (err) {
+      logger.error('Error saving message:', err);
+      throw new Error('Failed to save message.');
+    }
+  },
   async updateMessage(message) {
     try {
       const { messageId, ...update } = message;
       update.isEdited = true;
-      const updatedMessage = await Message.findOneAndUpdate({ messageId }, update, { new: true });
+      const updatedMessage = await Message.findOneAndUpdate({ messageId }, update, {
+        new: true,
+      });
 
       if (!updatedMessage) {
         throw new Error('Message not found.');

api/models/Transaction.js

@@ -2,6 +2,7 @@ const mongoose = require('mongoose');
 const { isEnabled } = require('../server/utils/handleText');
 const transactionSchema = require('./schema/transaction');
 const { getMultiplier } = require('./tx');
+const { logger } = require('~/config');
 const Balance = require('./Balance');
 const cancelRate = 1.15;
 
@@ -10,8 +11,8 @@ transactionSchema.methods.calculateTokenValue = function () {
   if (!this.valueKey || !this.tokenType) {
     this.tokenValue = this.rawAmount;
   }
-  const { valueKey, tokenType, model } = this;
-  const multiplier = getMultiplier({ valueKey, tokenType, model });
+  const { valueKey, tokenType, model, endpointTokenConfig } = this;
+  const multiplier = getMultiplier({ valueKey, tokenType, model, endpointTokenConfig });
   this.rate = multiplier;
   this.tokenValue = this.rawAmount * multiplier;
   if (this.context && this.tokenType === 'completion' && this.context === 'incomplete') {
@@ -25,6 +26,7 @@ transactionSchema.statics.create = async function (transactionData) {
   const Transaction = this;
 
   const transaction = new Transaction(transactionData);
+  transaction.endpointTokenConfig = transactionData.endpointTokenConfig;
   transaction.calculateTokenValue();
 
   // Save the transaction
@@ -35,11 +37,37 @@ transactionSchema.statics.create = async function (transactionData) {
   }
 
   // Adjust the user's balance
-  return await Balance.findOneAndUpdate(
+  const updatedBalance = await Balance.findOneAndUpdate(
     { user: transaction.user },
     { $inc: { tokenCredits: transaction.tokenValue } },
     { upsert: true, new: true },
   ).lean();
+
+  return {
+    rate: transaction.rate,
+    user: transaction.user.toString(),
+    balance: updatedBalance.tokenCredits,
+    [transaction.tokenType]: transaction.tokenValue,
+  };
 };
 
-module.exports = mongoose.model('Transaction', transactionSchema);
+const Transaction = mongoose.model('Transaction', transactionSchema);
+
+/**
+ * Queries and retrieves transactions based on a given filter.
+ * @async
+ * @function getTransactions
+ * @param {Object} filter - MongoDB filter object to apply when querying transactions.
+ * @returns {Promise<Array>} A promise that resolves to an array of matched transactions.
+ * @throws {Error} Throws an error if querying the database fails.
+ */
+async function getTransactions(filter) {
+  try {
+    return await Transaction.find(filter).lean();
+  } catch (error) {
+    logger.error('Error querying transactions:', error);
+    throw error;
+  }
+}
+
+module.exports = { Transaction, getTransactions };

api/models/checkBalance.js

@@ -1,5 +1,6 @@
+const { ViolationTypes } = require('librechat-data-provider');
+const { logViolation } = require('~/cache');
 const Balance = require('./Balance');
-const { logViolation } = require('../cache');
 /**
  * Checks the balance for a user and determines if they can spend a certain amount.
  * If the user cannot spend the amount, it logs a violation and denies the request.
@@ -14,6 +15,7 @@ const { logViolation } = require('../cache');
  * @param {('prompt' | 'completion')} params.txData.tokenType - The type of token.
  * @param {number} params.txData.amount - The amount of tokens.
  * @param {string} params.txData.model - The model name or identifier.
+ * @param {string} [params.txData.endpointTokenConfig] - The token configuration for the endpoint.
  * @returns {Promise<boolean>} Returns true if the user can spend the amount, otherwise denies the request.
  * @throws {Error} Throws an error if there's an issue with the balance check.
  */
@@ -24,7 +26,7 @@ const checkBalance = async ({ req, res, txData }) => {
     return true;
   }
 
-  const type = 'token_balance';
+  const type = ViolationTypes.TOKEN_BALANCE;
   const errorMessage = {
     type,
     balance,

api/models/index.js

@@ -1,12 +1,14 @@
 const {
   getMessages,
   saveMessage,
+  recordMessage,
   updateMessage,
   deleteMessagesSince,
   deleteMessages,
 } = require('./Message');
 const { getConvoTitle, getConvo, saveConvo, deleteConvos } = require('./Conversation');
 const { getPreset, getPresets, savePreset, deletePresets } = require('./Preset');
+const { hashPassword, getUser, updateUser } = require('./userMethods');
 const {
   findFileById,
   createFile,
@@ -20,17 +22,20 @@ const Key = require('./Key');
 const User = require('./User');
 const Session = require('./Session');
 const Balance = require('./Balance');
-const Transaction = require('./Transaction');
 
 module.exports = {
   User,
   Key,
   Session,
   Balance,
-  Transaction,
+
+  hashPassword,
+  updateUser,
+  getUser,
 
   getMessages,
   saveMessage,
+  recordMessage,
   updateMessage,
   deleteMessagesSince,
   deleteMessages,

api/models/plugins/mongoMeili.js

@@ -183,6 +183,15 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
       if (object.conversationId && object.conversationId.includes('|')) {
         object.conversationId = object.conversationId.replace(/\|/g, '--');
       }
+
+      if (object.content && Array.isArray(object.content)) {
+        object.text = object.content
+          .filter((item) => item.type === 'text' && item.text && item.text.value)
+          .map((item) => item.text.value)
+          .join(' ');
+        delete object.content;
+      }
+
       return object;
     }
 

api/models/schema/action.js

@@ -0,0 +1,59 @@
+const mongoose = require('mongoose');
+
+const { Schema } = mongoose;
+
+const AuthSchema = new Schema(
+  {
+    authorization_type: String,
+    custom_auth_header: String,
+    type: {
+      type: String,
+      enum: ['service_http', 'oauth', 'none'],
+    },
+    authorization_content_type: String,
+    authorization_url: String,
+    client_url: String,
+    scope: String,
+    token_exchange_method: {
+      type: String,
+      enum: ['default_post', 'basic_auth_header', null],
+    },
+  },
+  { _id: false },
+);
+
+const actionSchema = new Schema({
+  user: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'User',
+    index: true,
+    required: true,
+  },
+  action_id: {
+    type: String,
+    index: true,
+    required: true,
+  },
+  type: {
+    type: String,
+    default: 'action_prototype',
+  },
+  settings: Schema.Types.Mixed,
+  assistant_id: String,
+  metadata: {
+    api_key: String, // private, encrypted
+    auth: AuthSchema,
+    domain: {
+      type: String,
+      required: true,
+    },
+    // json_schema: Schema.Types.Mixed,
+    privacy_policy_url: String,
+    raw_spec: String,
+    oauth_client_id: String, // private, encrypted
+    oauth_client_secret: String, // private, encrypted
+  },
+});
+// }, { minimize: false }); // Prevent removal of empty objects
+
+module.exports = actionSchema;

api/models/schema/assistant.js

@@ -0,0 +1,33 @@
+const mongoose = require('mongoose');
+
+const assistantSchema = mongoose.Schema(
+  {
+    user: {
+      type: mongoose.Schema.Types.ObjectId,
+      ref: 'User',
+      required: true,
+    },
+    assistant_id: {
+      type: String,
+      index: true,
+      required: true,
+    },
+    avatar: {
+      type: {
+        filepath: String,
+        source: String,
+      },
+      default: undefined,
+    },
+    access_level: {
+      type: Number,
+    },
+    file_ids: { type: [String], default: undefined },
+    actions: { type: [String], default: undefined },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+module.exports = assistantSchema;

api/models/schema/convoSchema.js

@@ -18,36 +18,29 @@ const convoSchema = mongoose.Schema(
     user: {
       type: String,
       index: true,
-      // default: null,
     },
     messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }],
     // google only
-    examples: [{ type: mongoose.Schema.Types.Mixed }],
+    examples: { type: [{ type: mongoose.Schema.Types.Mixed }], default: undefined },
     agentOptions: {
       type: mongoose.Schema.Types.Mixed,
-      // default: null,
     },
     ...conversationPreset,
     // for bingAI only
     bingConversationId: {
       type: String,
-      // default: null,
     },
     jailbreakConversationId: {
       type: String,
-      // default: null,
     },
     conversationSignature: {
       type: String,
-      // default: null,
     },
     clientId: {
       type: String,
-      // default: null,
     },
     invocationId: {
       type: Number,
-      // default: 1,
     },
   },
   { timestamps: true },
@@ -62,7 +55,7 @@ if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
   });
 }
 
-convoSchema.index({ createdAt: 1 });
+convoSchema.index({ createdAt: 1, updatedAt: 1 });
 
 const Conversation = mongoose.models.Conversation || mongoose.model('Conversation', convoSchema);
 

api/models/schema/defaults.js

@@ -5,150 +5,143 @@ const conversationPreset = {
     default: null,
     required: true,
   },
+  endpointType: {
+    type: String,
+  },
   // for azureOpenAI, openAI, chatGPTBrowser only
   model: {
     type: String,
-    // default: null,
     required: false,
   },
   // for azureOpenAI, openAI only
   chatGptLabel: {
     type: String,
-    // default: null,
     required: false,
   },
   // for google only
   modelLabel: {
     type: String,
-    // default: null,
     required: false,
   },
   promptPrefix: {
     type: String,
-    // default: null,
     required: false,
   },
   temperature: {
     type: Number,
-    // default: 1,
     required: false,
   },
   top_p: {
     type: Number,
-    // default: 1,
     required: false,
   },
   // for google only
   topP: {
     type: Number,
-    // default: 0.95,
     required: false,
   },
   topK: {
     type: Number,
-    // default: 40,
     required: false,
   },
   maxOutputTokens: {
     type: Number,
-    // default: 1024,
     required: false,
   },
   presence_penalty: {
     type: Number,
-    // default: 0,
     required: false,
   },
   frequency_penalty: {
     type: Number,
-    // default: 0,
     required: false,
   },
   // for bingai only
   jailbreak: {
     type: Boolean,
-    // default: false,
   },
   context: {
     type: String,
-    // default: null,
   },
   systemMessage: {
     type: String,
-    // default: null,
   },
   toneStyle: {
     type: String,
-    // default: null,
+  },
+  file_ids: { type: [{ type: String }], default: undefined },
+  // deprecated
+  resendImages: {
+    type: Boolean,
+  },
+  // files
+  resendFiles: {
+    type: Boolean,
+  },
+  imageDetail: {
+    type: String,
+  },
+  /* assistants */
+  assistant_id: {
+    type: String,
+  },
+  instructions: {
+    type: String,
   },
 };
 
 const agentOptions = {
   model: {
     type: String,
-    // default: null,
     required: false,
   },
   // for azureOpenAI, openAI only
   chatGptLabel: {
     type: String,
-    // default: null,
     required: false,
   },
-  // for google only
   modelLabel: {
     type: String,
-    // default: null,
     required: false,
   },
   promptPrefix: {
     type: String,
-    // default: null,
     required: false,
   },
   temperature: {
     type: Number,
-    // default: 1,
     required: false,
   },
   top_p: {
     type: Number,
-    // default: 1,
     required: false,
   },
   // for google only
   topP: {
     type: Number,
-    // default: 0.95,
     required: false,
   },
   topK: {
     type: Number,
-    // default: 40,
     required: false,
   },
   maxOutputTokens: {
     type: Number,
-    // default: 1024,
     required: false,
   },
   presence_penalty: {
     type: Number,
-    // default: 0,
     required: false,
   },
   frequency_penalty: {
     type: Number,
-    // default: 0,
     required: false,
   },
   context: {
     type: String,
-    // default: null,
   },
   systemMessage: {
     type: String,
-    // default: null,
   },
 };
 

api/models/schema/fileSchema.js

@@ -1,7 +1,10 @@
+const { FileSources } = require('librechat-data-provider');
 const mongoose = require('mongoose');
 
 /**
  * @typedef {Object} MongoFile
+ * @property {mongoose.Schema.Types.ObjectId} [_id] - MongoDB Document ID
+ * @property {number} [__v] - MongoDB Version Key
  * @property {mongoose.Schema.Types.ObjectId} user - User ID
  * @property {string} [conversationId] - Optional conversation ID
  * @property {string} file_id - File identifier
@@ -12,9 +15,15 @@ const mongoose = require('mongoose');
  * @property {'file'} object - Type of object, always 'file'
  * @property {string} type - Type of file
  * @property {number} usage - Number of uses of the file
+ * @property {string} [context] - Context of the file origin
+ * @property {boolean} [embedded] - Whether or not the file is embedded in vector db
+ * @property {string} [model] - The model to identify the group region of the file (for Azure OpenAI hosting)
+ * @property {string} [source] - The source of the file
  * @property {number} [width] - Optional width of the file
  * @property {number} [height] - Optional height of the file
  * @property {Date} [expiresAt] - Optional height of the file
+ * @property {Date} [createdAt] - Date when the file was created
+ * @property {Date} [updatedAt] - Date when the file was updated
  */
 const fileSchema = mongoose.Schema(
   {
@@ -42,11 +51,6 @@ const fileSchema = mongoose.Schema(
       type: Number,
       required: true,
     },
-    usage: {
-      type: Number,
-      required: true,
-      default: 0,
-    },
     filename: {
       type: String,
       required: true,
@@ -60,10 +64,29 @@ const fileSchema = mongoose.Schema(
       required: true,
       default: 'file',
     },
+    embedded: {
+      type: Boolean,
+    },
     type: {
       type: String,
       required: true,
     },
+    context: {
+      type: String,
+      // required: true,
+    },
+    usage: {
+      type: Number,
+      required: true,
+      default: 0,
+    },
+    source: {
+      type: String,
+      default: FileSources.local,
+    },
+    model: {
+      type: String,
+    },
     width: Number,
     height: Number,
     expiresAt: {

api/models/schema/messageSchema.js

@@ -17,14 +17,18 @@ const messageSchema = mongoose.Schema(
     user: {
       type: String,
       index: true,
+      required: true,
       default: null,
     },
     model: {
       type: String,
+      default: null,
+    },
+    endpoint: {
+      type: String,
     },
     conversationSignature: {
       type: String,
-      // required: true
     },
     clientId: {
       type: String,
@@ -34,7 +38,6 @@ const messageSchema = mongoose.Schema(
     },
     parentMessageId: {
       type: String,
-      // required: true
     },
     tokenCount: {
       type: Number,
@@ -44,12 +47,10 @@ const messageSchema = mongoose.Schema(
     },
     sender: {
       type: String,
-      required: true,
       meiliIndex: true,
     },
     text: {
       type: String,
-      required: true,
       meiliIndex: true,
     },
     summary: {
@@ -68,10 +69,6 @@ const messageSchema = mongoose.Schema(
       type: Boolean,
       default: false,
     },
-    cancelled: {
-      type: Boolean,
-      default: false,
-    },
     error: {
       type: Boolean,
       default: false,
@@ -85,22 +82,34 @@ const messageSchema = mongoose.Schema(
       select: false,
       default: false,
     },
-    files: [{ type: mongoose.Schema.Types.Mixed }],
+    files: { type: [{ type: mongoose.Schema.Types.Mixed }], default: undefined },
     plugin: {
-      latest: {
-        type: String,
-        required: false,
-      },
-      inputs: {
-        type: [mongoose.Schema.Types.Mixed],
-        required: false,
-      },
-      outputs: {
-        type: String,
-        required: false,
+      type: {
+        latest: {
+          type: String,
+          required: false,
+        },
+        inputs: {
+          type: [mongoose.Schema.Types.Mixed],
+          required: false,
+          default: undefined,
+        },
+        outputs: {
+          type: String,
+          required: false,
+        },
       },
+      default: undefined,
+    },
+    plugins: { type: [{ type: mongoose.Schema.Types.Mixed }], default: undefined },
+    content: {
+      type: [{ type: mongoose.Schema.Types.Mixed }],
+      default: undefined,
+      meiliIndex: true,
+    },
+    thread_id: {
+      type: String,
     },
-    plugins: [{ type: mongoose.Schema.Types.Mixed }],
   },
   { timestamps: true },
 );

api/models/spendTokens.js

@@ -1,4 +1,4 @@
-const Transaction = require('./Transaction');
+const { Transaction } = require('./Transaction');
 const { logger } = require('~/config');
 
 /**
@@ -11,6 +11,7 @@ const { logger } = require('~/config');
  * @param {String} txData.conversationId - The ID of the conversation.
  * @param {String} txData.model - The model name.
  * @param {String} txData.context - The context in which the transaction is made.
+ * @param {String} [txData.endpointTokenConfig] - The current endpoint token config.
  * @param {String} [txData.valueKey] - The value key (optional).
  * @param {Object} tokenUsage - The number of tokens used.
  * @param {Number} tokenUsage.promptTokens - The number of prompt tokens used.
@@ -20,6 +21,15 @@ const { logger } = require('~/config');
  */
 const spendTokens = async (txData, tokenUsage) => {
   const { promptTokens, completionTokens } = tokenUsage;
+  logger.debug(
+    `[spendTokens] conversationId: ${txData.conversationId}${
+      txData?.context ? ` | Context: ${txData?.context}` : ''
+    } | Token usage: `,
+    {
+      promptTokens,
+      completionTokens,
+    },
+  );
   let prompt, completion;
   try {
     if (promptTokens >= 0) {
@@ -41,7 +51,16 @@ const spendTokens = async (txData, tokenUsage) => {
       rawAmount: -completionTokens,
     });
 
-    logger.debug('[spendTokens] post-transaction', { prompt, completion });
+    prompt &&
+      completion &&
+      logger.debug('[spendTokens] Transaction data record against balance:', {
+        user: prompt.user,
+        prompt: prompt.prompt,
+        promptRate: prompt.rate,
+        completion: completion.completion,
+        completionRate: completion.rate,
+        balance: completion.balance,
+      });
   } catch (err) {
     logger.error('[spendTokens]', err);
   }

api/models/tx.js

@@ -12,6 +12,13 @@ const tokenValues = {
   '16k': { prompt: 3, completion: 4 },
   'gpt-3.5-turbo-1106': { prompt: 1, completion: 2 },
   'gpt-4-1106': { prompt: 10, completion: 30 },
+  'gpt-3.5-turbo-0125': { prompt: 0.5, completion: 1.5 },
+  'claude-3-opus': { prompt: 15, completion: 75 },
+  'claude-3-sonnet': { prompt: 3, completion: 15 },
+  'claude-3-haiku': { prompt: 0.25, completion: 1.25 },
+  'claude-2.1': { prompt: 8, completion: 24 },
+  'claude-2': { prompt: 8, completion: 24 },
+  'claude-': { prompt: 0.8, completion: 2.4 },
 };
 
 /**
@@ -29,16 +36,24 @@ const getValueKey = (model, endpoint) => {
 
   if (modelName.includes('gpt-3.5-turbo-16k')) {
     return '16k';
+  } else if (modelName.includes('gpt-3.5-turbo-0125')) {
+    return 'gpt-3.5-turbo-0125';
   } else if (modelName.includes('gpt-3.5-turbo-1106')) {
     return 'gpt-3.5-turbo-1106';
   } else if (modelName.includes('gpt-3.5')) {
     return '4k';
   } else if (modelName.includes('gpt-4-1106')) {
     return 'gpt-4-1106';
+  } else if (modelName.includes('gpt-4-0125')) {
+    return 'gpt-4-1106';
+  } else if (modelName.includes('gpt-4-turbo')) {
+    return 'gpt-4-1106';
   } else if (modelName.includes('gpt-4-32k')) {
     return '32k';
   } else if (modelName.includes('gpt-4')) {
     return '8k';
+  } else if (tokenValues[modelName]) {
+    return modelName;
   }
 
   return undefined;
@@ -53,9 +68,14 @@ const getValueKey = (model, endpoint) => {
  * @param {string} [params.tokenType] - The type of token (e.g., 'prompt' or 'completion').
  * @param {string} [params.model] - The model name to derive the value key from if not provided.
  * @param {string} [params.endpoint] - The endpoint name to derive the value key from if not provided.
+ * @param {EndpointTokenConfig} [params.endpointTokenConfig] - The token configuration for the endpoint.
  * @returns {number} The multiplier for the given parameters, or a default value if not found.
  */
-const getMultiplier = ({ valueKey, tokenType, model, endpoint }) => {
+const getMultiplier = ({ valueKey, tokenType, model, endpoint, endpointTokenConfig }) => {
+  if (endpointTokenConfig) {
+    return endpointTokenConfig?.[model]?.[tokenType] ?? defaultRate;
+  }
+
   if (valueKey && tokenType) {
     return tokenValues[valueKey][tokenType] ?? defaultRate;
   }

api/models/tx.spec.js

@@ -84,6 +84,15 @@ describe('getMultiplier', () => {
     expect(getMultiplier({ tokenType: 'completion', model: 'gpt-4-1106-vision-preview' })).toBe(
       tokenValues['gpt-4-1106'].completion,
     );
+    expect(getMultiplier({ tokenType: 'completion', model: 'gpt-4-0125-preview' })).toBe(
+      tokenValues['gpt-4-1106'].completion,
+    );
+    expect(getMultiplier({ tokenType: 'completion', model: 'gpt-4-turbo-vision-preview' })).toBe(
+      tokenValues['gpt-4-1106'].completion,
+    );
+    expect(getMultiplier({ tokenType: 'completion', model: 'gpt-3.5-turbo-0125' })).toBe(
+      tokenValues['gpt-3.5-turbo-0125'].completion,
+    );
   });
 
   it('should return defaultRate if derived valueKey does not match any known patterns', () => {

api/models/userMethods.js

@@ -0,0 +1,46 @@
+const bcrypt = require('bcryptjs');
+const User = require('./User');
+
+const hashPassword = async (password) => {
+  const hashedPassword = await new Promise((resolve, reject) => {
+    bcrypt.hash(password, 10, function (err, hash) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(hash);
+      }
+    });
+  });
+
+  return hashedPassword;
+};
+
+/**
+ * Retrieve a user by ID and convert the found user document to a plain object.
+ *
+ * @param {string} userId - The ID of the user to find and return as a plain object.
+ * @returns {Promise<Object>} A plain object representing the user document, or `null` if no user is found.
+ */
+const getUser = async function (userId) {
+  return await User.findById(userId).lean();
+};
+
+/**
+ * Update a user with new data without overwriting existing properties.
+ *
+ * @param {string} userId - The ID of the user to update.
+ * @param {Object} updateData - An object containing the properties to update.
+ * @returns {Promise<Object>} The updated user document as a plain object, or `null` if no user is found.
+ */
+const updateUser = async function (userId, updateData) {
+  return await User.findByIdAndUpdate(userId, updateData, {
+    new: true,
+    runValidators: true,
+  }).lean();
+};
+
+module.exports = {
+  hashPassword,
+  updateUser,
+  getUser,
+};

api/package.json

@@ -1,13 +1,19 @@
 {
   "name": "@librechat/backend",
-  "version": "0.6.5",
+  "version": "0.7.0",
   "description": "",
   "scripts": {
     "start": "echo 'please run this from the root directory'",
     "server-dev": "echo 'please run this from the root directory'",
     "test": "cross-env NODE_ENV=test jest",
     "b:test": "NODE_ENV=test bun jest",
-    "test:ci": "jest --ci"
+    "test:ci": "jest --ci",
+    "add-balance": "node ./add-balance.js",
+    "list-balances": "node ./list-balances.js",
+    "user-stats": "node ./user-stats.js",
+    "create-user": "node ./create-user.js",
+    "ban-user": "node ./ban-user.js",
+    "delete-user": "node ./delete-user.js"
   },
   "repository": {
     "type": "git",
@@ -25,13 +31,14 @@
   "bugs": {
     "url": "https://github.com/danny-avila/LibreChat/issues"
   },
-  "homepage": "https://github.com/danny-avila/LibreChat#readme",
+  "homepage": "https://librechat.ai",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.5.4",
+    "@anthropic-ai/sdk": "^0.16.1",
     "@azure/search-documents": "^12.0.0",
     "@keyv/mongo": "^2.1.8",
     "@keyv/redis": "^2.8.1",
-    "@langchain/google-genai": "^0.0.2",
+    "@langchain/community": "^0.0.17",
+    "@langchain/google-genai": "^0.0.8",
     "axios": "^1.3.4",
     "bcryptjs": "^2.4.3",
     "cheerio": "^1.0.0-rc.12",
@@ -44,6 +51,8 @@
     "express-mongo-sanitize": "^2.2.0",
     "express-rate-limit": "^6.9.0",
     "express-session": "^1.17.3",
+    "file-type": "^18.7.0",
+    "firebase": "^10.8.0",
     "googleapis": "^126.0.1",
     "handlebars": "^4.7.7",
     "html": "^1.0.0",
@@ -53,16 +62,17 @@
     "keyv": "^4.5.4",
     "keyv-file": "^0.2.0",
     "klona": "^2.0.6",
-    "langchain": "^0.0.186",
+    "langchain": "^0.0.214",
     "librechat-data-provider": "*",
     "lodash": "^4.17.21",
-    "meilisearch": "^0.33.0",
+    "meilisearch": "^0.38.0",
+    "mime": "^3.0.0",
     "module-alias": "^2.2.3",
     "mongoose": "^7.1.1",
     "multer": "^1.4.5-lts.1",
     "nodejs-gpt": "^1.37.4",
     "nodemailer": "^6.9.4",
-    "openai": "^4.20.1",
+    "openai": "^4.29.0",
     "openai-chat-tokens": "^0.2.8",
     "openid-client": "^5.4.2",
     "passport": "^0.6.0",

api/server/controllers/AskController.js

@@ -1,7 +1,8 @@
-const { getResponseSender } = require('librechat-data-provider');
-const { sendMessage, createOnProgress } = require('~/server/utils');
-const { saveMessage, getConvoTitle, getConvo } = require('~/models');
+const throttle = require('lodash/throttle');
+const { getResponseSender, Constants } = require('librechat-data-provider');
 const { createAbortController, handleAbortError } = require('~/server/middleware');
+const { sendMessage, createOnProgress } = require('~/server/utils');
+const { saveMessage, getConvo } = require('~/models');
 const { logger } = require('~/config');
 
 const AskController = async (req, res, next, initializeClient, addTitle) => {
@@ -9,25 +10,25 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
     text,
     endpointOption,
     conversationId,
+    modelDisplayLabel,
     parentMessageId = null,
     overrideParentMessageId = null,
   } = req.body;
 
   logger.debug('[AskController]', { text, conversationId, ...endpointOption });
 
-  let metadata;
   let userMessage;
   let promptTokens;
   let userMessageId;
   let responseMessageId;
-  let lastSavedTimestamp = 0;
-  let saveDelay = 100;
-  const sender = getResponseSender({ ...endpointOption, model: endpointOption.modelOptions.model });
+  const sender = getResponseSender({
+    ...endpointOption,
+    model: endpointOption.modelOptions.model,
+    modelDisplayLabel,
+  });
   const newConvo = !conversationId;
   const user = req.user.id;
 
-  const addMetadata = (data) => (metadata = data);
-
   const getReqData = (data = {}) => {
     for (let key in data) {
       if (key === 'userMessage') {
@@ -49,11 +50,8 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
     const { client } = await initializeClient({ req, res, endpointOption });
 
     const { onProgress: progressCallback, getPartialText } = createOnProgress({
-      onProgress: ({ text: partialText }) => {
-        const currentTimestamp = Date.now();
-
-        if (currentTimestamp - lastSavedTimestamp > saveDelay) {
-          lastSavedTimestamp = currentTimestamp;
+      onProgress: throttle(
+        ({ text: partialText }) => {
           saveMessage({
             messageId: responseMessageId,
             sender,
@@ -62,16 +60,13 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
             text: partialText,
             model: client.modelOptions.model,
             unfinished: true,
-            cancelled: false,
             error: false,
             user,
           });
-        }
-
-        if (saveDelay < 500) {
-          saveDelay = 500;
-        }
-      },
+        },
+        3000,
+        { trailing: false },
+      ),
     });
 
     getText = getPartialText;
@@ -88,6 +83,20 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
 
     const { abortController, onStart } = createAbortController(req, res, getAbortData);
 
+    res.on('close', () => {
+      logger.debug('[AskController] Request closed');
+      if (!abortController) {
+        return;
+      } else if (abortController.signal.aborted) {
+        return;
+      } else if (abortController.requestCompleted) {
+        return;
+      }
+
+      abortController.abort();
+      logger.debug('[AskController] Request aborted on close');
+    });
+
     const messageOptions = {
       user,
       parentMessageId,
@@ -95,7 +104,6 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
       overrideParentMessageId,
       getReqData,
       onStart,
-      addMetadata,
       abortController,
       onProgress: progressCallback.call(null, {
         res,
@@ -110,28 +118,34 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
       response.parentMessageId = overrideParentMessageId;
     }
 
-    if (metadata) {
-      response = { ...response, ...metadata };
-    }
+    response.endpoint = endpointOption.endpoint;
+
+    const conversation = await getConvo(user, conversationId);
+    conversation.title =
+      conversation && !conversation.title ? null : conversation?.title || 'New Chat';
 
     if (client.options.attachments) {
       userMessage.files = client.options.attachments;
+      conversation.model = endpointOption.modelOptions.model;
       delete userMessage.image_urls;
     }
 
-    sendMessage(res, {
-      title: await getConvoTitle(user, conversationId),
-      final: true,
-      conversation: await getConvo(user, conversationId),
-      requestMessage: userMessage,
-      responseMessage: response,
-    });
-    res.end();
+    if (!abortController.signal.aborted) {
+      sendMessage(res, {
+        final: true,
+        conversation,
+        title: conversation.title,
+        requestMessage: userMessage,
+        responseMessage: response,
+      });
+      res.end();
+
+      await saveMessage({ ...response, user });
+    }
 
-    await saveMessage({ ...response, user });
     await saveMessage(userMessage);
 
-    if (addTitle && parentMessageId === '00000000-0000-0000-0000-000000000000' && newConvo) {
+    if (addTitle && parentMessageId === Constants.NO_PARENT && newConvo) {
       addTitle(req, {
         text,
         response,

api/server/controllers/EditController.js

@@ -1,7 +1,8 @@
+const throttle = require('lodash/throttle');
 const { getResponseSender } = require('librechat-data-provider');
-const { sendMessage, createOnProgress } = require('~/server/utils');
-const { saveMessage, getConvoTitle, getConvo } = require('~/models');
 const { createAbortController, handleAbortError } = require('~/server/middleware');
+const { sendMessage, createOnProgress } = require('~/server/utils');
+const { saveMessage, getConvo } = require('~/models');
 const { logger } = require('~/config');
 
 const EditController = async (req, res, next, initializeClient) => {
@@ -10,6 +11,7 @@ const EditController = async (req, res, next, initializeClient) => {
     generation,
     endpointOption,
     conversationId,
+    modelDisplayLabel,
     responseMessageId,
     isContinued = false,
     parentMessageId = null,
@@ -24,16 +26,16 @@ const EditController = async (req, res, next, initializeClient) => {
     ...endpointOption,
   });
 
-  let metadata;
   let userMessage;
   let promptTokens;
-  let lastSavedTimestamp = 0;
-  let saveDelay = 100;
-  const sender = getResponseSender({ ...endpointOption, model: endpointOption.modelOptions.model });
+  const sender = getResponseSender({
+    ...endpointOption,
+    model: endpointOption.modelOptions.model,
+    modelDisplayLabel,
+  });
   const userMessageId = parentMessageId;
   const user = req.user.id;
 
-  const addMetadata = (data) => (metadata = data);
   const getReqData = (data = {}) => {
     for (let key in data) {
       if (key === 'userMessage') {
@@ -48,11 +50,8 @@ const EditController = async (req, res, next, initializeClient) => {
 
   const { onProgress: progressCallback, getPartialText } = createOnProgress({
     generation,
-    onProgress: ({ text: partialText }) => {
-      const currentTimestamp = Date.now();
-
-      if (currentTimestamp - lastSavedTimestamp > saveDelay) {
-        lastSavedTimestamp = currentTimestamp;
+    onProgress: throttle(
+      ({ text: partialText }) => {
         saveMessage({
           messageId: responseMessageId,
           sender,
@@ -61,17 +60,14 @@ const EditController = async (req, res, next, initializeClient) => {
           text: partialText,
           model: endpointOption.modelOptions.model,
           unfinished: true,
-          cancelled: false,
           isEdited: true,
           error: false,
           user,
         });
-      }
-
-      if (saveDelay < 500) {
-        saveDelay = 500;
-      }
-    },
+      },
+      3000,
+      { trailing: false },
+    ),
   });
 
   const getAbortData = () => ({
@@ -86,6 +82,20 @@ const EditController = async (req, res, next, initializeClient) => {
 
   const { abortController, onStart } = createAbortController(req, res, getAbortData);
 
+  res.on('close', () => {
+    logger.debug('[EditController] Request closed');
+    if (!abortController) {
+      return;
+    } else if (abortController.signal.aborted) {
+      return;
+    } else if (abortController.requestCompleted) {
+      return;
+    }
+
+    abortController.abort();
+    logger.debug('[EditController] Request aborted on close');
+  });
+
   try {
     const { client } = await initializeClient({ req, res, endpointOption });
 
@@ -100,7 +110,6 @@ const EditController = async (req, res, next, initializeClient) => {
       overrideParentMessageId,
       getReqData,
       onStart,
-      addMetadata,
       abortController,
       onProgress: progressCallback.call(null, {
         res,
@@ -109,20 +118,26 @@ const EditController = async (req, res, next, initializeClient) => {
       }),
     });
 
-    if (metadata) {
-      response = { ...response, ...metadata };
+    const conversation = await getConvo(user, conversationId);
+    conversation.title =
+      conversation && !conversation.title ? null : conversation?.title || 'New Chat';
+
+    if (client.options.attachments) {
+      conversation.model = endpointOption.modelOptions.model;
     }
 
-    await saveMessage({ ...response, user });
+    if (!abortController.signal.aborted) {
+      sendMessage(res, {
+        final: true,
+        conversation,
+        title: conversation.title,
+        requestMessage: userMessage,
+        responseMessage: response,
+      });
+      res.end();
 
-    sendMessage(res, {
-      title: await getConvoTitle(user, conversationId),
-      final: true,
-      conversation: await getConvo(user, conversationId),
-      requestMessage: userMessage,
-      responseMessage: response,
-    });
-    res.end();
+      await saveMessage({ ...response, user });
+    }
   } catch (error) {
     const partialText = getPartialText();
     handleAbortError(res, req, error, {

api/server/controllers/EndpointController.js

@@ -1,17 +1,35 @@
+const { CacheKeys, EModelEndpoint, orderEndpointsConfig } = require('librechat-data-provider');
+const { loadDefaultEndpointsConfig, loadConfigEndpoints } = require('~/server/services/Config');
 const { getLogStores } = require('~/cache');
-const { CacheKeys } = require('~/common/enums');
-const { loadDefaultEndpointsConfig } = require('~/server/services/Config');
 
 async function endpointController(req, res) {
-  const cache = getLogStores(CacheKeys.CONFIG);
-  const config = await cache.get(CacheKeys.DEFAULT_CONFIG);
-  if (config) {
-    res.send(config);
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  const cachedEndpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG);
+  if (cachedEndpointsConfig) {
+    res.send(cachedEndpointsConfig);
     return;
   }
-  const defaultConfig = await loadDefaultEndpointsConfig();
-  await cache.set(CacheKeys.DEFAULT_CONFIG, defaultConfig);
-  res.send(JSON.stringify(defaultConfig));
+
+  const defaultEndpointsConfig = await loadDefaultEndpointsConfig(req);
+  const customConfigEndpoints = await loadConfigEndpoints(req);
+
+  /** @type {TEndpointsConfig} */
+  const mergedConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints };
+  if (mergedConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) {
+    const { disableBuilder, retrievalModels, capabilities, ..._rest } =
+      req.app.locals[EModelEndpoint.assistants];
+    mergedConfig[EModelEndpoint.assistants] = {
+      ...mergedConfig[EModelEndpoint.assistants],
+      retrievalModels,
+      disableBuilder,
+      capabilities,
+    };
+  }
+
+  const endpointsConfig = orderEndpointsConfig(mergedConfig);
+
+  await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig);
+  res.send(JSON.stringify(endpointsConfig));
 }
 
 module.exports = endpointController;

api/server/controllers/ErrorController.js

@@ -3,23 +3,24 @@ const { logger } = require('~/config');
 //handle duplicates
 const handleDuplicateKeyError = (err, res) => {
   logger.error('Duplicate key error:', err.keyValue);
-  const field = Object.keys(err.keyValue);
+  const field = `${JSON.stringify(Object.keys(err.keyValue))}`;
   const code = 409;
-  const error = `An document with that ${field} already exists.`;
-  res.status(code).send({ messages: error, fields: field });
+  res
+    .status(code)
+    .send({ messages: `An document with that ${field} already exists.`, fields: field });
 };
 
 //handle validation errors
 const handleValidationError = (err, res) => {
   logger.error('Validation error:', err.errors);
   let errors = Object.values(err.errors).map((el) => el.message);
-  let fields = Object.values(err.errors).map((el) => el.path);
+  let fields = `${JSON.stringify(Object.values(err.errors).map((el) => el.path))}`;
   let code = 400;
   if (errors.length > 1) {
-    const formattedErrors = errors.join(' ');
-    res.status(code).send({ messages: formattedErrors, fields: fields });
+    errors = errors.join(' ');
+    res.status(code).send({ messages: `${JSON.stringify(errors)}`, fields: fields });
   } else {
-    res.status(code).send({ messages: errors, fields: fields });
+    res.status(code).send({ messages: `${JSON.stringify(errors)}`, fields: fields });
   }
 };
 

api/server/controllers/ModelController.js

@@ -1,17 +1,40 @@
+const { CacheKeys } = require('librechat-data-provider');
+const { loadDefaultModels, loadConfigModels } = require('~/server/services/Config');
 const { getLogStores } = require('~/cache');
-const { CacheKeys } = require('~/common/enums');
-const { loadDefaultModels } = require('~/server/services/Config');
+
+const getModelsConfig = async (req) => {
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  let modelsConfig = await cache.get(CacheKeys.MODELS_CONFIG);
+  if (!modelsConfig) {
+    modelsConfig = await loadModels(req);
+  }
+
+  return modelsConfig;
+};
+
+/**
+ * Loads the models from the config.
+ * @param {Express.Request} req - The Express request object.
+ * @returns {Promise<TModelsConfig>} The models config.
+ */
+async function loadModels(req) {
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  const cachedModelsConfig = await cache.get(CacheKeys.MODELS_CONFIG);
+  if (cachedModelsConfig) {
+    return cachedModelsConfig;
+  }
+  const defaultModelsConfig = await loadDefaultModels(req);
+  const customModelsConfig = await loadConfigModels(req);
+
+  const modelConfig = { ...defaultModelsConfig, ...customModelsConfig };
+
+  await cache.set(CacheKeys.MODELS_CONFIG, modelConfig);
+  return modelConfig;
+}
 
 async function modelController(req, res) {
-  const cache = getLogStores(CacheKeys.CONFIG);
-  let modelConfig = await cache.get(CacheKeys.MODELS_CONFIG);
-  if (modelConfig) {
-    res.send(modelConfig);
-    return;
-  }
-  modelConfig = await loadDefaultModels();
-  await cache.set(CacheKeys.MODELS_CONFIG, modelConfig);
+  const modelConfig = await loadModels(req);
   res.send(modelConfig);
 }
 
-module.exports = modelController;
+module.exports = { modelController, loadModels, getModelsConfig };

api/server/controllers/OverrideController.js

@@ -1,9 +1,9 @@
-const { getLogStores } = require('~/cache');
-const { CacheKeys } = require('~/common/enums');
+const { CacheKeys } = require('librechat-data-provider');
 const { loadOverrideConfig } = require('~/server/services/Config');
+const { getLogStores } = require('~/cache');
 
 async function overrideController(req, res) {
-  const cache = getLogStores(CacheKeys.CONFIG);
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
   let overrideConfig = await cache.get(CacheKeys.OVERRIDE_CONFIG);
   if (overrideConfig) {
     res.send(overrideConfig);
@@ -15,7 +15,7 @@ async function overrideController(req, res) {
   overrideConfig = await loadOverrideConfig();
   const { endpointsConfig, modelsConfig } = overrideConfig;
   if (endpointsConfig) {
-    await cache.set(CacheKeys.DEFAULT_CONFIG, endpointsConfig);
+    await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig);
   }
   if (modelsConfig) {
     await cache.set(CacheKeys.MODELS_CONFIG, modelsConfig);

api/server/controllers/PluginController.js

@@ -1,9 +1,14 @@
-const path = require('path');
 const { promises: fs } = require('fs');
+const { CacheKeys } = require('librechat-data-provider');
 const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
-const { CacheKeys } = require('~/common/enums');
 const { getLogStores } = require('~/cache');
 
+/**
+ * Filters out duplicate plugins from the list of plugins.
+ *
+ * @param {TPlugin[]} plugins The list of plugins to filter.
+ * @returns {TPlugin[]} The list of plugins with duplicates removed.
+ */
 const filterUniquePlugins = (plugins) => {
   const seen = new Set();
   return plugins.filter((plugin) => {
@@ -13,35 +18,47 @@ const filterUniquePlugins = (plugins) => {
   });
 };
 
+/**
+ * Determines if a plugin is authenticated by checking if all required authentication fields have non-empty values.
+ * Supports alternate authentication fields, allowing validation against multiple possible environment variables.
+ *
+ * @param {TPlugin} plugin The plugin object containing the authentication configuration.
+ * @returns {boolean} True if the plugin is authenticated for all required fields, false otherwise.
+ */
 const isPluginAuthenticated = (plugin) => {
   if (!plugin.authConfig || plugin.authConfig.length === 0) {
     return false;
   }
 
   return plugin.authConfig.every((authFieldObj) => {
-    const envValue = process.env[authFieldObj.authField];
-    if (envValue === 'user_provided') {
-      return false;
+    const authFieldOptions = authFieldObj.authField.split('||');
+    let isFieldAuthenticated = false;
+
+    for (const fieldOption of authFieldOptions) {
+      const envValue = process.env[fieldOption];
+      if (envValue && envValue.trim() !== '' && envValue !== 'user_provided') {
+        isFieldAuthenticated = true;
+        break;
+      }
     }
-    return envValue && envValue.trim() !== '';
+
+    return isFieldAuthenticated;
   });
 };
 
 const getAvailablePluginsController = async (req, res) => {
   try {
-    const cache = getLogStores(CacheKeys.CONFIG);
+    const cache = getLogStores(CacheKeys.CONFIG_STORE);
     const cachedPlugins = await cache.get(CacheKeys.PLUGINS);
     if (cachedPlugins) {
       res.status(200).json(cachedPlugins);
       return;
     }
 
-    const manifestFile = await fs.readFile(
-      path.join(__dirname, '..', '..', 'app', 'clients', 'tools', 'manifest.json'),
-      'utf8',
-    );
+    const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
 
-    const jsonData = JSON.parse(manifestFile);
+    const jsonData = JSON.parse(pluginManifest);
+    /** @type {TPlugin[]} */
     const uniquePlugins = filterUniquePlugins(jsonData);
     const authenticatedPlugins = uniquePlugins.map((plugin) => {
       if (isPluginAuthenticated(plugin)) {
@@ -58,6 +75,53 @@ const getAvailablePluginsController = async (req, res) => {
   }
 };
 
+/**
+ * Retrieves and returns a list of available tools, either from a cache or by reading a plugin manifest file.
+ *
+ * This function first attempts to retrieve the list of tools from a cache. If the tools are not found in the cache,
+ * it reads a plugin manifest file, filters for unique plugins, and determines if each plugin is authenticated.
+ * Only plugins that are marked as available in the application's local state are included in the final list.
+ * The resulting list of tools is then cached and sent to the client.
+ *
+ * @param {object} req - The request object, containing information about the HTTP request.
+ * @param {object} res - The response object, used to send back the desired HTTP response.
+ * @returns {Promise<void>} A promise that resolves when the function has completed.
+ */
+const getAvailableTools = async (req, res) => {
+  try {
+    const cache = getLogStores(CacheKeys.CONFIG_STORE);
+    const cachedTools = await cache.get(CacheKeys.TOOLS);
+    if (cachedTools) {
+      res.status(200).json(cachedTools);
+      return;
+    }
+
+    const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
+
+    const jsonData = JSON.parse(pluginManifest);
+    /** @type {TPlugin[]} */
+    const uniquePlugins = filterUniquePlugins(jsonData);
+
+    const authenticatedPlugins = uniquePlugins.map((plugin) => {
+      if (isPluginAuthenticated(plugin)) {
+        return { ...plugin, authenticated: true };
+      } else {
+        return plugin;
+      }
+    });
+
+    const tools = authenticatedPlugins.filter(
+      (plugin) => req.app.locals.availableTools[plugin.pluginKey] !== undefined,
+    );
+
+    await cache.set(CacheKeys.TOOLS, tools);
+    res.status(200).json(tools);
+  } catch (error) {
+    res.status(500).json({ message: error.message });
+  }
+};
+
 module.exports = {
+  getAvailableTools,
   getAvailablePluginsController,
 };

api/server/controllers/UserController.js

@@ -8,16 +8,19 @@ const getUserController = async (req, res) => {
 
 const updateUserPluginsController = async (req, res) => {
   const { user } = req;
-  const { pluginKey, action, auth } = req.body;
+  const { pluginKey, action, auth, isAssistantTool } = req.body;
   let authService;
   try {
-    const userPluginsService = await updateUserPluginsService(user, pluginKey, action);
+    if (!isAssistantTool) {
+      const userPluginsService = await updateUserPluginsService(user, pluginKey, action);
 
-    if (userPluginsService instanceof Error) {
-      logger.error('[userPluginsService]', userPluginsService);
-      const { status, message } = userPluginsService;
-      res.status(status).send({ message });
+      if (userPluginsService instanceof Error) {
+        logger.error('[userPluginsService]', userPluginsService);
+        const { status, message } = userPluginsService;
+        res.status(status).send({ message });
+      }
     }
+
     if (auth) {
       const keys = Object.keys(auth);
       const values = Object.values(auth);

api/server/index.js

@@ -1,45 +1,55 @@
+require('dotenv').config();
 const path = require('path');
 require('module-alias')({ base: path.resolve(__dirname, '..') });
 const cors = require('cors');
+const axios = require('axios');
 const express = require('express');
 const passport = require('passport');
 const mongoSanitize = require('express-mongo-sanitize');
 const errorController = require('./controllers/ErrorController');
+const { jwtLogin, passportLogin } = require('~/strategies');
 const configureSocialLogins = require('./socialLogins');
 const { connectDb, indexSync } = require('~/lib/db');
+const AppService = require('./services/AppService');
+const noIndex = require('./middleware/noIndex');
+const { isEnabled } = require('~/server/utils');
 const { logger } = require('~/config');
 
-const paths = require('~/config/paths');
 const routes = require('./routes');
 
 const { PORT, HOST, ALLOW_SOCIAL_LOGIN } = process.env ?? {};
 
 const port = Number(PORT) || 3080;
 const host = HOST || 'localhost';
-const projectPath = path.join(__dirname, '..', '..', 'client');
-const { jwtLogin, passportLogin } = require('~/strategies');
 
 const startServer = async () => {
+  if (typeof Bun !== 'undefined') {
+    axios.defaults.headers.common['Accept-Encoding'] = 'gzip';
+  }
   await connectDb();
   logger.info('Connected to MongoDB');
   await indexSync();
 
   const app = express();
-  app.locals.config = paths;
+  app.disable('x-powered-by');
+  await AppService(app);
+
+  app.get('/health', (_req, res) => res.status(200).send('OK'));
 
   // Middleware
+  app.use(noIndex);
   app.use(errorController);
   app.use(express.json({ limit: '3mb' }));
   app.use(mongoSanitize());
   app.use(express.urlencoded({ extended: true, limit: '3mb' }));
-  app.use(express.static(path.join(projectPath, 'dist')));
-  app.use(express.static(path.join(projectPath, 'public')));
+  app.use(express.static(app.locals.paths.dist));
+  app.use(express.static(app.locals.paths.publicPath));
   app.set('trust proxy', 1); // trust first proxy
   app.use(cors());
 
   if (!ALLOW_SOCIAL_LOGIN) {
     console.warn(
-      'Social logins are disabled. Set Envrionment Variable "ALLOW_SOCIAL_LOGIN" to true to enable them.',
+      'Social logins are disabled. Set Environment Variable "ALLOW_SOCIAL_LOGIN" to true to enable them.',
     );
   }
 
@@ -48,7 +58,7 @@ const startServer = async () => {
   passport.use(await jwtLogin());
   passport.use(passportLogin());
 
-  if (ALLOW_SOCIAL_LOGIN?.toLowerCase() === 'true') {
+  if (isEnabled(ALLOW_SOCIAL_LOGIN)) {
     configureSocialLogins(app);
   }
 
@@ -71,10 +81,10 @@ const startServer = async () => {
   app.use('/api/plugins', routes.plugins);
   app.use('/api/config', routes.config);
   app.use('/api/assistants', routes.assistants);
-  app.use('/api/files', routes.files);
+  app.use('/api/files', await routes.files.initialize());
 
   app.use((req, res) => {
-    res.status(404).sendFile(path.join(projectPath, 'dist', 'index.html'));
+    res.status(404).sendFile(path.join(app.locals.paths.dist, 'index.html'));
   });
 
   app.listen(port, host, () => {

api/server/middleware/abortMiddleware.js

@@ -1,23 +1,40 @@
+const { EModelEndpoint } = require('librechat-data-provider');
 const { sendMessage, sendError, countTokens, isEnabled } = require('~/server/utils');
 const { saveMessage, getConvo, getConvoTitle } = require('~/models');
 const clearPendingReq = require('~/cache/clearPendingReq');
 const abortControllers = require('./abortControllers');
 const { redactMessage } = require('~/config/parsers');
 const spendTokens = require('~/models/spendTokens');
+const { abortRun } = require('./abortRun');
 const { logger } = require('~/config');
 
 async function abortMessage(req, res) {
-  const { abortKey } = req.body;
+  let { abortKey, conversationId, endpoint } = req.body;
+
+  if (!abortKey && conversationId) {
+    abortKey = conversationId;
+  }
+
+  if (endpoint === EModelEndpoint.assistants) {
+    return await abortRun(req, res);
+  }
 
   if (!abortControllers.has(abortKey) && !res.headersSent) {
-    return res.status(404).send({ message: 'Request not found' });
+    return res.status(204).send({ message: 'Request not found' });
   }
 
   const { abortController } = abortControllers.get(abortKey);
-  const ret = await abortController.abortCompletion();
+  const finalEvent = await abortController.abortCompletion();
   logger.debug('[abortMessage] Aborted request', { abortKey });
   abortControllers.delete(abortKey);
-  res.send(JSON.stringify(ret));
+
+  if (res.headersSent && finalEvent) {
+    return sendMessage(res, finalEvent);
+  }
+
+  res.setHeader('Content-Type', 'application/json');
+
+  res.send(JSON.stringify(finalEvent));
 }
 
 const handleAbort = () => {
@@ -58,7 +75,6 @@ const createAbortController = (req, res, getAbortData) => {
       finish_reason: 'incomplete',
       model: endpointOption.modelOptions.model,
       unfinished: false,
-      cancelled: true,
       error: false,
       isCreatedByUser: false,
       tokenCount: completionTokens,
@@ -84,11 +100,17 @@ const createAbortController = (req, res, getAbortData) => {
 };
 
 const handleAbortError = async (res, req, error, data) => {
-  logger.error('[handleAbortError] response error and aborting request', error);
+  logger.error('[handleAbortError] AI response error; aborting request:', error);
   const { sender, conversationId, messageId, parentMessageId, partialText } = data;
 
-  const respondWithError = async () => {
-    const options = {
+  if (error.stack && error.stack.includes('google')) {
+    logger.warn(
+      `AI Response error for conversation ${conversationId} likely caused by Google censor/filter`,
+    );
+  }
+
+  const respondWithError = async (partialText) => {
+    let options = {
       sender,
       messageId,
       conversationId,
@@ -97,6 +119,16 @@ const handleAbortError = async (res, req, error, data) => {
       shouldSaveMessage: true,
       user: req.user.id,
     };
+
+    if (partialText) {
+      options = {
+        ...options,
+        error: false,
+        unfinished: true,
+        text: partialText,
+      };
+    }
+
     const callback = async () => {
       if (abortControllers.has(conversationId)) {
         const { abortController } = abortControllers.get(conversationId);
@@ -113,7 +145,7 @@ const handleAbortError = async (res, req, error, data) => {
       return await abortMessage(req, res);
     } catch (err) {
       logger.error('[handleAbortError] error while trying to abort message', err);
-      return respondWithError();
+      return respondWithError(partialText);
     }
   } else {
     return respondWithError();

api/server/middleware/abortRun.js

@@ -0,0 +1,93 @@
+const { CacheKeys, RunStatus, isUUID } = require('librechat-data-provider');
+const { initializeClient } = require('~/server/services/Endpoints/assistants');
+const { checkMessageGaps, recordUsage } = require('~/server/services/Threads');
+const { getConvo } = require('~/models/Conversation');
+const getLogStores = require('~/cache/getLogStores');
+const { sendMessage } = require('~/server/utils');
+const { logger } = require('~/config');
+
+const three_minutes = 1000 * 60 * 3;
+
+async function abortRun(req, res) {
+  res.setHeader('Content-Type', 'application/json');
+  const { abortKey } = req.body;
+  const [conversationId, latestMessageId] = abortKey.split(':');
+  const conversation = await getConvo(req.user.id, conversationId);
+
+  if (conversation?.model) {
+    req.body.model = conversation.model;
+  }
+
+  if (!isUUID.safeParse(conversationId).success) {
+    logger.error('[abortRun] Invalid conversationId', { conversationId });
+    return res.status(400).send({ message: 'Invalid conversationId' });
+  }
+
+  const cacheKey = `${req.user.id}:${conversationId}`;
+  const cache = getLogStores(CacheKeys.ABORT_KEYS);
+  const runValues = await cache.get(cacheKey);
+  const [thread_id, run_id] = runValues.split(':');
+
+  if (!run_id) {
+    logger.warn('[abortRun] Couldn\'t find run for cancel request', { thread_id });
+    return res.status(204).send({ message: 'Run not found' });
+  } else if (run_id === 'cancelled') {
+    logger.warn('[abortRun] Run already cancelled', { thread_id });
+    return res.status(204).send({ message: 'Run already cancelled' });
+  }
+
+  let runMessages = [];
+  /** @type {{ openai: OpenAI }} */
+  const { openai } = await initializeClient({ req, res });
+
+  try {
+    await cache.set(cacheKey, 'cancelled', three_minutes);
+    const cancelledRun = await openai.beta.threads.runs.cancel(thread_id, run_id);
+    logger.debug('[abortRun] Cancelled run:', cancelledRun);
+  } catch (error) {
+    logger.error('[abortRun] Error cancelling run', error);
+    if (
+      error?.message?.includes(RunStatus.CANCELLED) ||
+      error?.message?.includes(RunStatus.CANCELLING)
+    ) {
+      return res.end();
+    }
+  }
+
+  try {
+    const run = await openai.beta.threads.runs.retrieve(thread_id, run_id);
+    await recordUsage({
+      ...run.usage,
+      model: run.model,
+      user: req.user.id,
+      conversationId,
+    });
+  } catch (error) {
+    logger.error('[abortRun] Error fetching or processing run', error);
+  }
+
+  runMessages = await checkMessageGaps({
+    openai,
+    latestMessageId,
+    thread_id,
+    run_id,
+    conversationId,
+  });
+
+  const finalEvent = {
+    title: 'New Chat',
+    final: true,
+    conversation,
+    runMessages,
+  };
+
+  if (res.headersSent && finalEvent) {
+    return sendMessage(res, finalEvent);
+  }
+
+  res.json(finalEvent);
+}
+
+module.exports = {
+  abortRun,
+};

api/server/middleware/buildEndpointOption.js

@@ -1,22 +1,35 @@
-const { processFiles } = require('~/server/services/Files');
-const openAI = require('~/server/services/Endpoints/openAI');
-const google = require('~/server/services/Endpoints/google');
-const anthropic = require('~/server/services/Endpoints/anthropic');
-const gptPlugins = require('~/server/services/Endpoints/gptPlugins');
 const { parseConvo, EModelEndpoint } = require('librechat-data-provider');
+const { getModelsConfig } = require('~/server/controllers/ModelController');
+const assistants = require('~/server/services/Endpoints/assistants');
+const gptPlugins = require('~/server/services/Endpoints/gptPlugins');
+const { processFiles } = require('~/server/services/Files/process');
+const anthropic = require('~/server/services/Endpoints/anthropic');
+const openAI = require('~/server/services/Endpoints/openAI');
+const custom = require('~/server/services/Endpoints/custom');
+const google = require('~/server/services/Endpoints/google');
 
 const buildFunction = {
   [EModelEndpoint.openAI]: openAI.buildOptions,
   [EModelEndpoint.google]: google.buildOptions,
+  [EModelEndpoint.custom]: custom.buildOptions,
   [EModelEndpoint.azureOpenAI]: openAI.buildOptions,
   [EModelEndpoint.anthropic]: anthropic.buildOptions,
   [EModelEndpoint.gptPlugins]: gptPlugins.buildOptions,
+  [EModelEndpoint.assistants]: assistants.buildOptions,
 };
 
-function buildEndpointOption(req, res, next) {
-  const { endpoint } = req.body;
-  const parsedBody = parseConvo(endpoint, req.body);
-  req.body.endpointOption = buildFunction[endpoint](endpoint, parsedBody);
+async function buildEndpointOption(req, res, next) {
+  const { endpoint, endpointType } = req.body;
+  const parsedBody = parseConvo({ endpoint, endpointType, conversation: req.body });
+  req.body.endpointOption = buildFunction[endpointType ?? endpoint](
+    endpoint,
+    parsedBody,
+    endpointType,
+  );
+
+  const modelsConfig = await getModelsConfig(req);
+  req.body.endpointOption.modelsConfig = modelsConfig;
+
   if (req.body.files) {
     // hold the promise
     req.body.endpointOption.attachments = processFiles(req.body.files);

api/server/middleware/denyRequest.js

@@ -1,7 +1,7 @@
 const crypto = require('crypto');
-const { saveMessage } = require('~/models');
+const { getResponseSender, Constants } = require('librechat-data-provider');
 const { sendMessage, sendError } = require('~/server/utils');
-const { getResponseSender } = require('librechat-data-provider');
+const { saveMessage } = require('~/models');
 
 /**
  * Denies a request by sending an error message and optionally saves the user's message.
@@ -38,8 +38,7 @@ const denyRequest = async (req, res, errorMessage) => {
   };
   sendMessage(res, { message: userMessage, created: true });
 
-  const shouldSaveMessage =
-    _convoId && parentMessageId && parentMessageId !== '00000000-0000-0000-0000-000000000000';
+  const shouldSaveMessage = _convoId && parentMessageId && parentMessageId !== Constants.NO_PARENT;
 
   if (shouldSaveMessage) {
     await saveMessage({ ...userMessage, user: req.user.id });