feat: add bottleneck for tts routes

refactor: slightly increase request threshold

.env.example

@@ -2,11 +2,9 @@
 #                       LibreChat Configuration                       #
 #=====================================================================#
 # Please refer to the reference documentation for assistance          #
-# with configuring your LibreChat environment. The guide is           #
-# available both online and within your local LibreChat               #
-# directory:                                                          #
-# Online: https://docs.librechat.ai/install/configuration/dotenv.html #
-# Locally: ./docs/install/configuration/dotenv.md                     #
+# with configuring your LibreChat environment.                        #
+#                                                                     #
+# https://www.librechat.ai/docs/configuration/dotenv                  #
 #=====================================================================#
 
 #==================================================#
@@ -23,6 +21,13 @@ DOMAIN_SERVER=http://localhost:3080
 
 NO_INDEX=true
 
+#===============#
+# JSON Logging  #
+#===============#
+
+# Use when process console logs in cloud deployment like GCP/AWS
+CONSOLE_JSON=false
+
 #===============#
 # Debug Logging #
 #===============#
@@ -55,15 +60,17 @@ PROXY=
 #===================================#
 # Known Endpoints - librechat.yaml  #
 #===================================#
-# https://docs.librechat.ai/install/configuration/ai_endpoints.html
+# https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints
 
-# GROQ_API_KEY=
-# SHUTTLEAI_KEY=
-# OPENROUTER_KEY=
-# MISTRAL_API_KEY=
 # ANYSCALE_API_KEY=
+# APIPIE_API_KEY=
 # FIREWORKS_API_KEY=
+# GROQ_API_KEY=
+# HUGGINGFACE_TOKEN=
+# MISTRAL_API_KEY=
+# OPENROUTER_KEY=
 # PERPLEXITY_API_KEY=
+# SHUTTLEAI_API_KEY=
 # TOGETHERAI_API_KEY=
 
 #============#
@@ -71,7 +78,7 @@ PROXY=
 #============#
 
 ANTHROPIC_API_KEY=user_provided
-# ANTHROPIC_MODELS=claude-3-opus-20240229,claude-3-sonnet-20240229,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
+# ANTHROPIC_MODELS=claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k
 # ANTHROPIC_REVERSE_PROXY=
 
 #============#
@@ -106,15 +113,34 @@ BINGAI_TOKEN=user_provided
 #============#
 
 GOOGLE_KEY=user_provided
-# GOOGLE_MODELS=gemini-pro,gemini-pro-vision,chat-bison,chat-bison-32k,codechat-bison,codechat-bison-32k,text-bison,text-bison-32k,text-unicorn,code-gecko,code-bison,code-bison-32k
 # GOOGLE_REVERSE_PROXY=
 
+# Gemini API
+# GOOGLE_MODELS=gemini-1.5-flash-latest,gemini-1.0-pro,gemini-1.0-pro-001,gemini-1.0-pro-latest,gemini-1.0-pro-vision-latest,gemini-1.5-pro-latest,gemini-pro,gemini-pro-vision
+
+# Vertex AI
+# GOOGLE_MODELS=gemini-1.5-flash-preview-0514,gemini-1.5-pro-preview-0514,gemini-1.0-pro-vision-001,gemini-1.0-pro-002,gemini-1.0-pro-001,gemini-pro-vision,gemini-1.0-pro
+
+# Google Gemini Safety Settings
+# NOTE (Vertex AI): You do not have access to the BLOCK_NONE setting by default.
+# To use this restricted HarmBlockThreshold setting, you will need to either:
+#
+# (a) Get access through an allowlist via your Google account team
+# (b) Switch your account type to monthly invoiced billing following this instruction:
+#     https://cloud.google.com/billing/docs/how-to/invoiced-billing
+#
+# GOOGLE_SAFETY_SEXUALLY_EXPLICIT=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_HATE_SPEECH=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_HARASSMENT=BLOCK_ONLY_HIGH
+# GOOGLE_SAFETY_DANGEROUS_CONTENT=BLOCK_ONLY_HIGH
+
+
 #============#
 # OpenAI     #
 #============#
 
 OPENAI_API_KEY=user_provided
-# OPENAI_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k
+# OPENAI_MODELS=gpt-4o,gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k
 
 DEBUG_OPENAI=false
 
@@ -128,7 +154,7 @@ DEBUG_OPENAI=false
 
 # OPENAI_REVERSE_PROXY=
 
-# OPENAI_ORGANIZATION= 
+# OPENAI_ORGANIZATION=
 
 #====================#
 #   Assistants API   #
@@ -136,19 +162,29 @@ DEBUG_OPENAI=false
 
 ASSISTANTS_API_KEY=user_provided
 # ASSISTANTS_BASE_URL=
-# ASSISTANTS_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-16k,gpt-3.5-turbo,gpt-4,gpt-4-0314,gpt-4-32k-0314,gpt-4-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-1106,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview
+# ASSISTANTS_MODELS=gpt-4o,gpt-3.5-turbo-0125,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-16k,gpt-3.5-turbo,gpt-4,gpt-4-0314,gpt-4-32k-0314,gpt-4-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-1106,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview
+
+#==========================#
+#   Azure Assistants API   #
+#==========================#
+
+# Note: You should map your credentials with custom variables according to your Azure OpenAI Configuration
+# The models for Azure Assistants are also determined by your Azure OpenAI configuration.
+
+# More info, including how to enable use of Assistants with Azure here:
+# https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints/azure#using-assistants-with-azure
 
 #============#
 # OpenRouter #
 #============#
-
+# !!!Warning: Use the variable above instead of this one. Using this one will override the OpenAI endpoint
 # OPENROUTER_API_KEY=
 
 #============#
 # Plugins    #
 #============#
 
-# PLUGIN_MODELS=gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613
+# PLUGIN_MODELS=gpt-4o,gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613
 
 DEBUG_PLUGINS=true
 
@@ -185,7 +221,7 @@ AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
 
 # Google
 #-----------------
-GOOGLE_API_KEY=
+GOOGLE_SEARCH_API_KEY=
 GOOGLE_CSE_ID=
 
 # SerpAPI
@@ -221,6 +257,14 @@ MEILI_NO_ANALYTICS=true
 MEILI_HOST=http://0.0.0.0:7700
 MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
 
+
+#==================================================#
+#          Speech to Text & Text to Speech         #
+#==================================================#
+
+STT_API_KEY=
+TTS_API_KEY=
+
 #===================================================#
 #                    User System                    #
 #===================================================#
@@ -309,23 +353,34 @@ OPENID_ISSUER=
 OPENID_SESSION_SECRET=
 OPENID_SCOPE="openid profile email"
 OPENID_CALLBACK_URL=/oauth/openid/callback
+OPENID_REQUIRED_ROLE=
+OPENID_REQUIRED_ROLE_TOKEN_KIND=
+OPENID_REQUIRED_ROLE_PARAMETER_PATH=
 
 OPENID_BUTTON_LABEL=
 OPENID_IMAGE_URL=
 
+# LDAP
+LDAP_URL=
+LDAP_BIND_DN=
+LDAP_BIND_CREDENTIALS=
+LDAP_USER_SEARCH_BASE=
+LDAP_SEARCH_FILTER=mail={{username}}
+LDAP_CA_CERT_PATH=
+
 #========================#
 # Email Password Reset   #
 #========================#
 
-EMAIL_SERVICE=                  
-EMAIL_HOST=                     
-EMAIL_PORT=25                   
-EMAIL_ENCRYPTION=               
-EMAIL_ENCRYPTION_HOSTNAME=      
-EMAIL_ALLOW_SELFSIGNED=         
-EMAIL_USERNAME=                 
-EMAIL_PASSWORD=                 
-EMAIL_FROM_NAME=                
+EMAIL_SERVICE=
+EMAIL_HOST=
+EMAIL_PORT=25
+EMAIL_ENCRYPTION=
+EMAIL_ENCRYPTION_HOSTNAME=
+EMAIL_ALLOW_SELFSIGNED=
+EMAIL_USERNAME=
+EMAIL_PASSWORD=
+EMAIL_FROM_NAME=
 EMAIL_FROM=noreply@librechat.ai
 
 #========================#

.eslintrc.js

@@ -132,6 +132,13 @@ module.exports = {
         },
       ],
     },
+    {
+      files: './config/translations/**/*.ts',
+      parser: '@typescript-eslint/parser',
+      parserOptions: {
+        project: './config/translations/tsconfig.json',
+      },
+    },
     {
       files: ['./packages/data-provider/specs/**/*.ts'],
       parserOptions: {

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -50,7 +50,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml

@@ -43,7 +43,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/ISSUE_TEMPLATE/QUESTION.yml

@@ -44,7 +44,7 @@ body:
     id: terms
     attributes:
       label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/CODE_OF_CONDUCT.md)
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/danny-avila/LibreChat/blob/main/.github/CODE_OF_CONDUCT.md)
       options:
         - label: I agree to follow this project's Code of Conduct
           required: true

.github/pull_request_template.md

@@ -1,7 +1,10 @@
 # Pull Request Template
 
+⚠️ Before Submitting a PR, Please Review:
+- Please ensure that you have thoroughly read and understood the [Contributing Docs](https://github.com/danny-avila/LibreChat/blob/main/.github/CONTRIBUTING.md) before submitting your Pull Request.
 
-### ⚠️ Before Submitting a PR, read the [Contributing Docs](https://github.com/danny-avila/LibreChat/blob/main/.github/CONTRIBUTING.md) in full!
+⚠️ Documentation Updates Notice:
+- Kindly note that documentation updates are managed in this repository: [librechat.ai](https://github.com/LibreChat-AI/librechat.ai)
 
 ## Summary
 
@@ -16,8 +19,6 @@ Please delete any irrelevant options.
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] This change requires a documentation update
 - [ ] Translation update
-- [ ] Documentation update
-
 
 ## Testing
 
@@ -37,4 +38,4 @@ Please delete any irrelevant options.
 - [ ] I have written tests demonstrating that my changes are effective or that my feature works
 - [ ] Local unit tests pass with my changes
 - [ ] Any changes dependent on mine have been merged and published in downstream modules.
-- [ ] New documents have been locally validated with mkdocs
+- [ ] A pull request for updating the documentation has been submitted.

.github/workflows/backend-review.yml

@@ -51,6 +51,9 @@ jobs:
             exit 1
           fi
 
+      - name: Prepare .env.test file
+        run: cp api/test/.env.test.example api/test/.env.test
+
       - name: Run unit tests
         run: cd api && npm run test:ci
 
@@ -60,4 +63,4 @@ jobs:
       - name: Run linters
         uses: wearerequired/lint-action@v2
         with:
-          eslint: true
+          eslint: true

.github/workflows/container.yml

@@ -1,83 +0,0 @@
-name: Docker Compose Build on Tag
-
-# The workflow is triggered when a tag is pushed
-on:
-  push:
-    tags:
-      - "*"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      # Set up QEMU for cross-platform builds
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Prepare Docker Build
-      - name: Build Docker images
-        run: |
-          cp .env.example .env
-
-      # Tag and push librechat-api
-      - name: Docker metadata for librechat-api
-        id: meta-librechat-api
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ghcr.io/${{ github.repository_owner }}/librechat-api
-          tags: |
-            type=raw,value=latest
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Build and librechat-api
-        uses: docker/build-push-action@v5
-        with:
-          file: Dockerfile.multi
-          context: .
-          push: true
-          tags: ${{ steps.meta-librechat-api.outputs.tags }}
-          platforms: linux/amd64,linux/arm64
-          target: api-build
-
-      # Tag and push librechat
-      - name: Docker metadata for librechat
-        id: meta-librechat
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ghcr.io/${{ github.repository_owner }}/librechat
-          tags: |
-            type=raw,value=latest
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Build and librechat
-        uses: docker/build-push-action@v5
-        with:
-          file: Dockerfile
-          context: .
-          push: true
-          tags: ${{ steps.meta-librechat.outputs.tags }}
-          platforms: linux/amd64,linux/arm64
-          target: node

.github/workflows/frontend-review.yml

@@ -1,11 +1,6 @@
-#github action to run unit tests for frontend with jest
 name: Frontend Unit Tests
+
 on:
-  # push:
-  #   branches: 
-  #     - main
-  #     - dev
-  #     - release/*
   pull_request:
     branches: 
       - main
@@ -14,9 +9,10 @@ on:
     paths:
       - 'client/**'
       - 'packages/**'
+
 jobs:
-  tests_frontend:
-    name: Run frontend unit tests
+  tests_frontend_ubuntu:
+    name: Run frontend unit tests on Ubuntu
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
@@ -35,4 +31,26 @@ jobs:
 
       - name: Run unit tests
         run: npm run test:ci --verbose
-        working-directory: client
+        working-directory: client
+
+  tests_frontend_windows:
+    name: Run frontend unit tests on Windows
+    timeout-minutes: 60
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build Client
+        run: npm run frontend:ci
+
+      - name: Run unit tests
+        run: npm run test:ci --verbose
+        working-directory: client

.github/workflows/latest-images-main.yml

@@ -1,88 +0,0 @@
-name: Docker Compose Build Latest Tag (Manual Dispatch)
-
-# The workflow is manually triggered
-on:
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Check out the repository
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # Fetch all tags and set the latest tag
-      - name: Fetch tags and set the latest tag
-        run: |
-          git fetch --tags
-          echo "LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)" >> $GITHUB_ENV
-
-      # Set up Docker
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      # Set up QEMU
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      # Log in to GitHub Container Registry
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Prepare Docker Build
-      - name: Build Docker images
-        run: cp .env.example .env
-
-      # Docker metadata for librechat-api
-      - name: Docker metadata for librechat-api
-        id: meta-librechat-api
-        uses: docker/metadata-action@v5
-        with:
-          images: ghcr.io/${{ github.repository_owner }}/librechat-api
-          tags: |
-            type=raw,value=${{ env.LATEST_TAG }},enable=true
-            type=raw,value=latest,enable=true
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      # Build and push librechat-api
-      - name: Build and push librechat-api
-        uses: docker/build-push-action@v5
-        with:
-          file: Dockerfile.multi
-          context: .
-          push: true
-          tags: ${{ steps.meta-librechat-api.outputs.tags }}
-          platforms: linux/amd64,linux/arm64
-          target: api-build
-
-      # Docker metadata for librechat
-      - name: Docker metadata for librechat
-        id: meta-librechat
-        uses: docker/metadata-action@v5
-        with:
-          images: ghcr.io/${{ github.repository_owner }}/librechat
-          tags: |
-            type=raw,value=${{ env.LATEST_TAG }},enable=true
-            type=raw,value=latest,enable=true
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      # Build and push librechat
-      - name: Build and push librechat
-        uses: docker/build-push-action@v5
-        with:
-          file: Dockerfile
-          context: .
-          push: true
-          tags: ${{ steps.meta-librechat.outputs.tags }}
-          platforms: linux/amd64,linux/arm64
-          target: node

.github/workflows/main-image-workflow.yml

@@ -1,12 +1,20 @@
 name: Docker Compose Build Latest Main Image Tag (Manual Dispatch)
 
-# The workflow is manually triggered
 on:
   workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
 
     steps:
       - name: Checkout
@@ -17,12 +25,15 @@ jobs:
           git fetch --tags
           echo "LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)" >> $GITHUB_ENV
 
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
+      # Set up QEMU
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -30,26 +41,29 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Docker metadata for librechat
-      - name: Docker metadata for librechat
-        id: meta-librechat
-        uses: docker/metadata-action@v5
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
         with:
-          images: ghcr.io/${{ github.repository_owner }}/librechat
-          tags: |
-            type=raw,value=${{ env.LATEST_TAG }},enable=true
-            type=raw,value=latest,enable=true
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{major}}.{{minor}}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      # Build and push librechat with only linux/amd64 platform
-      - name: Build and push librechat
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
         uses: docker/build-push-action@v5
         with:
-          file: Dockerfile
           context: .
+          file: ${{ matrix.file }}
           push: true
-          tags: ${{ steps.meta-librechat.outputs.tags }}
-          platforms: linux/amd64
-          target: node
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ env.LATEST_TAG }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.github/workflows/mkdocs.yaml

@@ -1,27 +0,0 @@
-name: mkdocs
-on:
-  push:
-    branches:
-      - main
-permissions:
-  contents: write
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.x
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
-      - uses: actions/cache@v3
-        with:
-          key: mkdocs-material-${{ env.cache_id }}
-          path: .cache
-          restore-keys: |
-            mkdocs-material-
-      - run: pip install mkdocs-material 
-      - run: pip install mkdocs-nav-weight
-      - run: pip install mkdocs-publisher
-      - run: pip install mkdocs-exclude
-      - run: mkdocs gh-deploy --force

.github/workflows/tag-images.yml

@@ -0,0 +1,67 @@
+name: Docker Images Build on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - target: api-build
+            file: Dockerfile.multi
+            image_name: librechat-api
+          - target: node
+            file: Dockerfile
+            image_name: librechat
+
+    steps:
+      # Check out the repository
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Set up QEMU
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Login to Docker Hub
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Prepare the environment
+      - name: Prepare environment
+        run: |
+          cp .env.example .env
+
+      # Build and push Docker images for each target
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:${{ github.ref_name }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image_name }}:latest
+          platforms: linux/amd64,linux/arm64
+          target: ${{ matrix.target }}

.gitignore

@@ -21,6 +21,10 @@ coverage
 # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
 .grunt
 
+# translation services
+config/translations/stores/*
+client/src/localization/languages/*_missing_keys.json
+
 # Compiled Dirs (http://nodejs.org/api/addons.html)
 build/
 dist/
@@ -50,6 +54,7 @@ bower_components/
 
 #config file
 librechat.yaml
+librechat.yml
 
 # Environment
 .npmrc
@@ -68,6 +73,8 @@ src/style - official.css
 /playwright/.cache/
 .DS_Store
 *.code-workspace
+.idx
+monospace.json
 .idea
 *.iml
 *.pem
@@ -75,6 +82,7 @@ config.local.ts
 **/storageState.json
 junit.xml
 **/.venv/
+**/venv/
 
 # docker override file
 docker-compose.override.yaml
@@ -92,4 +100,7 @@ auth.json
 !client/src/components/Nav/SettingsTabs/Data/
 
 # User uploads
-uploads/
+uploads/
+
+# owner
+release/

.husky/pre-commit

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env sh 
 set -e
 . "$(dirname -- "$0")/_/husky.sh"
 [ -n "$CI" ] && exit 0

Dockerfile

@@ -1,8 +1,8 @@
-# Base node image
-FROM node:18-alpine AS node
+# v0.7.2
+
+# Base node image
+FROM node:20-alpine AS node
 
-RUN apk add g++ make py3-pip
-RUN npm install -g node-gyp
 RUN apk --no-cache add curl
 
 RUN mkdir -p /app && chown node:node /app
@@ -12,15 +12,21 @@ USER node
 
 COPY --chown=node:node . .
 
-# Allow mounting of these files, which have no default
-# values.
-RUN touch .env
-RUN npm config set fetch-retry-maxtimeout 300000
-RUN npm install --no-audit
+RUN \
+    # Allow mounting of these files, which have no default
+    touch .env ; \
+    # Create directories for the volumes to inherit the correct permissions
+    mkdir -p /app/client/public/images /app/api/logs ; \
+    npm config set fetch-retry-maxtimeout 600000 ; \
+    npm config set fetch-retries 5 ; \
+    npm config set fetch-retry-mintimeout 15000 ; \
+    npm install --no-audit; \
+    # React client build
+    NODE_OPTIONS="--max-old-space-size=2048" npm run frontend; \
+    npm prune --production; \
+    npm cache clean --force
 
-# React client build
-ENV NODE_OPTIONS="--max-old-space-size=2048"
-RUN npm run frontend
+RUN mkdir -p /app/client/public/images /app/api/logs
 
 # Node API setup
 EXPOSE 3080

Dockerfile.multi

@@ -1,3 +1,5 @@
+# v0.7.2
+
 # Build API, Client and Data Provider
 FROM node:20-alpine AS base
 
@@ -5,29 +7,31 @@ FROM node:20-alpine AS base
 FROM base AS data-provider-build
 WORKDIR /app/packages/data-provider
 COPY ./packages/data-provider ./
-RUN npm install
+RUN npm install; npm cache clean --force
 RUN npm run build
+RUN npm prune --production
 
 # React client build
-FROM data-provider-build AS client-build
+FROM base AS client-build
 WORKDIR /app/client
-COPY ./client/ ./
+COPY ./client/package*.json ./
 # Copy data-provider to client's node_modules
-RUN mkdir -p /app/client/node_modules/librechat-data-provider/
-RUN cp -R /app/packages/data-provider/* /app/client/node_modules/librechat-data-provider/
-RUN npm install
+COPY --from=data-provider-build /app/packages/data-provider/ /app/client/node_modules/librechat-data-provider/
+RUN npm install; npm cache clean --force
+COPY ./client/ ./
 ENV NODE_OPTIONS="--max-old-space-size=2048"
 RUN npm run build
 
 # Node API setup
-FROM data-provider-build AS api-build
+FROM base AS api-build
 WORKDIR /app/api
 COPY api/package*.json ./
 COPY api/ ./
+# Copy helper scripts
+COPY config/ ./
 # Copy data-provider to API's node_modules
-RUN mkdir -p /app/api/node_modules/librechat-data-provider/
-RUN cp -R /app/packages/data-provider/* /app/api/node_modules/librechat-data-provider/
-RUN npm install
+COPY --from=data-provider-build /app/packages/data-provider/ /app/api/node_modules/librechat-data-provider/
+RUN npm install --include prod; npm cache clean --force
 COPY --from=client-build /app/client/dist /app/client/dist
 EXPOSE 3080
 ENV HOST=0.0.0.0

README.md

@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://librechat.ai">
-    <img src="docs/assets/LibreChat.svg" height="256">
+    <img src="client/public/assets/logo.svg" height="256">
   </a>
   <h1 align="center">
     <a href="https://librechat.ai">LibreChat</a>
@@ -27,7 +27,7 @@
 </p>
 
 <p align="center">
-<a href="https://railway.app/template/b5k2mn?referralCode=HI9hWz">
+<a href="https://railway.app/template/b5k2mn?referralCode=myKrVZ">
   <img src="https://railway.app/button.svg" alt="Deploy on Railway" height="30">
 </a>
 <a href="https://zeabur.com/templates/0X2ZY8">
@@ -41,23 +41,35 @@
 # 📃 Features
 
 - 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and latest updates
+- 🤖 AI model selection:
+  - OpenAI, Azure OpenAI, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins, Assistants API (including Azure Assistants)
+- ✅ Compatible across both **[Remote & Local AI services](https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints):**
+  - groq, Ollama, Cohere, Mistral AI, Apple MLX, koboldcpp, OpenRouter, together.ai, Perplexity, ShuttleAI, and more
+- 💾 Create, Save, & Share Custom Presets
+- 🔀 Switch between AI Endpoints and Presets, mid-chat
+- 🔄 Edit, Resubmit, and Continue Messages with Conversation branching
+- 🌿 Fork Messages & Conversations for Advanced Context control
 - 💬 Multimodal Chat:
-  - Upload and analyze images with GPT-4 and Gemini Vision 📸
-  - General file support now available through the Assistants API integration. 🗃️
-  - Local RAG in Active Development 🚧
+    - Upload and analyze images with Claude 3, GPT-4 (including `gpt-4o`), and Gemini Vision 📸
+    - Chat with Files using Custom Endpoints, OpenAI, Azure, Anthropic, & Google. 🗃️
+    - Advanced Agents with Files, Code Interpreter, Tools, and API Actions 🔦
+      - Available through the [OpenAI Assistants API](https://platform.openai.com/docs/assistants/overview) 🌤️
+      - Non-OpenAI Agents in Active Development 🚧
 - 🌎 Multilingual UI:
   - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
   - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands, עברית
-- 🤖 AI model selection: OpenAI, Azure OpenAI, BingAI, ChatGPT, Google Vertex AI, Anthropic (Claude), Plugins, Assistants API (including Azure Assistants)
-- 💾 Create, Save, & Share Custom Presets
-- 🔄 Edit, Resubmit, and Continue messages with conversation branching
+- 🎨 Customizable Dropdown & Interface: Adapts to both power users and newcomers.
+- 📥 Import Conversations from LibreChat, ChatGPT, Chatbot UI
 - 📤 Export conversations as screenshots, markdown, text, json.
 - 🔍 Search all messages/conversations
 - 🔌 Plugins, including web access, image generation with DALL-E-3 and more
 - 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
-- ⚙️ Configure Proxy, Reverse Proxy, Docker, many Deployment options, and completely Open-Source
+- ⚙️ Configure Proxy, Reverse Proxy, Docker, & many Deployment options:
+  - Use completely local or deploy on the cloud
+- 📖 Completely Open-Source & Built in Public
+- 🧑‍🤝‍🧑 Community-driven development, support, and feedback
 
-[For a thorough review of our features, see our docs here](https://docs.librechat.ai/features/plugins/introduction.html) 📚
+[For a thorough review of our features, see our docs here](https://docs.librechat.ai/) 📚
 
 ## 🪶 All-In-One AI Conversations with LibreChat
 
@@ -65,37 +77,49 @@ LibreChat brings together the future of assistant AIs with the revolutionary tec
 
 With LibreChat, you no longer need to opt for ChatGPT Plus and can instead use free or pay-per-call APIs. We welcome contributions, cloning, and forking to enhance the capabilities of this advanced chatbot platform.
 
-<!-- https://github.com/danny-avila/LibreChat/assets/110412045/c1eb0c0f-41f6-4335-b982-84b278b53d59 -->
-
-[![Watch the video](https://img.youtube.com/vi/pNIOs1ovsXw/maxresdefault.jpg)](https://youtu.be/pNIOs1ovsXw)
+[![Watch the video](https://img.youtube.com/vi/YLVUW5UP9N0/maxresdefault.jpg)](https://www.youtube.com/watch?v=YLVUW5UP9N0)
 Click on the thumbnail to open the video☝️
 
 ---
 
-## 📚 Documentation
+## 🌐 Resources
 
-For more information on how to use our advanced features, install and configure our software, and access our guidelines and tutorials, please check out our documentation at [docs.librechat.ai](https://docs.librechat.ai)
+**GitHub Repo:**
+  - **RAG API:** [github.com/danny-avila/rag_api](https://github.com/danny-avila/rag_api)
+  - **Website:** [github.com/LibreChat-AI/librechat.ai](https://github.com/LibreChat-AI/librechat.ai)
+
+**Other:**
+  - **Website:** [librechat.ai](https://librechat.ai)
+  - **Documentation:** [docs.librechat.ai](https://docs.librechat.ai)
+  - **Blog:** [blog.librechat.ai](https://docs.librechat.ai)
 
 ---
 
 ## 📝 Changelog
 
-Keep up with the latest updates by visiting the releases page - [Releases](https://github.com/danny-avila/LibreChat/releases)
+Keep up with the latest updates by visiting the releases page and notes:
+- [Releases](https://github.com/danny-avila/LibreChat/releases)
+- [Changelog](https://www.librechat.ai/changelog) 
 
-**⚠️ [Breaking Changes](docs/general_info/breaking_changes.md)**
-Please consult the breaking changes before updating.
+**⚠️ Please consult the [changelog](https://www.librechat.ai/changelog) for breaking changes before updating.**
 
 ---
 
 ## ⭐ Star History
 
 <p align="center">
-<a href="https://trendshift.io/repositories/4685" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4685" alt="danny-avila%2FLibreChat | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+  <a href="https://star-history.com/#danny-avila/LibreChat&Date">
+    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
+  </a>
+</p>
+<p align="center">
+  <a href="https://trendshift.io/repositories/4685" target="_blank" style="padding: 10px;">
+    <img src="https://trendshift.io/api/badge/repositories/4685" alt="danny-avila%2FLibreChat | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
+  </a>
+  <a href="https://runacap.com/ross-index/q1-24/" target="_blank" rel="noopener" style="margin-left: 20px;">
+    <img style="width: 260px; height: 56px" src="https://runacap.com/wp-content/uploads/2024/04/ROSS_badge_white_Q1_2024.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q1 2024 | Runa Capital" width="260" height="56"/>
+  </a>
 </p>
-
-<a href="https://star-history.com/#danny-avila/LibreChat&Date">
-  <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date&theme=dark" onerror="this.src='https://api.star-history.com/svg?repos=danny-avila/LibreChat&type=Date'" />
-</a>
 
 ---
 

api/app/bingai.js

@@ -1,5 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
+const { EModelEndpoint } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
 const { logger } = require('~/config');
 
@@ -23,10 +24,7 @@ const askBing = async ({
 
   let key = null;
   if (expiresAt && isUserProvided) {
-    checkUserKeyExpiry(
-      expiresAt,
-      'Your BingAI Cookies have expired. Please provide your cookies again.',
-    );
+    checkUserKeyExpiry(expiresAt, EModelEndpoint.bingAI);
     key = await getUserKey({ userId, name: 'bingAI' });
   }
 

api/app/chatgpt-browser.js

@@ -1,6 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
-const { Constants } = require('librechat-data-provider');
+const { Constants, EModelEndpoint } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('../server/services/UserService');
 
 const browserClient = async ({
@@ -18,10 +18,7 @@ const browserClient = async ({
 
   let key = null;
   if (expiresAt && isUserProvided) {
-    checkUserKeyExpiry(
-      expiresAt,
-      'Your ChatGPT Access Token has expired. Please provide your token again.',
-    );
+    checkUserKeyExpiry(expiresAt, EModelEndpoint.chatGPTBrowser);
     key = await getUserKey({ userId, name: 'chatGPTBrowser' });
   }
 

api/app/clients/AnthropicClient.js

@@ -1,4 +1,5 @@
 const Anthropic = require('@anthropic-ai/sdk');
+const { HttpsProxyAgent } = require('https-proxy-agent');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const {
   getResponseSender,
@@ -7,10 +8,10 @@ const {
 } = require('librechat-data-provider');
 const { encodeAndFormat } = require('~/server/services/Files/images/encode');
 const {
-  titleFunctionPrompt,
-  parseTitleFromPrompt,
   truncateText,
   formatMessage,
+  titleFunctionPrompt,
+  parseParamFromPrompt,
   createContextHandlers,
 } = require('./prompts');
 const spendTokens = require('~/models/spendTokens');
@@ -75,7 +76,9 @@ class AnthropicClient extends BaseClient {
     this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
 
     this.maxContextTokens =
-      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ?? 100000;
+      this.options.maxContextTokens ??
+      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ??
+      100000;
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500;
     this.maxPromptTokens =
       this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens;
@@ -121,9 +124,14 @@ class AnthropicClient extends BaseClient {
   getClient() {
     /** @type {Anthropic.default.RequestOptions} */
     const options = {
+      fetch: this.fetch,
       apiKey: this.apiKey,
     };
 
+    if (this.options.proxy) {
+      options.httpAgent = new HttpsProxyAgent(this.options.proxy);
+    }
+
     if (this.options.reverseProxyUrl) {
       options.baseURL = this.options.reverseProxyUrl;
     }
@@ -652,9 +660,13 @@ class AnthropicClient extends BaseClient {
 
   getSaveOptions() {
     return {
+      maxContextTokens: this.options.maxContextTokens,
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
       resendFiles: this.options.resendFiles,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
@@ -742,7 +754,7 @@ class AnthropicClient extends BaseClient {
           context: 'title',
         });
         const text = response.content[0].text;
-        title = parseTitleFromPrompt(text);
+        title = parseParamFromPrompt(text, 'title');
       } catch (e) {
         logger.error('[AnthropicClient] There was an issue generating the title', e);
       }

api/app/clients/BaseClient.js

@@ -1,4 +1,5 @@
 const crypto = require('crypto');
+const fetch = require('node-fetch');
 const { supportsBalanceCheck, Constants } = require('librechat-data-provider');
 const { getConvo, getMessages, saveMessage, updateMessage, saveConvo } = require('~/models');
 const { addSpaceIfNeeded, isEnabled } = require('~/server/utils');
@@ -17,13 +18,14 @@ class BaseClient {
       month: 'long',
       day: 'numeric',
     });
+    this.fetch = this.fetch.bind(this);
   }
 
   setOptions() {
     throw new Error('Method \'setOptions\' must be implemented.');
   }
 
-  getCompletion() {
+  async getCompletion() {
     throw new Error('Method \'getCompletion\' must be implemented.');
   }
 
@@ -54,6 +56,22 @@ class BaseClient {
     });
   }
 
+  /**
+   * Makes an HTTP request and logs the process.
+   *
+   * @param {RequestInfo} url - The URL to make the request to. Can be a string or a Request object.
+   * @param {RequestInit} [init] - Optional init options for the request.
+   * @returns {Promise<Response>} - A promise that resolves to the response of the fetch request.
+   */
+  async fetch(_url, init) {
+    let url = _url;
+    if (this.options.directEndpoint) {
+      url = this.options.reverseProxyUrl;
+    }
+    logger.debug(`Making request to ${url}`);
+    return await fetch(url, init);
+  }
+
   getBuildMessagesOptions() {
     throw new Error('Subclasses must implement getBuildMessagesOptions');
   }
@@ -373,6 +391,14 @@ class BaseClient {
     const { user, head, isEdited, conversationId, responseMessageId, saveOptions, userMessage } =
       await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     const { generation = '' } = opts;
 
     // It's not necessary to push to currentMessages
@@ -456,6 +482,8 @@ class BaseClient {
       sender: this.sender,
       text: addSpaceIfNeeded(generation) + completion,
       promptTokens,
+      iconURL: this.options.iconURL,
+      endpoint: this.options.endpoint,
       ...(this.metadata ?? {}),
     };
 
@@ -525,8 +553,19 @@ class BaseClient {
     return _messages;
   }
 
+  /**
+   * Save a message to the database.
+   * @param {TMessage} message
+   * @param {Partial<TConversation>} endpointOptions
+   * @param {string | null} user
+   */
   async saveMessageToDatabase(message, endpointOptions, user = null) {
-    await saveMessage({ ...message, endpoint: this.options.endpoint, user, unfinished: false });
+    await saveMessage({
+      ...message,
+      endpoint: this.options.endpoint,
+      unfinished: false,
+      user,
+    });
     await saveConvo(user, {
       conversationId: message.conversationId,
       endpoint: this.options.endpoint,
@@ -556,11 +595,11 @@ class BaseClient {
    * the message is considered a root message.
    *
    * @param {Object} options - The options for the function.
-   * @param {Array} options.messages - An array of message objects. Each object should have either an 'id' or 'messageId' property, and may have a 'parentMessageId' property.
+   * @param {TMessage[]} options.messages - An array of message objects. Each object should have either an 'id' or 'messageId' property, and may have a 'parentMessageId' property.
    * @param {string} options.parentMessageId - The ID of the parent message to start the traversal from.
    * @param {Function} [options.mapMethod] - An optional function to map over the ordered messages. If provided, it will be applied to each message in the resulting array.
    * @param {boolean} [options.summary=false] - If set to true, the traversal modifies messages with 'summary' and 'summaryTokenCount' properties and stops at the message with a 'summary' property.
-   * @returns {Array} An array containing the messages in the order they should be displayed, starting with the most recent message with a 'summary' property if the 'summary' option is true, and ending with the message identified by 'parentMessageId'.
+   * @returns {TMessage[]} An array containing the messages in the order they should be displayed, starting with the most recent message with a 'summary' property if the 'summary' option is true, and ending with the message identified by 'parentMessageId'.
    */
   static getMessagesForConversation({
     messages,

api/app/clients/ChatGPTClient.js

@@ -3,10 +3,13 @@ const crypto = require('crypto');
 const {
   EModelEndpoint,
   resolveHeaders,
+  CohereConstants,
   mapModelToAzureConfig,
 } = require('librechat-data-provider');
+const { CohereClient } = require('cohere-ai');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
+const { createCoherePayload } = require('./llm');
 const { Agent, ProxyAgent } = require('undici');
 const BaseClient = require('./BaseClient');
 const { logger } = require('~/config');
@@ -147,7 +150,8 @@ class ChatGPTClient extends BaseClient {
     return tokenizer;
   }
 
-  async getCompletion(input, onProgress, abortController = null) {
+  /** @type {getCompletion} */
+  async getCompletion(input, onProgress, onTokenProgress, abortController = null) {
     if (!abortController) {
       abortController = new AbortController();
     }
@@ -305,6 +309,11 @@ class ChatGPTClient extends BaseClient {
       });
     }
 
+    if (baseURL.startsWith(CohereConstants.API_URL)) {
+      const payload = createCoherePayload({ modelOptions });
+      return await this.cohereChatCompletion({ payload, onTokenProgress });
+    }
+
     if (baseURL.includes('v1') && !baseURL.includes('/completions') && !this.isChatCompletion) {
       baseURL = baseURL.split('v1')[0] + 'v1/completions';
     } else if (
@@ -408,6 +417,43 @@ class ChatGPTClient extends BaseClient {
     return response.json();
   }
 
+  /** @type {cohereChatCompletion} */
+  async cohereChatCompletion({ payload, onTokenProgress }) {
+    const cohere = new CohereClient({
+      token: this.apiKey,
+      environment: this.completionsUrl,
+    });
+
+    if (!payload.stream) {
+      const chatResponse = await cohere.chat(payload);
+      return chatResponse.text;
+    }
+
+    const chatStream = await cohere.chatStream(payload);
+    let reply = '';
+    for await (const message of chatStream) {
+      if (!message) {
+        continue;
+      }
+
+      if (message.eventType === 'text-generation' && message.text) {
+        onTokenProgress(message.text);
+        reply += message.text;
+      }
+      /*
+      Cohere API Chinese Unicode character replacement hotfix.
+      Should be un-commented when the following issue is resolved:
+      https://github.com/cohere-ai/cohere-typescript/issues/151
+
+      else if (message.eventType === 'stream-end' && message.response) {
+        reply = message.response.text;
+      }
+      */
+    }
+
+    return reply;
+  }
+
   async generateTitle(userMessage, botMessage) {
     const instructionsPayload = {
       role: 'system',

api/app/clients/GoogleClient.js

@@ -1,7 +1,9 @@
 const { google } = require('googleapis');
 const { Agent, ProxyAgent } = require('undici');
-const { GoogleVertexAI } = require('langchain/llms/googlevertexai');
+const { ChatVertexAI } = require('@langchain/google-vertexai');
 const { ChatGoogleGenerativeAI } = require('@langchain/google-genai');
+const { GoogleGenerativeAI: GenAI } = require('@google/generative-ai');
+const { GoogleVertexAI } = require('@langchain/community/llms/googlevertexai');
 const { ChatGoogleVertexAI } = require('langchain/chat_models/googlevertexai');
 const { AIMessage, HumanMessage, SystemMessage } = require('langchain/schema');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
@@ -10,6 +12,7 @@ const {
   getResponseSender,
   endpointSettings,
   EModelEndpoint,
+  VisionModes,
   AuthKeys,
 } = require('librechat-data-provider');
 const { encodeAndFormat } = require('~/server/services/Files/images');
@@ -126,7 +129,7 @@ class GoogleClient extends BaseClient {
 
     this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments));
 
-    // TODO: as of 12/14/23, only gemini models are "Generative AI" models provided by Google
+    /** @type {boolean} Whether using a "GenerativeAI" Model */
     this.isGenerativeModel = this.modelOptions.model.includes('gemini');
     const { isGenerativeModel } = this;
     this.isChatModel = !isGenerativeModel && this.modelOptions.model.includes('chat');
@@ -135,7 +138,10 @@ class GoogleClient extends BaseClient {
       !isGenerativeModel && !isChatModel && /code|text/.test(this.modelOptions.model);
     const { isTextModel } = this;
 
-    this.maxContextTokens = getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google);
+    this.maxContextTokens =
+      this.options.maxContextTokens ??
+      getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google);
+
     // The max prompt tokens is determined by the max context tokens minus the max response tokens.
     // Earlier messages will be dropped until the prompt is within the limit.
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || settings.maxOutputTokens.default;
@@ -234,7 +240,7 @@ class GoogleClient extends BaseClient {
       this.isVisionModel = true;
     }
 
-    if (this.isVisionModel && !attachments) {
+    if (this.isVisionModel && !attachments && this.modelOptions.model.includes('gemini-pro')) {
       this.modelOptions.model = 'gemini-pro';
       this.isVisionModel = false;
     }
@@ -247,6 +253,40 @@ class GoogleClient extends BaseClient {
     })).bind(this);
   }
 
+  /**
+   * Formats messages for generative AI
+   * @param {TMessage[]} messages
+   * @returns
+   */
+  async formatGenerativeMessages(messages) {
+    const formattedMessages = [];
+    const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    const files = await this.addImageURLs(latestMessage, attachments, VisionModes.generative);
+    this.options.attachments = files;
+    messages[messages.length - 1] = latestMessage;
+
+    for (const _message of messages) {
+      const role = _message.isCreatedByUser ? this.userLabel : this.modelLabel;
+      const parts = [];
+      parts.push({ text: _message.text });
+      if (!_message.image_urls?.length) {
+        formattedMessages.push({ role, parts });
+        continue;
+      }
+
+      for (const images of _message.image_urls) {
+        if (images.inlineData) {
+          parts.push({ inlineData: images.inlineData });
+        }
+      }
+
+      formattedMessages.push({ role, parts });
+    }
+
+    return formattedMessages;
+  }
+
   /**
    *
    * Adds image URLs to the message object and returns the files
@@ -255,17 +295,23 @@ class GoogleClient extends BaseClient {
    * @param {MongoFile[]} files
    * @returns {Promise<MongoFile[]>}
    */
-  async addImageURLs(message, attachments) {
+  async addImageURLs(message, attachments, mode = '') {
     const { files, image_urls } = await encodeAndFormat(
       this.options.req,
       attachments,
       EModelEndpoint.google,
+      mode,
     );
     message.image_urls = image_urls.length ? image_urls : undefined;
     return files;
   }
 
-  async buildVisionMessages(messages = [], parentMessageId) {
+  /**
+   * Builds the augmented prompt for attachments
+   * TODO: Add File API Support
+   * @param {TMessage[]} messages
+   */
+  async buildAugmentedPrompt(messages = []) {
     const attachments = await this.options.attachments;
     const latestMessage = { ...messages[messages.length - 1] };
     this.contextHandlers = createContextHandlers(this.options.req, latestMessage.text);
@@ -281,6 +327,12 @@ class GoogleClient extends BaseClient {
       this.augmentedPrompt = await this.contextHandlers.createContext();
       this.options.promptPrefix = this.augmentedPrompt + this.options.promptPrefix;
     }
+  }
+
+  async buildVisionMessages(messages = [], parentMessageId) {
+    const attachments = await this.options.attachments;
+    const latestMessage = { ...messages[messages.length - 1] };
+    await this.buildAugmentedPrompt(messages);
 
     const { prompt } = await this.buildMessagesPrompt(messages, parentMessageId);
 
@@ -301,15 +353,26 @@ class GoogleClient extends BaseClient {
     return { prompt: payload };
   }
 
+  /** @param {TMessage[]} [messages=[]]  */
+  async buildGenerativeMessages(messages = []) {
+    this.userLabel = 'user';
+    this.modelLabel = 'model';
+    const promises = [];
+    promises.push(await this.formatGenerativeMessages(messages));
+    promises.push(this.buildAugmentedPrompt(messages));
+    const [formattedMessages] = await Promise.all(promises);
+    return { prompt: formattedMessages };
+  }
+
   async buildMessages(messages = [], parentMessageId) {
     if (!this.isGenerativeModel && !this.project_id) {
       throw new Error(
         '[GoogleClient] a Service Account JSON Key is required for PaLM 2 and Codey models (Vertex AI)',
       );
-    } else if (this.isGenerativeModel && (!this.apiKey || this.apiKey === 'user_provided')) {
-      throw new Error(
-        '[GoogleClient] an API Key is required for Gemini models (Generative Language API)',
-      );
+    }
+
+    if (!this.project_id && this.modelOptions.model.includes('1.5')) {
+      return await this.buildGenerativeMessages(messages);
     }
 
     if (this.options.attachments && this.isGenerativeModel) {
@@ -526,13 +589,24 @@ class GoogleClient extends BaseClient {
   }
 
   createLLM(clientOptions) {
-    if (this.isGenerativeModel) {
-      return new ChatGoogleGenerativeAI({ ...clientOptions, apiKey: this.apiKey });
+    const model = clientOptions.modelName ?? clientOptions.model;
+    if (this.project_id && this.isTextModel) {
+      return new GoogleVertexAI(clientOptions);
+    } else if (this.project_id && this.isChatModel) {
+      return new ChatGoogleVertexAI(clientOptions);
+    } else if (this.project_id) {
+      return new ChatVertexAI(clientOptions);
+    } else if (model.includes('1.5')) {
+      return new GenAI(this.apiKey).getGenerativeModel(
+        {
+          ...clientOptions,
+          model,
+        },
+        { apiVersion: 'v1beta' },
+      );
     }
 
-    return this.isTextModel
-      ? new GoogleVertexAI(clientOptions)
-      : new ChatGoogleVertexAI(clientOptions);
+    return new ChatGoogleGenerativeAI({ ...clientOptions, apiKey: this.apiKey });
   }
 
   async getCompletion(_payload, options = {}) {
@@ -544,7 +618,7 @@ class GoogleClient extends BaseClient {
 
     let clientOptions = { ...parameters, maxRetries: 2 };
 
-    if (!this.isGenerativeModel) {
+    if (this.project_id) {
       clientOptions['authOptions'] = {
         credentials: {
           ...this.serviceKey,
@@ -557,7 +631,7 @@ class GoogleClient extends BaseClient {
       clientOptions = { ...clientOptions, ...this.modelOptions };
     }
 
-    if (this.isGenerativeModel) {
+    if (this.isGenerativeModel && !this.project_id) {
       clientOptions.modelName = clientOptions.model;
       delete clientOptions.model;
     }
@@ -588,16 +662,56 @@ class GoogleClient extends BaseClient {
       messages.unshift(new SystemMessage(context));
     }
 
+    const modelName = clientOptions.modelName ?? clientOptions.model ?? '';
+    if (modelName?.includes('1.5') && !this.project_id) {
+      /** @type {GenerativeModel} */
+      const client = model;
+      const requestOptions = {
+        contents: _payload,
+      };
+
+      if (this.options?.promptPrefix?.length) {
+        requestOptions.systemInstruction = {
+          parts: [
+            {
+              text: this.options.promptPrefix,
+            },
+          ],
+        };
+      }
+
+      const safetySettings = _payload.safetySettings;
+      requestOptions.safetySettings = safetySettings;
+
+      const delay = modelName.includes('flash') ? 8 : 14;
+      const result = await client.generateContentStream(requestOptions);
+      for await (const chunk of result.stream) {
+        const chunkText = chunk.text();
+        await this.generateTextStream(chunkText, onProgress, {
+          delay,
+        });
+        reply += chunkText;
+      }
+      return reply;
+    }
+
+    const safetySettings = _payload.safetySettings;
     const stream = await model.stream(messages, {
       signal: abortController.signal,
       timeout: 7000,
+      safetySettings: safetySettings,
     });
 
+    let delay = this.isGenerativeModel ? 12 : 8;
+    if (modelName.includes('flash')) {
+      delay = 5;
+    }
     for await (const chunk of stream) {
-      await this.generateTextStream(chunk?.content ?? chunk, onProgress, {
-        delay: this.isGenerativeModel ? 12 : 8,
+      const chunkText = chunk?.content ?? chunk;
+      await this.generateTextStream(chunkText, onProgress, {
+        delay,
       });
-      reply += chunk?.content ?? chunk;
+      reply += chunkText;
     }
 
     return reply;
@@ -607,6 +721,9 @@ class GoogleClient extends BaseClient {
     return {
       promptPrefix: this.options.promptPrefix,
       modelLabel: this.options.modelLabel,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
@@ -616,6 +733,33 @@ class GoogleClient extends BaseClient {
   }
 
   async sendCompletion(payload, opts = {}) {
+    const modelName = payload.parameters?.model;
+
+    if (modelName && modelName.toLowerCase().includes('gemini')) {
+      const safetySettings = [
+        {
+          category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+          threshold:
+            process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_HATE_SPEECH',
+          threshold: process.env.GOOGLE_SAFETY_HATE_SPEECH || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_HARASSMENT',
+          threshold: process.env.GOOGLE_SAFETY_HARASSMENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+        {
+          category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+          threshold:
+            process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+        },
+      ];
+
+      payload.safetySettings = safetySettings;
+    }
+
     let reply = '';
     reply = await this.getCompletion(payload, opts);
     return reply.trim();

api/app/clients/OllamaClient.js

@@ -0,0 +1,154 @@
+const { z } = require('zod');
+const axios = require('axios');
+const { Ollama } = require('ollama');
+const { deriveBaseURL } = require('~/utils');
+const { logger } = require('~/config');
+
+const ollamaPayloadSchema = z.object({
+  mirostat: z.number().optional(),
+  mirostat_eta: z.number().optional(),
+  mirostat_tau: z.number().optional(),
+  num_ctx: z.number().optional(),
+  repeat_last_n: z.number().optional(),
+  repeat_penalty: z.number().optional(),
+  temperature: z.number().optional(),
+  seed: z.number().nullable().optional(),
+  stop: z.array(z.string()).optional(),
+  tfs_z: z.number().optional(),
+  num_predict: z.number().optional(),
+  top_k: z.number().optional(),
+  top_p: z.number().optional(),
+  stream: z.optional(z.boolean()),
+  model: z.string(),
+});
+
+/**
+ * @param {string} imageUrl
+ * @returns {string}
+ * @throws {Error}
+ */
+const getValidBase64 = (imageUrl) => {
+  const parts = imageUrl.split(';base64,');
+
+  if (parts.length === 2) {
+    return parts[1];
+  } else {
+    logger.error('Invalid or no Base64 string found in URL.');
+  }
+};
+
+class OllamaClient {
+  constructor(options = {}) {
+    const host = deriveBaseURL(options.baseURL ?? 'http://localhost:11434');
+    /** @type {Ollama} */
+    this.client = new Ollama({ host });
+  }
+
+  /**
+   * Fetches Ollama models from the specified base API path.
+   * @param {string} baseURL
+   * @returns {Promise<string[]>} The Ollama models.
+   */
+  static async fetchModels(baseURL) {
+    let models = [];
+    if (!baseURL) {
+      return models;
+    }
+    try {
+      const ollamaEndpoint = deriveBaseURL(baseURL);
+      /** @type {Promise<AxiosResponse<OllamaListResponse>>} */
+      const response = await axios.get(`${ollamaEndpoint}/api/tags`);
+      models = response.data.models.map((tag) => tag.name);
+      return models;
+    } catch (error) {
+      const logMessage =
+        'Failed to fetch models from Ollama API. If you are not using Ollama directly, and instead, through some aggregator or reverse proxy that handles fetching via OpenAI spec, ensure the name of the endpoint doesn\'t start with `ollama` (case-insensitive).';
+      logger.error(logMessage, error);
+      return [];
+    }
+  }
+
+  /**
+   * @param {ChatCompletionMessage[]} messages
+   * @returns {OllamaMessage[]}
+   */
+  static formatOpenAIMessages(messages) {
+    const ollamaMessages = [];
+
+    for (const message of messages) {
+      if (typeof message.content === 'string') {
+        ollamaMessages.push({
+          role: message.role,
+          content: message.content,
+        });
+        continue;
+      }
+
+      let aggregatedText = '';
+      let imageUrls = [];
+
+      for (const content of message.content) {
+        if (content.type === 'text') {
+          aggregatedText += content.text + ' ';
+        } else if (content.type === 'image_url') {
+          imageUrls.push(getValidBase64(content.image_url.url));
+        }
+      }
+
+      const ollamaMessage = {
+        role: message.role,
+        content: aggregatedText.trim(),
+      };
+
+      if (imageUrls.length > 0) {
+        ollamaMessage.images = imageUrls;
+      }
+
+      ollamaMessages.push(ollamaMessage);
+    }
+
+    return ollamaMessages;
+  }
+
+  /***
+   * @param {Object} params
+   * @param {ChatCompletionPayload} params.payload
+   * @param {onTokenProgress} params.onProgress
+   * @param {AbortController} params.abortController
+   */
+  async chatCompletion({ payload, onProgress, abortController = null }) {
+    let intermediateReply = '';
+
+    const parameters = ollamaPayloadSchema.parse(payload);
+    const messages = OllamaClient.formatOpenAIMessages(payload.messages);
+
+    if (parameters.stream) {
+      const stream = await this.client.chat({
+        messages,
+        ...parameters,
+      });
+
+      for await (const chunk of stream) {
+        const token = chunk.message.content;
+        intermediateReply += token;
+        onProgress(token);
+        if (abortController.signal.aborted) {
+          stream.controller.abort();
+          break;
+        }
+      }
+    }
+    // TODO: regular completion
+    else {
+      // const generation = await this.client.generate(payload);
+    }
+
+    return intermediateReply;
+  }
+  catch(err) {
+    logger.error('[OllamaClient.chatCompletion]', err);
+    throw err;
+  }
+}
+
+module.exports = { OllamaClient, ollamaPayloadSchema };

api/app/clients/OpenAIClient.js

@@ -1,10 +1,13 @@
 const OpenAI = require('openai');
+const { OllamaClient } = require('./OllamaClient');
 const { HttpsProxyAgent } = require('https-proxy-agent');
 const {
+  Constants,
   ImageDetail,
   EModelEndpoint,
   resolveHeaders,
   ImageDetailCost,
+  CohereConstants,
   getResponseSender,
   validateVisionModel,
   mapModelToAzureConfig,
@@ -16,13 +19,20 @@ const {
   getModelMaxTokens,
   genAzureChatCompletion,
 } = require('~/utils');
-const { truncateText, formatMessage, createContextHandlers, CUT_OFF_PROMPT } = require('./prompts');
+const {
+  truncateText,
+  formatMessage,
+  CUT_OFF_PROMPT,
+  titleInstruction,
+  createContextHandlers,
+} = require('./prompts');
 const { encodeAndFormat } = require('~/server/services/Files/images/encode');
+const { updateTokenWebsocket } = require('~/server/services/Files/Audio');
+const { isEnabled, sleep } = require('~/server/utils');
 const { handleOpenAIErrors } = require('./tools/util');
 const spendTokens = require('~/models/spendTokens');
 const { createLLM, RunManager } = require('./llm');
 const ChatGPTClient = require('./ChatGPTClient');
-const { isEnabled } = require('~/server/utils');
 const { summaryBuffer } = require('./memory');
 const { runTitleChain } = require('./chains');
 const { tokenSplit } = require('./document');
@@ -39,7 +49,10 @@ class OpenAIClient extends BaseClient {
     super(apiKey, options);
     this.ChatGPTClient = new ChatGPTClient();
     this.buildPrompt = this.ChatGPTClient.buildPrompt.bind(this);
+    /** @type {getCompletion} */
     this.getCompletion = this.ChatGPTClient.getCompletion.bind(this);
+    /** @type {cohereChatCompletion} */
+    this.cohereChatCompletion = this.ChatGPTClient.cohereChatCompletion.bind(this);
     this.contextStrategy = options.contextStrategy
       ? options.contextStrategy.toLowerCase()
       : 'discard';
@@ -48,6 +61,9 @@ class OpenAIClient extends BaseClient {
     this.azure = options.azure || false;
     this.setOptions(options);
     this.metadata = {};
+
+    /** @type {string | undefined} - The API Completions URL */
+    this.completionsUrl;
   }
 
   // TODO: PluginsClient calls this 3x, unneeded
@@ -114,6 +130,10 @@ class OpenAIClient extends BaseClient {
       this.useOpenRouter = true;
     }
 
+    if (this.options.endpoint?.toLowerCase() === 'ollama') {
+      this.isOllama = true;
+    }
+
     this.FORCE_PROMPT =
       isEnabled(OPENAI_FORCE_PROMPT) ||
       (reverseProxy && reverseProxy.includes('completions') && !reverseProxy.includes('chat'));
@@ -146,11 +166,13 @@ class OpenAIClient extends BaseClient {
       model.startsWith('text-chat') || model.startsWith('text-davinci-002-render');
 
     this.maxContextTokens =
+      this.options.maxContextTokens ??
       getModelMaxTokens(
         model,
         this.options.endpointType ?? this.options.endpoint,
         this.options.endpointTokenConfig,
-      ) ?? 4095; // 1 less than maximum
+      ) ??
+      4095; // 1 less than maximum
 
     if (this.shouldSummarize) {
       this.maxContextTokens = Math.floor(this.maxContextTokens / 2);
@@ -187,16 +209,6 @@ class OpenAIClient extends BaseClient {
 
     this.setupTokens();
 
-    if (!this.modelOptions.stop && !this.isVisionModel) {
-      const stopTokens = [this.startToken];
-      if (this.endToken && this.endToken !== this.startToken) {
-        stopTokens.push(this.endToken);
-      }
-      stopTokens.push(`\n${this.userLabel}:`);
-      stopTokens.push('<|diff_marker|>');
-      this.modelOptions.stop = stopTokens;
-    }
-
     if (reverseProxy) {
       this.completionsUrl = reverseProxy;
       this.langchainProxy = extractBaseURL(reverseProxy);
@@ -230,23 +242,52 @@ class OpenAIClient extends BaseClient {
    * @param {MongoFile[]} attachments
    */
   checkVisionRequest(attachments) {
-    const availableModels = this.options.modelsConfig?.[this.options.endpoint];
-    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
-
-    const visionModelAvailable = availableModels?.includes(this.defaultVisionModel);
-    if (
-      attachments &&
-      attachments.some((file) => file?.type && file?.type?.includes('image')) &&
-      visionModelAvailable &&
-      !this.isVisionModel
-    ) {
-      this.modelOptions.model = this.defaultVisionModel;
-      this.isVisionModel = true;
+    if (!attachments) {
+      return;
     }
 
+    const availableModels = this.options.modelsConfig?.[this.options.endpoint];
+    if (!availableModels) {
+      return;
+    }
+
+    let visionRequestDetected = false;
+    for (const file of attachments) {
+      if (file?.type?.includes('image')) {
+        visionRequestDetected = true;
+        break;
+      }
+    }
+    if (!visionRequestDetected) {
+      return;
+    }
+
+    this.isVisionModel = validateVisionModel({ model: this.modelOptions.model, availableModels });
     if (this.isVisionModel) {
       delete this.modelOptions.stop;
+      return;
     }
+
+    for (const model of availableModels) {
+      if (!validateVisionModel({ model, availableModels })) {
+        continue;
+      }
+      this.modelOptions.model = model;
+      this.isVisionModel = true;
+      delete this.modelOptions.stop;
+      return;
+    }
+
+    if (!availableModels.includes(this.defaultVisionModel)) {
+      return;
+    }
+    if (!validateVisionModel({ model: this.defaultVisionModel, availableModels })) {
+      return;
+    }
+
+    this.modelOptions.model = this.defaultVisionModel;
+    this.isVisionModel = true;
+    delete this.modelOptions.stop;
   }
 
   setupTokens() {
@@ -268,7 +309,7 @@ class OpenAIClient extends BaseClient {
     let tokenizer;
     this.encoding = 'text-davinci-003';
     if (this.isChatCompletion) {
-      this.encoding = 'cl100k_base';
+      this.encoding = this.modelOptions.model.includes('gpt-4o') ? 'o200k_base' : 'cl100k_base';
       tokenizer = this.constructor.getTokenizer(this.encoding);
     } else if (this.isUnofficialChatGptModel) {
       const extendSpecialTokens = {
@@ -373,10 +414,14 @@ class OpenAIClient extends BaseClient {
 
   getSaveOptions() {
     return {
+      maxContextTokens: this.options.maxContextTokens,
       chatGptLabel: this.options.chatGptLabel,
       promptPrefix: this.options.promptPrefix,
       resendFiles: this.options.resendFiles,
       imageDetail: this.options.imageDetail,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
       ...this.modelOptions,
     };
   }
@@ -398,7 +443,11 @@ class OpenAIClient extends BaseClient {
    * @returns {Promise<MongoFile[]>}
    */
   async addImageURLs(message, attachments) {
-    const { files, image_urls } = await encodeAndFormat(this.options.req, attachments);
+    const { files, image_urls } = await encodeAndFormat(
+      this.options.req,
+      attachments,
+      this.options.endpoint,
+    );
     message.image_urls = image_urls.length ? image_urls : undefined;
     return files;
   }
@@ -533,18 +582,20 @@ class OpenAIClient extends BaseClient {
     return result;
   }
 
+  /** @type {sendCompletion} */
   async sendCompletion(payload, opts = {}) {
     let reply = '';
     let result = null;
     let streamResult = null;
     this.modelOptions.user = this.user;
     const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null;
-    const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion || typeof Bun !== 'undefined');
+    const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion);
     if (typeof opts.onProgress === 'function' && useOldMethod) {
-      await this.getCompletion(
+      const completionResult = await this.getCompletion(
         payload,
         (progressMessage) => {
           if (progressMessage === '[DONE]') {
+            updateTokenWebsocket('[DONE]');
             return;
           }
 
@@ -574,8 +625,13 @@ class OpenAIClient extends BaseClient {
           opts.onProgress(token);
           reply += token;
         },
+        opts.onProgress,
         opts.abortController || new AbortController(),
       );
+
+      if (completionResult && typeof completionResult === 'string') {
+        reply = completionResult;
+      }
     } else if (typeof opts.onProgress === 'function' || this.options.useChatCompletion) {
       reply = await this.chatCompletion({
         payload,
@@ -586,9 +642,14 @@ class OpenAIClient extends BaseClient {
       result = await this.getCompletion(
         payload,
         null,
+        opts.onProgress,
         opts.abortController || new AbortController(),
       );
 
+      if (result && typeof result === 'string') {
+        return result.trim();
+      }
+
       logger.debug('[OpenAIClient] sendCompletion: result', result);
 
       if (this.isChatCompletion) {
@@ -697,6 +758,12 @@ class OpenAIClient extends BaseClient {
    *                            In case of failure, it will return the default title, "New Chat".
    */
   async titleConvo({ text, conversationId, responseText = '' }) {
+    this.conversationId = conversationId;
+
+    if (this.options.attachments) {
+      delete this.options.attachments;
+    }
+
     let title = 'New Chat';
     const convo = `||>User:
 "${truncateText(text)}"
@@ -705,7 +772,10 @@ class OpenAIClient extends BaseClient {
 
     const { OPENAI_TITLE_MODEL } = process.env ?? {};
 
-    const model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+    let model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+    if (model === Constants.CURRENT_MODEL) {
+      model = this.modelOptions.model;
+    }
 
     const modelOptions = {
       // TODO: remove the gpt fallback and make it specific to endpoint
@@ -759,9 +829,8 @@ class OpenAIClient extends BaseClient {
 
       const instructionsPayload = [
         {
-          role: 'system',
-          content: `Detect user language and write in the same language an extremely concise title for this conversation, which you must accurately detect.
-Write in the detected language. Title in 5 Words or Less. No Punctuation or Quotation. Do not mention the language. All first letters of every word should be capitalized and write the title in User Language only.
+          role: this.options.titleMessageRole ?? 'system',
+          content: `Please generate ${titleInstruction}
 
 ${convo}
 
@@ -769,10 +838,22 @@ ${convo}
         },
       ];
 
+      const promptTokens = this.getTokenCountForMessage(instructionsPayload[0]);
+
       try {
+        let useChatCompletion = true;
+
+        if (this.options.reverseProxyUrl === CohereConstants.API_URL) {
+          useChatCompletion = false;
+        }
+
         title = (
-          await this.sendPayload(instructionsPayload, { modelOptions, useChatCompletion: true })
+          await this.sendPayload(instructionsPayload, { modelOptions, useChatCompletion })
         ).replaceAll('"', '');
+
+        const completionTokens = this.getTokenCount(title);
+
+        this.recordTokenUsage({ promptTokens, completionTokens, context: 'title' });
       } catch (e) {
         logger.error(
           '[OpenAIClient] There was an issue generating the title with the completion method',
@@ -795,6 +876,7 @@ ${convo}
         context: 'title',
         tokenBuffer: 150,
       });
+
       title = await runTitleChain({ llm, text, convo, signal: this.abortController.signal });
     } catch (e) {
       if (e?.message?.toLowerCase()?.includes('abort')) {
@@ -820,7 +902,11 @@ ${convo}
 
     // TODO: remove the gpt fallback and make it specific to endpoint
     const { OPENAI_SUMMARY_MODEL = 'gpt-3.5-turbo' } = process.env ?? {};
-    const model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL;
+    let model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL;
+    if (model === Constants.CURRENT_MODEL) {
+      model = this.modelOptions.model;
+    }
+
     const maxContextTokens =
       getModelMaxTokens(
         model,
@@ -924,13 +1010,13 @@ ${convo}
     }
   }
 
-  async recordTokenUsage({ promptTokens, completionTokens }) {
+  async recordTokenUsage({ promptTokens, completionTokens, context = 'message' }) {
     await spendTokens(
       {
-        user: this.user,
+        context,
         model: this.modelOptions.model,
-        context: 'message',
         conversationId: this.conversationId,
+        user: this.user ?? this.options.req.user?.id,
         endpointTokenConfig: this.options.endpointTokenConfig,
       },
       { promptTokens, completionTokens },
@@ -1022,7 +1108,12 @@ ${convo}
       }
 
       if (this.azure || this.options.azure) {
-        // Azure does not accept `model` in the body, so we need to remove it.
+        /* Azure Bug, extremely short default `max_tokens` response */
+        if (!modelOptions.max_tokens && modelOptions.model === 'gpt-4-vision-preview') {
+          modelOptions.max_tokens = 4000;
+        }
+
+        /* Azure does not accept `model` in the body, so we need to remove it. */
         delete modelOptions.model;
 
         opts.baseURL = this.langchainProxy
@@ -1043,15 +1134,13 @@ ${convo}
       let chatCompletion;
       /** @type {OpenAI} */
       const openai = new OpenAI({
+        fetch: this.fetch,
         apiKey: this.apiKey,
         ...opts,
       });
 
-      /* hacky fixes for Mistral AI API:
-      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
-      - If there is only one message and it's a system message, change the role to user
-      */
-      if (opts.baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+      /* Re-orders system message to the top of the messages payload, as not allowed anywhere else */
+      if (modelOptions.messages && (opts.baseURL.includes('api.mistral.ai') || this.isOllama)) {
         const { messages } = modelOptions;
 
         const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
@@ -1062,10 +1151,16 @@ ${convo}
         }
 
         modelOptions.messages = messages;
+      }
 
-        if (messages.length === 1 && messages[0].role === 'system') {
-          modelOptions.messages[0].role = 'user';
-        }
+      /* If there is only one message and it's a system message, change the role to user */
+      if (
+        (opts.baseURL.includes('api.mistral.ai') || opts.baseURL.includes('api.perplexity.ai')) &&
+        modelOptions.messages &&
+        modelOptions.messages.length === 1 &&
+        modelOptions.messages[0]?.role === 'system'
+      ) {
+        modelOptions.messages[0].role = 'user';
       }
 
       if (this.options.addParams && typeof this.options.addParams === 'object') {
@@ -1089,6 +1184,15 @@ ${convo}
         });
       }
 
+      if (this.message_file_map && this.isOllama) {
+        const ollamaClient = new OllamaClient({ baseURL });
+        return await ollamaClient.chatCompletion({
+          payload: modelOptions,
+          onProgress,
+          abortController,
+        });
+      }
+
       let UnexpectedRoleError = false;
       if (modelOptions.stream) {
         const stream = await openai.beta.chat.completions
@@ -1119,6 +1223,8 @@ ${convo}
             }
           });
 
+        const azureDelay = this.modelOptions.model?.includes('gpt-4') ? 30 : 17;
+
         for await (const chunk of stream) {
           const token = chunk.choices[0]?.delta?.content || '';
           intermediateReply += token;
@@ -1127,6 +1233,10 @@ ${convo}
             stream.controller.abort();
             break;
           }
+
+          if (this.azure) {
+            await sleep(azureDelay);
+          }
         }
 
         if (!UnexpectedRoleError) {

api/app/clients/PluginsClient.js

@@ -42,8 +42,12 @@ class PluginsClient extends OpenAIClient {
     return {
       chatGptLabel: this.options.chatGptLabel,
       promptPrefix: this.options.promptPrefix,
+      tools: this.options.tools,
       ...this.modelOptions,
       agentOptions: this.agentOptions,
+      iconURL: this.options.iconURL,
+      greeting: this.options.greeting,
+      spec: this.options.spec,
     };
   }
 
@@ -144,9 +148,11 @@ class PluginsClient extends OpenAIClient {
       signal,
       pastMessages,
       tools: this.tools,
-      currentDateString: this.currentDateString,
       verbose: this.options.debug,
       returnIntermediateSteps: true,
+      customName: this.options.chatGptLabel,
+      currentDateString: this.currentDateString,
+      customInstructions: this.options.promptPrefix,
       callbackManager: CallbackManager.fromHandlers({
         async handleAgentAction(action, runId) {
           handleAction(action, runId, onAgentAction);
@@ -244,7 +250,8 @@ class PluginsClient extends OpenAIClient {
       this.setOptions(opts);
       return super.sendMessage(message, opts);
     }
-    logger.debug('[PluginsClient] sendMessage', { message, opts });
+
+    logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts });
     const {
       user,
       isEdited,
@@ -258,6 +265,14 @@ class PluginsClient extends OpenAIClient {
       onToolEnd,
     } = await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     this.currentMessages.push(userMessage);
 
     let {
@@ -304,6 +319,8 @@ class PluginsClient extends OpenAIClient {
     }
 
     const responseMessage = {
+      endpoint: EModelEndpoint.gptPlugins,
+      iconURL: this.options.iconURL,
       messageId: responseMessageId,
       conversationId,
       parentMessageId: userMessage.messageId,

api/app/clients/agents/CustomAgent/initializeCustomAgent.js

@@ -13,10 +13,18 @@ const initializeCustomAgent = async ({
   tools,
   model,
   pastMessages,
+  customName,
+  customInstructions,
   currentDateString,
   ...rest
 }) => {
   let prompt = CustomAgent.createPrompt(tools, { currentDateString, model: model.modelName });
+  if (customName) {
+    prompt = `You are "${customName}".\n${prompt}`;
+  }
+  if (customInstructions) {
+    prompt = `${prompt}\n${customInstructions}`;
+  }
 
   const chatPrompt = ChatPromptTemplate.fromMessages([
     new SystemMessagePromptTemplate(prompt),

api/app/clients/agents/Functions/initializeFunctionsAgent.js

@@ -10,6 +10,8 @@ const initializeFunctionsAgent = async ({
   tools,
   model,
   pastMessages,
+  customName,
+  customInstructions,
   currentDateString,
   ...rest
 }) => {
@@ -24,7 +26,13 @@ const initializeFunctionsAgent = async ({
     returnMessages: true,
   });
 
-  const prefix = addToolDescriptions(`Current Date: ${currentDateString}\n${PREFIX}`, tools);
+  let prefix = addToolDescriptions(`Current Date: ${currentDateString}\n${PREFIX}`, tools);
+  if (customName) {
+    prefix = `You are "${customName}".\n${prefix}`;
+  }
+  if (customInstructions) {
+    prefix = `${prefix}\n${customInstructions}`;
+  }
 
   return await initializeAgentExecutorWithOptions(tools, model, {
     agentType: 'openai-functions',

api/app/clients/llm/createCoherePayload.js

@@ -0,0 +1,85 @@
+const { CohereConstants } = require('librechat-data-provider');
+const { titleInstruction } = require('../prompts/titlePrompts');
+
+// Mapping OpenAI roles to Cohere roles
+const roleMap = {
+  user: CohereConstants.ROLE_USER,
+  assistant: CohereConstants.ROLE_CHATBOT,
+  system: CohereConstants.ROLE_SYSTEM, // Recognize and map the system role explicitly
+};
+
+/**
+ * Adjusts an OpenAI ChatCompletionPayload to conform with Cohere's expected chat payload format.
+ * Now includes handling for "system" roles explicitly mentioned.
+ *
+ * @param {Object} options - Object containing the model options.
+ * @param {ChatCompletionPayload} options.modelOptions - The OpenAI model payload options.
+ * @returns {CohereChatStreamRequest} Cohere-compatible chat API payload.
+ */
+function createCoherePayload({ modelOptions }) {
+  /** @type {string | undefined} */
+  let preamble;
+  let latestUserMessageContent = '';
+  const {
+    stream,
+    stop,
+    top_p,
+    temperature,
+    frequency_penalty,
+    presence_penalty,
+    max_tokens,
+    messages,
+    model,
+    ...rest
+  } = modelOptions;
+
+  // Filter out the latest user message and transform remaining messages to Cohere's chat_history format
+  let chatHistory = messages.reduce((acc, message, index, arr) => {
+    const isLastUserMessage = index === arr.length - 1 && message.role === 'user';
+
+    const messageContent =
+      typeof message.content === 'string'
+        ? message.content
+        : message.content.map((part) => (part.type === 'text' ? part.text : '')).join(' ');
+
+    if (isLastUserMessage) {
+      latestUserMessageContent = messageContent;
+    } else {
+      acc.push({
+        role: roleMap[message.role] || CohereConstants.ROLE_USER,
+        message: messageContent,
+      });
+    }
+
+    return acc;
+  }, []);
+
+  if (
+    chatHistory.length === 1 &&
+    chatHistory[0].role === CohereConstants.ROLE_SYSTEM &&
+    !latestUserMessageContent.length
+  ) {
+    const message = chatHistory[0].message;
+    latestUserMessageContent = message.includes(titleInstruction)
+      ? CohereConstants.TITLE_MESSAGE
+      : '.';
+    preamble = message;
+  }
+
+  return {
+    message: latestUserMessageContent,
+    model: model,
+    chatHistory,
+    stream: stream ?? false,
+    temperature: temperature,
+    frequencyPenalty: frequency_penalty,
+    presencePenalty: presence_penalty,
+    maxTokens: max_tokens,
+    stopSequences: stop,
+    preamble,
+    p: top_p,
+    ...rest,
+  };
+}
+
+module.exports = createCoherePayload;

api/app/clients/llm/index.js

@@ -1,7 +1,9 @@
 const createLLM = require('./createLLM');
 const RunManager = require('./RunManager');
+const createCoherePayload = require('./createCoherePayload');
 
 module.exports = {
   createLLM,
   RunManager,
+  createCoherePayload,
 };

api/app/clients/prompts/createContextHandlers.js

@@ -1,5 +1,6 @@
 const axios = require('axios');
 const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
 
 const footer = `Use the context as your learned knowledge to better answer the user.
 
@@ -55,7 +56,7 @@ function createContextHandlers(req, userMessageContent) {
         processedFiles.push(file);
         processedIds.add(file.file_id);
       } catch (error) {
-        console.error(`Error processing file ${file.filename}:`, error);
+        logger.error(`Error processing file ${file.filename}:`, error);
       }
     }
   };
@@ -144,8 +145,8 @@ function createContextHandlers(req, userMessageContent) {
 
       return prompt;
     } catch (error) {
-      console.error('Error creating context:', error);
-      throw error; // Re-throw the error to propagate it to the caller
+      logger.error('Error creating context:', error);
+      throw error;
     }
   };
 

api/app/clients/prompts/index.js

@@ -13,7 +13,7 @@ module.exports = {
   ...handleInputs,
   ...instructions,
   ...titlePrompts,
-  truncateText,
+  ...truncateText,
   createVisionPrompt,
   createContextHandlers,
 };

api/app/clients/prompts/titlePrompts.js

@@ -27,6 +27,8 @@ ${convo}`,
   return titlePrompt;
 };
 
+const titleInstruction =
+  'a concise, 5-word-or-less title for the conversation, using its same language, with no punctuation. Apply title case conventions appropriate for the language. Never directly mention the language name or the word "title"';
 const titleFunctionPrompt = `In this environment you have access to a set of tools you can use to generate the conversation title.
   
 You may call them like this:
@@ -51,36 +53,70 @@ Submit a brief title in the conversation's language, following the parameter des
 <parameter>
 <name>title</name>
 <type>string</type>
-<description>A concise, 5-word-or-less title for the conversation, using its same language, with no punctuation. Apply title case conventions appropriate for the language. For English, use AP Stylebook Title Case. Never directly mention the language name or the word "title"</description>
+<description>${titleInstruction}</description>
+</parameter>
+</parameters>
+</tool_description>
+</tools>`;
+
+const genTranslationPrompt = (
+  translationPrompt,
+) => `In this environment you have access to a set of tools you can use to translate text.
+  
+You may call them like this:
+<function_calls>
+<invoke>
+<tool_name>$TOOL_NAME</tool_name>
+<parameters>
+<$PARAMETER_NAME>$PARAMETER_VALUE</$PARAMETER_NAME>
+...
+</parameters>
+</invoke>
+</function_calls>
+
+Here are the tools available:
+<tools>
+<tool_description>
+<tool_name>submit_translation</tool_name>
+<description>
+Submit a translation in the target language, following the parameter description and its language closely.
+</description>
+<parameters>
+<parameter>
+<name>translation</name>
+<type>string</type>
+<description>${translationPrompt}
+ONLY include the generated translation without quotations, nor its related key</description>
 </parameter>
 </parameters>
 </tool_description>
 </tools>`;
 
 /**
- * Parses titles from title functions based on the provided prompt.
- * @param {string} prompt - The prompt containing the title function.
- * @returns {string} The parsed title. "New Chat" if no title is found.
+ * Parses specified parameter from the provided prompt.
+ * @param {string} prompt - The prompt containing the desired parameter.
+ * @param {string} paramName - The name of the parameter to extract.
+ * @returns {string} The parsed parameter's value or a default value if not found.
  */
-function parseTitleFromPrompt(prompt) {
-  const titleRegex = /<title>(.+?)<\/title>/;
-  const titleMatch = prompt.match(titleRegex);
+function parseParamFromPrompt(prompt, paramName) {
+  const paramRegex = new RegExp(`<${paramName}>([\\s\\S]+?)</${paramName}>`);
+  const paramMatch = prompt.match(paramRegex);
 
-  if (titleMatch && titleMatch[1]) {
-    const title = titleMatch[1].trim();
-
-    // // Capitalize the first letter of each word; Note: unnecessary due to title case prompting
-    // const capitalizedTitle = title.replace(/\b\w/g, (char) => char.toUpperCase());
-
-    return title;
+  if (paramMatch && paramMatch[1]) {
+    return paramMatch[1].trim();
   }
 
-  return 'New Chat';
+  if (prompt && prompt.length) {
+    return `NO TOOL INVOCATION: ${prompt}`;
+  }
+  return `No ${paramName} provided`;
 }
 
 module.exports = {
   langPrompt,
+  titleInstruction,
   createTitlePrompt,
   titleFunctionPrompt,
-  parseTitleFromPrompt,
+  parseParamFromPrompt,
+  genTranslationPrompt,
 };

api/app/clients/prompts/truncateText.js

@@ -1,10 +1,40 @@
 const MAX_CHAR = 255;
 
-function truncateText(text) {
-  if (text.length > MAX_CHAR) {
-    return `${text.slice(0, MAX_CHAR)}... [text truncated for brevity]`;
+/**
+ * Truncates a given text to a specified maximum length, appending ellipsis and a notification
+ * if the original text exceeds the maximum length.
+ *
+ * @param {string} text - The text to be truncated.
+ * @param {number} [maxLength=MAX_CHAR] - The maximum length of the text after truncation. Defaults to MAX_CHAR.
+ * @returns {string} The truncated text if the original text length exceeds maxLength, otherwise returns the original text.
+ */
+function truncateText(text, maxLength = MAX_CHAR) {
+  if (text.length > maxLength) {
+    return `${text.slice(0, maxLength)}... [text truncated for brevity]`;
   }
   return text;
 }
 
-module.exports = truncateText;
+/**
+ * Truncates a given text to a specified maximum length by showing the first half and the last half of the text,
+ * separated by ellipsis. This method ensures the output does not exceed the maximum length, including the addition
+ * of ellipsis and notification if the original text exceeds the maximum length.
+ *
+ * @param {string} text - The text to be truncated.
+ * @param {number} [maxLength=MAX_CHAR] - The maximum length of the output text after truncation. Defaults to MAX_CHAR.
+ * @returns {string} The truncated text showing the first half and the last half, or the original text if it does not exceed maxLength.
+ */
+function smartTruncateText(text, maxLength = MAX_CHAR) {
+  const ellipsis = '...';
+  const notification = ' [text truncated for brevity]';
+  const halfMaxLength = Math.floor((maxLength - ellipsis.length - notification.length) / 2);
+
+  if (text.length > maxLength) {
+    const startLastHalf = text.length - halfMaxLength;
+    return `${text.slice(0, halfMaxLength)}${ellipsis}${text.slice(startLastHalf)}${notification}`;
+  }
+
+  return text;
+}
+
+module.exports = { truncateText, smartTruncateText };

api/app/clients/specs/FakeClient.js

@@ -40,7 +40,8 @@ class FakeClient extends BaseClient {
       };
     }
 
-    this.maxContextTokens = getModelMaxTokens(this.modelOptions.model) ?? 4097;
+    this.maxContextTokens =
+      this.options.maxContextTokens ?? getModelMaxTokens(this.modelOptions.model) ?? 4097;
   }
   buildMessages() {}
   getTokenCount(str) {

api/app/clients/specs/OpenAIClient.test.js

@@ -144,6 +144,7 @@ describe('OpenAIClient', () => {
 
   const defaultOptions = {
     // debug: true,
+    req: {},
     openaiApiKey: 'new-api-key',
     modelOptions: {
       model,
@@ -157,12 +158,19 @@ describe('OpenAIClient', () => {
     azureOpenAIApiVersion: '2020-07-01-preview',
   };
 
+  let originalWarn;
+
   beforeAll(() => {
-    jest.spyOn(console, 'warn').mockImplementation(() => {});
+    originalWarn = console.warn;
+    console.warn = jest.fn();
   });
 
   afterAll(() => {
-    console.warn.mockRestore();
+    console.warn = originalWarn;
+  });
+
+  beforeEach(() => {
+    console.warn.mockClear();
   });
 
   beforeEach(() => {
@@ -662,4 +670,35 @@ describe('OpenAIClient', () => {
       expect(constructorArgs.baseURL).toBe(expectedURL);
     });
   });
+
+  describe('checkVisionRequest functionality', () => {
+    let client;
+    const attachments = [{ type: 'image/png' }];
+
+    beforeEach(() => {
+      client = new OpenAIClient('test-api-key', {
+        endpoint: 'ollama',
+        modelOptions: {
+          model: 'initial-model',
+        },
+        modelsConfig: {
+          ollama: ['initial-model', 'llava', 'other-model'],
+        },
+      });
+
+      client.defaultVisionModel = 'non-valid-default-model';
+    });
+
+    afterEach(() => {
+      jest.restoreAllMocks();
+    });
+
+    it('should set "llava" as the model if it is the first valid model when default validation fails', () => {
+      client.checkVisionRequest(attachments);
+
+      expect(client.modelOptions.model).toBe('llava');
+      expect(client.isVisionModel).toBeTruthy();
+      expect(client.modelOptions.stop).toBeUndefined();
+    });
+  });
 });

api/app/clients/tools/manifest.json

@@ -24,7 +24,7 @@
         "description": "This is your Google Custom Search Engine ID. For instructions on how to obtain this, see <a href='https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md'>Our Docs</a>."
       },
       {
-        "authField": "GOOGLE_API_KEY",
+        "authField": "GOOGLE_SEARCH_API_KEY",
         "label": "Google API Key",
         "description": "This is your Google Custom Search API Key. For instructions on how to obtain this, see <a href='https://github.com/danny-avila/LibreChat/blob/main/docs/features/plugins/google_search.md'>Our Docs</a>."
       }
@@ -60,7 +60,7 @@
     "name": "CodeSherpa",
     "pluginKey": "codesherpa_tools",
     "description": "[Experimental] A REPL for your chat. Requires https://github.com/iamgreggarcia/codesherpa",
-    "icon": "https://github.com/iamgreggarcia/codesherpa/blob/main/localserver/_logo.png",
+    "icon": "https://raw.githubusercontent.com/iamgreggarcia/codesherpa/main/localserver/_logo.png",
     "authConfig": [
       {
         "authField": "CODESHERPA_SERVER_URL",

api/app/clients/tools/structured/DALLE3.js

@@ -12,14 +12,15 @@ const { logger } = require('~/config');
 class DALLE3 extends Tool {
   constructor(fields = {}) {
     super();
-    /* Used to initialize the Tool without necessary variables. */
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
     this.override = fields.override ?? false;
-    /* Necessary for output to contain all image metadata. */
+    /** @type {boolean} Necessary for output to contain all image metadata. */
     this.returnMetadata = fields.returnMetadata ?? false;
 
     this.userId = fields.userId;
     this.fileStrategy = fields.fileStrategy;
     if (fields.processFileURL) {
+      /** @type {processFileURL} Necessary for output to contain all image metadata. */
       this.processFileURL = fields.processFileURL.bind(this);
     }
 
@@ -43,6 +44,7 @@ class DALLE3 extends Tool {
       config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
     }
 
+    /** @type {OpenAI} */
     this.openai = new OpenAI(config);
     this.name = 'dalle';
     this.description = `Use DALLE to create images from text descriptions.
@@ -164,13 +166,7 @@ Error Message: ${error.message}`;
       });
 
       if (this.returnMetadata) {
-        this.result = {
-          file_id: result.file_id,
-          filename: result.filename,
-          filepath: result.filepath,
-          height: result.height,
-          width: result.width,
-        };
+        this.result = result;
       } else {
         this.result = this.wrapInMarkdown(result.filepath);
       }

api/app/clients/tools/structured/GoogleSearch.js

@@ -9,7 +9,7 @@ class GoogleSearchResults extends Tool {
 
   constructor(fields = {}) {
     super(fields);
-    this.envVarApiKey = 'GOOGLE_API_KEY';
+    this.envVarApiKey = 'GOOGLE_SEARCH_API_KEY';
     this.envVarSearchEngineId = 'GOOGLE_CSE_ID';
     this.override = fields.override ?? false;
     this.apiKey = fields.apiKey ?? getEnvironmentVariable(this.envVarApiKey);

api/app/clients/tools/structured/StableDiffusion.js

@@ -4,14 +4,27 @@ const { z } = require('zod');
 const path = require('path');
 const axios = require('axios');
 const sharp = require('sharp');
+const { v4: uuidv4 } = require('uuid');
 const { StructuredTool } = require('langchain/tools');
+const { FileContext } = require('librechat-data-provider');
+const paths = require('~/config/paths');
 const { logger } = require('~/config');
 
 class StableDiffusionAPI extends StructuredTool {
   constructor(fields) {
     super();
-    /* Used to initialize the Tool without necessary variables. */
+    /** @type {string} User ID */
+    this.userId = fields.userId;
+    /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
+    this.req = fields.req;
+    /** @type {boolean} Used to initialize the Tool without necessary variables. */
     this.override = fields.override ?? false;
+    /** @type {boolean} Necessary for output to contain all image metadata. */
+    this.returnMetadata = fields.returnMetadata ?? false;
+    if (fields.uploadImageBuffer) {
+      /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */
+      this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);
+    }
 
     this.name = 'stable-diffusion';
     this.url = fields.SD_WEBUI_URL || this.getServerURL();
@@ -47,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {
 
   getMarkdownImageUrl(imageName) {
     const imageUrl = path
-      .join(this.relativeImageUrl, imageName)
+      .join(this.relativePath, this.userId, imageName)
       .replace(/\\/g, '/')
       .replace('public/', '');
     return `![generated image](/${imageUrl})`;
@@ -67,52 +80,78 @@ class StableDiffusionAPI extends StructuredTool {
     const payload = {
       prompt,
       negative_prompt,
-      sampler_index: 'DPM++ 2M Karras',
       cfg_scale: 4.5,
       steps: 22,
       width: 1024,
       height: 1024,
     };
-    const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
-    const image = response.data.images[0];
-    const pngPayload = { image: `data:image/png;base64,${image}` };
-    const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);
-    const info = response2.data.info;
+    let generationResponse;
+    try {
+      generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while generating image:', error);
+      return 'Error making API request.';
+    }
+    const image = generationResponse.data.images[0];
 
-    // Generate unique name
-    const imageName = `${Date.now()}.png`;
-    this.outputPath = path.resolve(
-      __dirname,
-      '..',
-      '..',
-      '..',
-      '..',
-      '..',
-      'client',
-      'public',
-      'images',
-    );
-    const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
-    this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+    /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */
+    let info = {};
+    try {
+      info = JSON.parse(generationResponse.data.info);
+    } catch (error) {
+      logger.error('[StableDiffusion] Error while getting image metadata:', error);
+    }
 
-    // Check if directory exists, if not create it
-    if (!fs.existsSync(this.outputPath)) {
-      fs.mkdirSync(this.outputPath, { recursive: true });
+    const file_id = uuidv4();
+    const imageName = `${file_id}.png`;
+    const { imageOutput: imageOutputPath, clientPath } = paths;
+    const filepath = path.join(imageOutputPath, this.userId, imageName);
+    this.relativePath = path.relative(clientPath, imageOutputPath);
+
+    if (!fs.existsSync(path.join(imageOutputPath, this.userId))) {
+      fs.mkdirSync(path.join(imageOutputPath, this.userId), { recursive: true });
     }
 
     try {
       const buffer = Buffer.from(image.split(',', 1)[0], 'base64');
+      if (this.returnMetadata && this.uploadImageBuffer && this.req) {
+        const file = await this.uploadImageBuffer({
+          req: this.req,
+          context: FileContext.image_generation,
+          resize: false,
+          metadata: {
+            buffer,
+            height: info.height,
+            width: info.width,
+            bytes: Buffer.byteLength(buffer),
+            filename: imageName,
+            type: 'image/png',
+            file_id,
+          },
+        });
+
+        const generationInfo = info.infotexts[0].split('\n').pop();
+        return {
+          ...file,
+          prompt,
+          metadata: {
+            negative_prompt,
+            seed: info.seed,
+            info: generationInfo,
+          },
+        };
+      }
+
       await sharp(buffer)
         .withMetadata({
           iptcpng: {
-            parameters: info,
+            parameters: info.infotexts[0],
           },
         })
-        .toFile(this.outputPath + '/' + imageName);
+        .toFile(filepath);
       this.result = this.getMarkdownImageUrl(imageName);
     } catch (error) {
       logger.error('[StableDiffusion] Error while saving the image:', error);
-      // this.result = theImageUrl;
     }
 
     return this.result;

api/app/clients/tools/util/handleTools.js

@@ -237,9 +237,11 @@ const loadTools = async ({
   }
 
   const imageGenOptions = {
+    req: options.req,
     fileStrategy: options.fileStrategy,
     processFileURL: options.processFileURL,
     returnMetadata: options.returnMetadata,
+    uploadImageBuffer: options.uploadImageBuffer,
   };
 
   const toolOptions = {

api/app/clients/tools/util/loadToolSuite.js

@@ -1,5 +1,6 @@
 const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const { availableTools } = require('../');
+const { logger } = require('~/config');
 
 /**
  * Loads a suite of tools with authentication values for a given user, supporting alternate authentication fields.
@@ -30,7 +31,7 @@ const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
           return value;
         }
       } catch (err) {
-        console.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
+        logger.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
       }
     }
     return null;
@@ -41,7 +42,7 @@ const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
     if (authValue !== null) {
       authValues[auth.authField] = authValue;
     } else {
-      console.warn(`No auth value found for ${auth.authField}`);
+      logger.warn(`[loadToolSuite] No auth value found for ${auth.authField}`);
     }
   }
 

api/cache/banViolation.js

@@ -1,6 +1,7 @@
-const Session = require('~/models/Session');
-const getLogStores = require('./getLogStores');
+const { ViolationTypes } = require('librechat-data-provider');
 const { isEnabled, math, removePorts } = require('~/server/utils');
+const getLogStores = require('./getLogStores');
+const Session = require('~/models/Session');
 const { logger } = require('~/config');
 
 const { BAN_VIOLATIONS, BAN_INTERVAL } = process.env ?? {};
@@ -48,7 +49,7 @@ const banViolation = async (req, res, errorMessage) => {
   await Session.deleteAllUserSessions(user_id);
   res.clearCookie('refreshToken');
 
-  const banLogs = getLogStores('ban');
+  const banLogs = getLogStores(ViolationTypes.BAN);
   const duration = errorMessage.duration || banLogs.opts.ttl;
 
   if (duration <= 0) {

api/cache/banViolation.spec.js

@@ -6,6 +6,7 @@ jest.mock('../models/Session');
 jest.mock('./getLogStores', () => {
   return jest.fn().mockImplementation(() => {
     const EventEmitter = require('events');
+    const { CacheKeys } = require('librechat-data-provider');
     const math = require('../server/utils/math');
     const mockGet = jest.fn();
     const mockSet = jest.fn();
@@ -33,7 +34,7 @@ jest.mock('./getLogStores', () => {
     }
 
     return new KeyvMongo('', {
-      namespace: 'bans',
+      namespace: CacheKeys.BANS,
       ttl: math(process.env.BAN_DURATION, 7200000),
     });
   });

api/cache/getLogStores.js

@@ -6,6 +6,8 @@ const keyvRedis = require('./keyvRedis');
 const keyvMongo = require('./keyvMongo');
 
 const { BAN_DURATION, USE_REDIS } = process.env ?? {};
+const THIRTY_MINUTES = 1800000;
+const TEN_MINUTES = 600000;
 
 const duration = math(BAN_DURATION, 7200000);
 
@@ -23,9 +25,13 @@ const config = isEnabled(USE_REDIS)
   ? new Keyv({ store: keyvRedis })
   : new Keyv({ namespace: CacheKeys.CONFIG_STORE });
 
+const audioRuns = isEnabled(USE_REDIS) // ttl: 30 minutes
+  ? new Keyv({ store: keyvRedis, ttl: TEN_MINUTES })
+  : new Keyv({ namespace: CacheKeys.AUDIO_RUNS, ttl: TEN_MINUTES });
+
 const tokenConfig = isEnabled(USE_REDIS) // ttl: 30 minutes
-  ? new Keyv({ store: keyvRedis, ttl: 1800000 })
-  : new Keyv({ namespace: CacheKeys.TOKEN_CONFIG, ttl: 1800000 });
+  ? new Keyv({ store: keyvRedis, ttl: THIRTY_MINUTES })
+  : new Keyv({ namespace: CacheKeys.TOKEN_CONFIG, ttl: THIRTY_MINUTES });
 
 const genTitle = isEnabled(USE_REDIS) // ttl: 2 minutes
   ? new Keyv({ store: keyvRedis, ttl: 120000 })
@@ -42,13 +48,20 @@ const abortKeys = isEnabled(USE_REDIS)
 const namespaces = {
   [CacheKeys.CONFIG_STORE]: config,
   pending_req,
-  ban: new Keyv({ store: keyvMongo, namespace: 'bans', ttl: duration }),
+  [ViolationTypes.BAN]: new Keyv({ store: keyvMongo, namespace: CacheKeys.BANS, ttl: duration }),
+  [CacheKeys.ENCODED_DOMAINS]: new Keyv({
+    store: keyvMongo,
+    namespace: CacheKeys.ENCODED_DOMAINS,
+    ttl: 0,
+  }),
   general: new Keyv({ store: logFile, namespace: 'violations' }),
   concurrent: createViolationInstance('concurrent'),
   non_browser: createViolationInstance('non_browser'),
   message_limit: createViolationInstance('message_limit'),
   token_balance: createViolationInstance(ViolationTypes.TOKEN_BALANCE),
   registrations: createViolationInstance('registrations'),
+  [ViolationTypes.TTS_LIMIT]: createViolationInstance(ViolationTypes.TTS_LIMIT),
+  [ViolationTypes.STT_LIMIT]: createViolationInstance(ViolationTypes.STT_LIMIT),
   [ViolationTypes.FILE_UPLOAD_LIMIT]: createViolationInstance(ViolationTypes.FILE_UPLOAD_LIMIT),
   [ViolationTypes.ILLEGAL_MODEL_REQUEST]: createViolationInstance(
     ViolationTypes.ILLEGAL_MODEL_REQUEST,
@@ -58,6 +71,7 @@ const namespaces = {
   [CacheKeys.TOKEN_CONFIG]: tokenConfig,
   [CacheKeys.GEN_TITLE]: genTitle,
   [CacheKeys.MODEL_QUERIES]: modelQueries,
+  [CacheKeys.AUDIO_RUNS]: audioRuns,
 };
 
 /**

api/config/parsers.js

@@ -27,26 +27,25 @@ function getMatchingSensitivePatterns(valueStr) {
 }
 
 /**
- * Redacts sensitive information from a console message.
- *
+ * Redacts sensitive information from a console message and trims it to a specified length if provided.
  * @param {string} str - The console message to be redacted.
- * @returns {string} - The redacted console message.
+ * @param {number} [trimLength] - The optional length at which to trim the redacted message.
+ * @returns {string} - The redacted and optionally trimmed console message.
  */
-function redactMessage(str) {
+function redactMessage(str, trimLength) {
   if (!str) {
     return '';
   }
 
   const patterns = getMatchingSensitivePatterns(str);
-
-  if (patterns.length === 0) {
-    return str;
-  }
-
   patterns.forEach((pattern) => {
     str = str.replace(pattern, '$1[REDACTED]');
   });
 
+  if (trimLength !== undefined && str.length > trimLength) {
+    return `${str.substring(0, trimLength)}...`;
+  }
+
   return str;
 }
 

api/config/paths.js

@@ -1,9 +1,13 @@
 const path = require('path');
 
 module.exports = {
+  root: path.resolve(__dirname, '..', '..'),
   uploads: path.resolve(__dirname, '..', '..', 'uploads'),
+  clientPath: path.resolve(__dirname, '..', '..', 'client'),
   dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),
   publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),
+  fonts: path.resolve(__dirname, '..', '..', 'client', 'public', 'fonts'),
+  assets: path.resolve(__dirname, '..', '..', 'client', 'public', 'assets'),
   imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),
   structuredTools: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'structured'),
   pluginManifest: path.resolve(__dirname, '..', 'app', 'clients', 'tools', 'manifest.json'),

api/config/winston.js

@@ -5,7 +5,15 @@ const { redactFormat, redactMessage, debugTraverse } = require('./parsers');
 
 const logDir = path.join(__dirname, '..', 'logs');
 
-const { NODE_ENV, DEBUG_LOGGING = true, DEBUG_CONSOLE = false } = process.env;
+const { NODE_ENV, DEBUG_LOGGING = true, DEBUG_CONSOLE = false, CONSOLE_JSON = false } = process.env;
+
+const useConsoleJson =
+  (typeof CONSOLE_JSON === 'string' && CONSOLE_JSON?.toLowerCase() === 'true') ||
+  CONSOLE_JSON === true;
+
+const useDebugConsole =
+  (typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE?.toLowerCase() === 'true') ||
+  DEBUG_CONSOLE === true;
 
 const levels = {
   error: 0,
@@ -33,7 +41,7 @@ const level = () => {
 
 const fileFormat = winston.format.combine(
   redactFormat(),
-  winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  winston.format.timestamp({ format: () => new Date().toISOString() }),
   winston.format.errors({ stack: true }),
   winston.format.splat(),
   // redactErrors(),
@@ -99,14 +107,20 @@ const consoleFormat = winston.format.combine(
   }),
 );
 
-if (
-  (typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE?.toLowerCase() === 'true') ||
-  DEBUG_CONSOLE === true
-) {
+if (useDebugConsole) {
   transports.push(
     new winston.transports.Console({
       level: 'debug',
-      format: winston.format.combine(fileFormat, debugTraverse),
+      format: useConsoleJson
+        ? winston.format.combine(fileFormat, debugTraverse, winston.format.json())
+        : winston.format.combine(fileFormat, debugTraverse),
+    }),
+  );
+} else if (useConsoleJson) {
+  transports.push(
+    new winston.transports.Console({
+      level: 'info',
+      format: winston.format.combine(fileFormat, winston.format.json()),
     }),
   );
 } else {

api/lib/db/indexSync.js

@@ -1,11 +1,28 @@
 const { MeiliSearch } = require('meilisearch');
-const Message = require('~/models/schema/messageSchema');
 const Conversation = require('~/models/schema/convoSchema');
+const Message = require('~/models/schema/messageSchema');
 const { logger } = require('~/config');
 
 const searchEnabled = process.env?.SEARCH?.toLowerCase() === 'true';
 let currentTimeout = null;
 
+class MeiliSearchClient {
+  static instance = null;
+
+  static getInstance() {
+    if (!MeiliSearchClient.instance) {
+      if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY) {
+        throw new Error('Meilisearch configuration is missing.');
+      }
+      MeiliSearchClient.instance = new MeiliSearch({
+        host: process.env.MEILI_HOST,
+        apiKey: process.env.MEILI_MASTER_KEY,
+      });
+    }
+    return MeiliSearchClient.instance;
+  }
+}
+
 // eslint-disable-next-line no-unused-vars
 async function indexSync(req, res, next) {
   if (!searchEnabled) {
@@ -13,20 +30,10 @@ async function indexSync(req, res, next) {
   }
 
   try {
-    if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY || !searchEnabled) {
-      throw new Error('Meilisearch not configured, search will be disabled.');
-    }
-
-    const client = new MeiliSearch({
-      host: process.env.MEILI_HOST,
-      apiKey: process.env.MEILI_MASTER_KEY,
-    });
+    const client = MeiliSearchClient.getInstance();
 
     const { status } = await client.health();
-    // logger.debug(`[indexSync] Meilisearch: ${status}`);
-    const result = status === 'available' && !!process.env.SEARCH;
-
-    if (!result) {
+    if (status !== 'available' || !process.env.SEARCH) {
       throw new Error('Meilisearch not available');
     }
 
@@ -37,12 +44,8 @@ async function indexSync(req, res, next) {
     const messagesIndexed = messages.numberOfDocuments;
     const convosIndexed = convos.numberOfDocuments;
 
-    logger.debug(
-      `[indexSync] There are ${messageCount} messages in the database, ${messagesIndexed} indexed`,
-    );
-    logger.debug(
-      `[indexSync] There are ${convoCount} convos in the database, ${convosIndexed} indexed`,
-    );
+    logger.debug(`[indexSync] There are ${messageCount} messages and ${messagesIndexed} indexed`);
+    logger.debug(`[indexSync] There are ${convoCount} convos and ${convosIndexed} indexed`);
 
     if (messageCount !== messagesIndexed) {
       logger.debug('[indexSync] Messages out of sync, indexing');
@@ -54,7 +57,6 @@ async function indexSync(req, res, next) {
       Conversation.syncWithMeili();
     }
   } catch (err) {
-    // logger.debug('[indexSync] in index sync');
     if (err.message.includes('not found')) {
       logger.debug('[indexSync] Creating indices...');
       currentTimeout = setTimeout(async () => {

api/models/Action.js

@@ -5,19 +5,18 @@ const Action = mongoose.model('action', actionSchema);
 
 /**
  * Update an action with new data without overwriting existing properties,
- * or create a new action if it doesn't exist.
+ * or create a new action if it doesn't exist, within a transaction session if provided.
  *
  * @param {Object} searchParams - The search parameters to find the action to update.
  * @param {string} searchParams.action_id - The ID of the action to update.
  * @param {string} searchParams.user - The user ID of the action's author.
  * @param {Object} updateData - An object containing the properties to update.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use.
  * @returns {Promise<Object>} The updated or newly created action document as a plain object.
  */
-const updateAction = async (searchParams, updateData) => {
-  return await Action.findOneAndUpdate(searchParams, updateData, {
-    new: true,
-    upsert: true,
-  }).lean();
+const updateAction = async (searchParams, updateData, session = null) => {
+  const options = { new: true, upsert: true, session };
+  return await Action.findOneAndUpdate(searchParams, updateData, options).lean();
 };
 
 /**
@@ -50,19 +49,37 @@ const getActions = async (searchParams, includeSensitive = false) => {
 };
 
 /**
- * Deletes an action by its ID.
+ * Deletes an action by params, within a transaction session if provided.
  *
- * @param {Object} searchParams - The search parameters to find the action to update.
- * @param {string} searchParams.action_id - The ID of the action to update.
+ * @param {Object} searchParams - The search parameters to find the action to delete.
+ * @param {string} searchParams.action_id - The ID of the action to delete.
  * @param {string} searchParams.user - The user ID of the action's author.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use (optional).
  * @returns {Promise<Object>} A promise that resolves to the deleted action document as a plain object, or null if no document was found.
  */
-const deleteAction = async (searchParams) => {
-  return await Action.findOneAndDelete(searchParams).lean();
+const deleteAction = async (searchParams, session = null) => {
+  const options = session ? { session } : {};
+  return await Action.findOneAndDelete(searchParams, options).lean();
+};
+
+/**
+ * Deletes actions by params, within a transaction session if provided.
+ *
+ * @param {Object} searchParams - The search parameters to find the actions to delete.
+ * @param {string} searchParams.action_id - The ID of the action(s) to delete.
+ * @param {string} searchParams.user - The user ID of the action's author.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use (optional).
+ * @returns {Promise<Number>} A promise that resolves to the number of deleted action documents.
+ */
+const deleteActions = async (searchParams, session = null) => {
+  const options = session ? { session } : {};
+  const result = await Action.deleteMany(searchParams, options);
+  return result.deletedCount;
 };
 
 module.exports = {
-  updateAction,
   getActions,
+  updateAction,
   deleteAction,
+  deleteActions,
 };

api/models/Assistant.js

@@ -5,19 +5,18 @@ const Assistant = mongoose.model('assistant', assistantSchema);
 
 /**
  * Update an assistant with new data without overwriting existing properties,
- * or create a new assistant if it doesn't exist.
+ * or create a new assistant if it doesn't exist, within a transaction session if provided.
  *
  * @param {Object} searchParams - The search parameters to find the assistant to update.
  * @param {string} searchParams.assistant_id - The ID of the assistant to update.
  * @param {string} searchParams.user - The user ID of the assistant's author.
  * @param {Object} updateData - An object containing the properties to update.
+ * @param {mongoose.ClientSession} [session] - The transaction session to use (optional).
  * @returns {Promise<Object>} The updated or newly created assistant document as a plain object.
  */
-const updateAssistant = async (searchParams, updateData) => {
-  return await Assistant.findOneAndUpdate(searchParams, updateData, {
-    new: true,
-    upsert: true,
-  }).lean();
+const updateAssistantDoc = async (searchParams, updateData, session = null) => {
+  const options = { new: true, upsert: true, session };
+  return await Assistant.findOneAndUpdate(searchParams, updateData, options).lean();
 };
 
 /**
@@ -40,8 +39,21 @@ const getAssistants = async (searchParams) => {
   return await Assistant.find(searchParams).lean();
 };
 
+/**
+ * Deletes an assistant based on the provided ID.
+ *
+ * @param {Object} searchParams - The search parameters to find the assistant to delete.
+ * @param {string} searchParams.assistant_id - The ID of the assistant to delete.
+ * @param {string} searchParams.user - The user ID of the assistant's author.
+ * @returns {Promise<void>} Resolves when the assistant has been successfully deleted.
+ */
+const deleteAssistant = async (searchParams) => {
+  return await Assistant.findOneAndDelete(searchParams);
+};
+
 module.exports = {
-  updateAssistant,
+  updateAssistantDoc,
+  deleteAssistant,
   getAssistants,
   getAssistant,
 };

api/models/Conversation.js

@@ -2,6 +2,12 @@ const Conversation = require('./schema/convoSchema');
 const { getMessages, deleteMessages } = require('./Message');
 const logger = require('~/config/winston');
 
+/**
+ * Retrieves a single conversation for a given user and conversation ID.
+ * @param {string} user - The user's ID.
+ * @param {string} conversationId - The conversation's ID.
+ * @returns {Promise<TConversation>} The conversation object.
+ */
 const getConvo = async (user, conversationId) => {
   try {
     return await Conversation.findOne({ user, conversationId }).lean();
@@ -15,7 +21,7 @@ module.exports = {
   Conversation,
   saveConvo: async (user, { conversationId, newConversationId, ...convo }) => {
     try {
-      const messages = await getMessages({ conversationId });
+      const messages = await getMessages({ conversationId }, '_id');
       const update = { ...convo, messages, user };
       if (newConversationId) {
         update.conversationId = newConversationId;
@@ -30,11 +36,35 @@ module.exports = {
       return { message: 'Error saving conversation' };
     }
   },
-  getConvosByPage: async (user, pageNumber = 1, pageSize = 25) => {
+  bulkSaveConvos: async (conversations) => {
     try {
-      const totalConvos = (await Conversation.countDocuments({ user })) || 1;
+      const bulkOps = conversations.map((convo) => ({
+        updateOne: {
+          filter: { conversationId: convo.conversationId, user: convo.user },
+          update: convo,
+          upsert: true,
+          timestamps: false,
+        },
+      }));
+
+      const result = await Conversation.bulkWrite(bulkOps);
+      return result;
+    } catch (error) {
+      logger.error('[saveBulkConversations] Error saving conversations in bulk', error);
+      throw new Error('Failed to save conversations in bulk.');
+    }
+  },
+  getConvosByPage: async (user, pageNumber = 1, pageSize = 25, isArchived = false) => {
+    const query = { user };
+    if (isArchived) {
+      query.isArchived = true;
+    } else {
+      query.$or = [{ isArchived: false }, { isArchived: { $exists: false } }];
+    }
+    try {
+      const totalConvos = (await Conversation.countDocuments(query)) || 1;
       const totalPages = Math.ceil(totalConvos / pageSize);
-      const convos = await Conversation.find({ user })
+      const convos = await Conversation.find(query)
         .sort({ updatedAt: -1 })
         .skip((pageNumber - 1) * pageSize)
         .limit(pageSize)

api/models/Message.js

@@ -10,6 +10,7 @@ module.exports = {
   async saveMessage({
     user,
     endpoint,
+    iconURL,
     messageId,
     newMessageId,
     conversationId,
@@ -35,6 +36,7 @@ module.exports = {
 
       const update = {
         user,
+        iconURL,
         endpoint,
         messageId: newMessageId || messageId,
         conversationId,
@@ -72,6 +74,25 @@ module.exports = {
       throw new Error('Failed to save message.');
     }
   },
+
+  async bulkSaveMessages(messages) {
+    try {
+      const bulkOps = messages.map((message) => ({
+        updateOne: {
+          filter: { messageId: message.messageId },
+          update: message,
+          upsert: true,
+        },
+      }));
+
+      const result = await Message.bulkWrite(bulkOps);
+      return result;
+    } catch (err) {
+      logger.error('Error saving messages in bulk:', err);
+      throw new Error('Failed to save messages in bulk.');
+    }
+  },
+
   /**
    * Records a message in the database.
    *
@@ -108,6 +129,14 @@ module.exports = {
       throw new Error('Failed to save message.');
     }
   },
+  async updateMessageText({ messageId, text }) {
+    try {
+      await Message.updateOne({ messageId }, { text });
+    } catch (err) {
+      logger.error('Error updating message text:', err);
+      throw new Error('Failed to update message text.');
+    }
+  },
   async updateMessage(message) {
     try {
       const { messageId, ...update } = message;
@@ -150,8 +179,18 @@ module.exports = {
     }
   },
 
-  async getMessages(filter) {
+  /**
+   * Retrieves messages from the database.
+   * @param {Record<string, unknown>} filter
+   * @param {string | undefined} [select]
+   * @returns
+   */
+  async getMessages(filter, select) {
     try {
+      if (select) {
+        return await Message.find(filter).select(select).sort({ createdAt: 1 }).lean();
+      }
+
       return await Message.find(filter).sort({ createdAt: 1 }).lean();
     } catch (err) {
       logger.error('Error getting messages:', err);

api/models/Preset.js

@@ -39,6 +39,12 @@ module.exports = {
     try {
       const setter = { $set: {} };
       const update = { presetId, ...preset };
+      if (preset.tools && Array.isArray(preset.tools)) {
+        update.tools =
+          preset.tools
+            .map((tool) => tool?.pluginKey ?? tool)
+            .filter((toolName) => typeof toolName === 'string') ?? [];
+      }
       if (newPresetId) {
         update.presetId = newPresetId;
       }

api/models/Share.js

@@ -0,0 +1,89 @@
+const crypto = require('crypto');
+const { getMessages } = require('./Message');
+const SharedLink = require('./schema/shareSchema');
+const logger = require('~/config/winston');
+
+module.exports = {
+  SharedLink,
+  getSharedMessages: async (shareId) => {
+    try {
+      const share = await SharedLink.findOne({ shareId })
+        .populate({
+          path: 'messages',
+          select: '-_id -__v -user',
+        })
+        .select('-_id -__v -user')
+        .lean();
+
+      if (!share || !share.conversationId || !share.isPublic) {
+        return null;
+      }
+
+      return share;
+    } catch (error) {
+      logger.error('[getShare] Error getting share link', error);
+      return { message: 'Error getting share link' };
+    }
+  },
+
+  getSharedLinks: async (user, pageNumber = 1, pageSize = 25, isPublic = true) => {
+    const query = { user, isPublic };
+    try {
+      const totalConvos = (await SharedLink.countDocuments(query)) || 1;
+      const totalPages = Math.ceil(totalConvos / pageSize);
+      const shares = await SharedLink.find(query)
+        .sort({ updatedAt: -1 })
+        .skip((pageNumber - 1) * pageSize)
+        .limit(pageSize)
+        .select('-_id -__v -user')
+        .lean();
+
+      return { sharedLinks: shares, pages: totalPages, pageNumber, pageSize };
+    } catch (error) {
+      logger.error('[getShareByPage] Error getting shares', error);
+      return { message: 'Error getting shares' };
+    }
+  },
+
+  createSharedLink: async (user, { conversationId, ...shareData }) => {
+    const share = await SharedLink.findOne({ conversationId }).select('-_id -__v -user').lean();
+    if (share) {
+      return share;
+    }
+
+    try {
+      const shareId = crypto.randomUUID();
+      const messages = await getMessages({ conversationId });
+      const update = { ...shareData, shareId, messages, user };
+      return await SharedLink.findOneAndUpdate({ conversationId: conversationId, user }, update, {
+        new: true,
+        upsert: true,
+      });
+    } catch (error) {
+      logger.error('[saveShareMessage] Error saving conversation', error);
+      return { message: 'Error saving conversation' };
+    }
+  },
+
+  updateSharedLink: async (user, { conversationId, ...shareData }) => {
+    const share = await SharedLink.findOne({ conversationId }).select('-_id -__v -user').lean();
+    if (!share) {
+      return { message: 'Share not found' };
+    }
+    // update messages to the latest
+    const messages = await getMessages({ conversationId });
+    const update = { ...shareData, messages, user };
+    return await SharedLink.findOneAndUpdate({ conversationId: conversationId, user }, update, {
+      new: true,
+      upsert: false,
+    });
+  },
+
+  deleteSharedLink: async (user, { shareId }) => {
+    const share = await SharedLink.findOne({ shareId, user });
+    if (!share) {
+      return { message: 'Share not found' };
+    }
+    return await SharedLink.findOneAndDelete({ shareId, user });
+  },
+};

api/models/Transaction.js

@@ -2,6 +2,7 @@ const mongoose = require('mongoose');
 const { isEnabled } = require('../server/utils/handleText');
 const transactionSchema = require('./schema/transaction');
 const { getMultiplier } = require('./tx');
+const { logger } = require('~/config');
 const Balance = require('./Balance');
 const cancelRate = 1.15;
 
@@ -11,7 +12,7 @@ transactionSchema.methods.calculateTokenValue = function () {
     this.tokenValue = this.rawAmount;
   }
   const { valueKey, tokenType, model, endpointTokenConfig } = this;
-  const multiplier = getMultiplier({ valueKey, tokenType, model, endpointTokenConfig });
+  const multiplier = Math.abs(getMultiplier({ valueKey, tokenType, model, endpointTokenConfig }));
   this.rate = multiplier;
   this.tokenValue = this.rawAmount * multiplier;
   if (this.context && this.tokenType === 'completion' && this.context === 'incomplete') {
@@ -35,18 +36,24 @@ transactionSchema.statics.create = async function (transactionData) {
     return;
   }
 
-  // Adjust the user's balance
-  const updatedBalance = await Balance.findOneAndUpdate(
+  let balance = await Balance.findOne({ user: transaction.user }).lean();
+  let incrementValue = transaction.tokenValue;
+
+  if (balance && balance?.tokenCredits + incrementValue < 0) {
+    incrementValue = -balance.tokenCredits;
+  }
+
+  balance = await Balance.findOneAndUpdate(
     { user: transaction.user },
-    { $inc: { tokenCredits: transaction.tokenValue } },
+    { $inc: { tokenCredits: incrementValue } },
     { upsert: true, new: true },
   ).lean();
 
   return {
     rate: transaction.rate,
     user: transaction.user.toString(),
-    balance: updatedBalance.tokenCredits,
-    [transaction.tokenType]: transaction.tokenValue,
+    balance: balance.tokenCredits,
+    [transaction.tokenType]: incrementValue,
   };
 };
 
@@ -64,7 +71,7 @@ async function getTransactions(filter) {
   try {
     return await Transaction.find(filter).lean();
   } catch (error) {
-    console.error('Error querying transactions:', error);
+    logger.error('Error querying transactions:', error);
     throw error;
   }
 }

api/models/plugins/mongoMeili.js

@@ -155,7 +155,7 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
             function (results, value, key) {
               return { ...results, [key]: 1 };
             },
-            { _id: 1 },
+            { _id: 1, __v: 1 },
           ),
         ).lean();
 
@@ -348,7 +348,7 @@ module.exports = function mongoMeili(schema, options) {
       try {
         meiliDoc = await client.index('convos').getDocument(doc.conversationId);
       } catch (error) {
-        logger.error(
+        logger.debug(
           '[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' +
             doc.conversationId,
           error,

api/models/schema/defaults.js

@@ -88,6 +88,28 @@ const conversationPreset = {
   instructions: {
     type: String,
   },
+  stop: { type: [{ type: String }], default: undefined },
+  isArchived: {
+    type: Boolean,
+    default: false,
+  },
+  /* UI Components */
+  iconURL: {
+    type: String,
+  },
+  greeting: {
+    type: String,
+  },
+  spec: {
+    type: String,
+  },
+  tools: { type: [{ type: String }], default: undefined },
+  maxContextTokens: {
+    type: Number,
+  },
+  max_tokens: {
+    type: Number,
+  },
 };
 
 const agentOptions = {

api/models/schema/fileSchema.js

@@ -15,7 +15,9 @@ const mongoose = require('mongoose');
  * @property {'file'} object - Type of object, always 'file'
  * @property {string} type - Type of file
  * @property {number} usage - Number of uses of the file
+ * @property {string} [context] - Context of the file origin
  * @property {boolean} [embedded] - Whether or not the file is embedded in vector db
+ * @property {string} [model] - The model to identify the group region of the file (for Azure OpenAI hosting)
  * @property {string} [source] - The source of the file
  * @property {number} [width] - Optional width of the file
  * @property {number} [height] - Optional height of the file
@@ -82,6 +84,9 @@ const fileSchema = mongoose.Schema(
       type: String,
       default: FileSources.local,
     },
+    model: {
+      type: String,
+    },
     width: Number,
     height: Number,
     expiresAt: {
@@ -94,4 +99,6 @@ const fileSchema = mongoose.Schema(
   },
 );
 
+fileSchema.index({ createdAt: 1, updatedAt: 1 });
+
 module.exports = fileSchema;

api/models/schema/messageSchema.js

@@ -110,6 +110,10 @@ const messageSchema = mongoose.Schema(
     thread_id: {
       type: String,
     },
+    /* frontend components */
+    iconURL: {
+      type: String,
+    },
   },
   { timestamps: true },
 );

api/models/schema/shareSchema.js

@@ -0,0 +1,38 @@
+const mongoose = require('mongoose');
+
+const shareSchema = mongoose.Schema(
+  {
+    conversationId: {
+      type: String,
+      required: true,
+    },
+    title: {
+      type: String,
+      index: true,
+    },
+    user: {
+      type: String,
+      index: true,
+    },
+    messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }],
+    shareId: {
+      type: String,
+      index: true,
+    },
+    isPublic: {
+      type: Boolean,
+      default: false,
+    },
+    isVisible: {
+      type: Boolean,
+      default: false,
+    },
+    isAnonymous: {
+      type: Boolean,
+      default: true,
+    },
+  },
+  { timestamps: true },
+);
+
+module.exports = mongoose.model('SharedLink', shareSchema);

api/models/schema/userSchema.js

@@ -64,6 +64,11 @@ const userSchema = mongoose.Schema(
       unique: true,
       sparse: true,
     },
+    ldapId: {
+      type: String,
+      unique: true,
+      sparse: true,
+    },
     githubId: {
       type: String,
       unique: true,

api/models/spendTokens.js

@@ -40,7 +40,7 @@ const spendTokens = async (txData, tokenUsage) => {
       });
     }
 
-    if (!completionTokens) {
+    if (!completionTokens && isNaN(completionTokens)) {
       logger.debug('[spendTokens] !completionTokens', { prompt, completion });
       return;
     }
@@ -54,7 +54,7 @@ const spendTokens = async (txData, tokenUsage) => {
     prompt &&
       completion &&
       logger.debug('[spendTokens] Transaction data record against balance:', {
-        user: prompt.user,
+        user: txData.user,
         prompt: prompt.prompt,
         promptRate: prompt.rate,
         completion: completion.completion,

api/models/tx.js

@@ -3,6 +3,7 @@ const defaultRate = 6;
 
 /**
  * Mapping of model token sizes to their respective multipliers for prompt and completion.
+ * The rates are 1 USD per 1M tokens.
  * @type {Object.<string, {prompt: number, completion: number}>}
  */
 const tokenValues = {
@@ -11,6 +12,7 @@ const tokenValues = {
   '4k': { prompt: 1.5, completion: 2 },
   '16k': { prompt: 3, completion: 4 },
   'gpt-3.5-turbo-1106': { prompt: 1, completion: 2 },
+  'gpt-4o': { prompt: 5, completion: 15 },
   'gpt-4-1106': { prompt: 10, completion: 30 },
   'gpt-3.5-turbo-0125': { prompt: 0.5, completion: 1.5 },
   'claude-3-opus': { prompt: 15, completion: 75 },
@@ -19,6 +21,15 @@ const tokenValues = {
   'claude-2.1': { prompt: 8, completion: 24 },
   'claude-2': { prompt: 8, completion: 24 },
   'claude-': { prompt: 0.8, completion: 2.4 },
+  'command-r-plus': { prompt: 3, completion: 15 },
+  'command-r': { prompt: 0.5, completion: 1.5 },
+  /* cohere doesn't have rates for the older command models,
+  so this was from https://artificialanalysis.ai/models/command-light/providers */
+  command: { prompt: 0.38, completion: 0.38 },
+  // 'gemini-1.5': { prompt: 7, completion: 21 }, // May 2nd, 2024 pricing
+  // 'gemini': { prompt: 0.5, completion: 1.5 }, // May 2nd, 2024 pricing
+  'gemini-1.5': { prompt: 0, completion: 0 }, // currently free
+  gemini: { prompt: 0, completion: 0 }, // currently free
 };
 
 /**
@@ -42,6 +53,10 @@ const getValueKey = (model, endpoint) => {
     return 'gpt-3.5-turbo-1106';
   } else if (modelName.includes('gpt-3.5')) {
     return '4k';
+  } else if (modelName.includes('gpt-4o')) {
+    return 'gpt-4o';
+  } else if (modelName.includes('gpt-4-vision')) {
+    return 'gpt-4-1106';
   } else if (modelName.includes('gpt-4-1106')) {
     return 'gpt-4-1106';
   } else if (modelName.includes('gpt-4-0125')) {

api/models/tx.spec.js

@@ -34,6 +34,20 @@ describe('getValueKey', () => {
     expect(getValueKey('openai/gpt-4-1106')).toBe('gpt-4-1106');
     expect(getValueKey('gpt-4-1106/openai/')).toBe('gpt-4-1106');
   });
+
+  it('should return "gpt-4-1106" for model type of "gpt-4-1106"', () => {
+    expect(getValueKey('gpt-4-vision-preview')).toBe('gpt-4-1106');
+    expect(getValueKey('openai/gpt-4-1106')).toBe('gpt-4-1106');
+    expect(getValueKey('gpt-4-turbo')).toBe('gpt-4-1106');
+    expect(getValueKey('gpt-4-0125')).toBe('gpt-4-1106');
+  });
+
+  it('should return "gpt-4o" for model type of "gpt-4o"', () => {
+    expect(getValueKey('gpt-4o-2024-05-13')).toBe('gpt-4o');
+    expect(getValueKey('openai/gpt-4o')).toBe('gpt-4o');
+    expect(getValueKey('gpt-4o-turbo')).toBe('gpt-4o');
+    expect(getValueKey('gpt-4o-0125')).toBe('gpt-4o');
+  });
 });
 
 describe('getMultiplier', () => {
@@ -77,6 +91,17 @@ describe('getMultiplier', () => {
     );
   });
 
+  it('should return the correct multiplier for gpt-4o', () => {
+    const valueKey = getValueKey('gpt-4o-2024-05-13');
+    expect(getMultiplier({ valueKey, tokenType: 'prompt' })).toBe(tokenValues['gpt-4o'].prompt);
+    expect(getMultiplier({ valueKey, tokenType: 'completion' })).toBe(
+      tokenValues['gpt-4o'].completion,
+    );
+    expect(getMultiplier({ valueKey, tokenType: 'completion' })).not.toBe(
+      tokenValues['gpt-4-1106'].completion,
+    );
+  });
+
   it('should derive the valueKey from the model if not provided for new models', () => {
     expect(
       getMultiplier({ tokenType: 'prompt', model: 'gpt-3.5-turbo-1106-some-other-info' }),

api/package.json

@@ -1,13 +1,19 @@
 {
   "name": "@librechat/backend",
-  "version": "0.6.10",
+  "version": "0.7.2",
   "description": "",
   "scripts": {
     "start": "echo 'please run this from the root directory'",
     "server-dev": "echo 'please run this from the root directory'",
     "test": "cross-env NODE_ENV=test jest",
     "b:test": "NODE_ENV=test bun jest",
-    "test:ci": "jest --ci"
+    "test:ci": "jest --ci",
+    "add-balance": "node ./add-balance.js",
+    "list-balances": "node ./list-balances.js",
+    "user-stats": "node ./user-stats.js",
+    "create-user": "node ./create-user.js",
+    "ban-user": "node ./ban-user.js",
+    "delete-user": "node ./delete-user.js"
   },
   "repository": {
     "type": "git",
@@ -29,14 +35,18 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.16.1",
     "@azure/search-documents": "^12.0.0",
+    "@google/generative-ai": "^0.5.0",
     "@keyv/mongo": "^2.1.8",
     "@keyv/redis": "^2.8.1",
-    "@langchain/community": "^0.0.17",
-    "@langchain/google-genai": "^0.0.8",
+    "@langchain/community": "^0.0.46",
+    "@langchain/google-genai": "^0.0.11",
+    "@langchain/google-vertexai": "^0.0.17",
+    "agenda": "^5.0.0",
     "axios": "^1.3.4",
     "bcryptjs": "^2.4.3",
+    "bottleneck": "^2.19.5",
     "cheerio": "^1.0.0-rc.12",
-    "cohere-ai": "^6.0.0",
+    "cohere-ai": "^7.9.1",
     "connect-redis": "^7.1.0",
     "cookie": "^0.5.0",
     "cors": "^2.8.5",
@@ -46,7 +56,7 @@
     "express-rate-limit": "^6.9.0",
     "express-session": "^1.17.3",
     "file-type": "^18.7.0",
-    "firebase": "^10.8.0",
+    "firebase": "^10.6.0",
     "googleapis": "^126.0.1",
     "handlebars": "^4.7.7",
     "html": "^1.0.0",
@@ -59,14 +69,15 @@
     "langchain": "^0.0.214",
     "librechat-data-provider": "*",
     "lodash": "^4.17.21",
-    "meilisearch": "^0.37.0",
+    "meilisearch": "^0.38.0",
     "mime": "^3.0.0",
     "module-alias": "^2.2.3",
     "mongoose": "^7.1.1",
     "multer": "^1.4.5-lts.1",
     "nodejs-gpt": "^1.37.4",
     "nodemailer": "^6.9.4",
-    "openai": "^4.29.0",
+    "ollama": "^0.5.0",
+    "openai": "^4.47.1",
     "openai-chat-tokens": "^0.2.8",
     "openid-client": "^5.4.2",
     "passport": "^0.6.0",
@@ -76,14 +87,16 @@
     "passport-github2": "^0.1.12",
     "passport-google-oauth20": "^2.0.0",
     "passport-jwt": "^4.0.1",
+    "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "pino": "^8.12.1",
     "sharp": "^0.32.6",
-    "tiktoken": "^1.0.10",
+    "tiktoken": "^1.0.15",
     "traverse": "^0.6.7",
     "ua-parser-js": "^1.0.36",
     "winston": "^3.11.0",
     "winston-daily-rotate-file": "^4.7.1",
+    "ws": "^8.17.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {

api/server/controllers/AskController.js

@@ -1,5 +1,5 @@
 const throttle = require('lodash/throttle');
-const { getResponseSender, Constants } = require('librechat-data-provider');
+const { getResponseSender, Constants, EModelEndpoint } = require('librechat-data-provider');
 const { createAbortController, handleAbortError } = require('~/server/middleware');
 const { sendMessage, createOnProgress } = require('~/server/utils');
 const { saveMessage, getConvo } = require('~/models');
@@ -48,7 +48,7 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
 
   try {
     const { client } = await initializeClient({ req, res, endpointOption });
-
+    const unfinished = endpointOption.endpoint === EModelEndpoint.google ? false : true;
     const { onProgress: progressCallback, getPartialText } = createOnProgress({
       onProgress: throttle(
         ({ text: partialText }) => {
@@ -59,7 +59,7 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
             parentMessageId: overrideParentMessageId ?? userMessageId,
             text: partialText,
             model: client.modelOptions.model,
-            unfinished: true,
+            unfinished,
             error: false,
             user,
           });
@@ -105,11 +105,12 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
       getReqData,
       onStart,
       abortController,
-      onProgress: progressCallback.call(null, {
+      progressCallback,
+      progressOptions: {
         res,
         text,
-        parentMessageId: overrideParentMessageId || userMessageId,
-      }),
+        // parentMessageId: overrideParentMessageId || userMessageId,
+      },
     };
 
     let response = await client.sendMessage(text, messageOptions);

api/server/controllers/AuthController.js

@@ -76,14 +76,14 @@ const refreshController = async (req, res) => {
   }
 
   try {
-    let payload;
-    payload = jwt.verify(refreshToken, process.env.JWT_REFRESH_SECRET);
-    const userId = payload.id;
-    const user = await User.findOne({ _id: userId });
+    const payload = jwt.verify(refreshToken, process.env.JWT_REFRESH_SECRET);
+    const user = await User.findOne({ _id: payload.id });
     if (!user) {
       return res.status(401).redirect('/login');
     }
 
+    const userId = payload.id;
+
     if (process.env.NODE_ENV === 'CI') {
       const token = await setAuthTokens(userId, res);
       const userObj = user.toJSON();
@@ -118,6 +118,6 @@ module.exports = {
   getUserController,
   refreshController,
   registrationController,
-  resetPasswordRequestController,
   resetPasswordController,
+  resetPasswordRequestController,
 };

api/server/controllers/EditController.js

@@ -1,5 +1,5 @@
 const throttle = require('lodash/throttle');
-const { getResponseSender } = require('librechat-data-provider');
+const { getResponseSender, EModelEndpoint } = require('librechat-data-provider');
 const { createAbortController, handleAbortError } = require('~/server/middleware');
 const { sendMessage, createOnProgress } = require('~/server/utils');
 const { saveMessage, getConvo } = require('~/models');
@@ -48,6 +48,7 @@ const EditController = async (req, res, next, initializeClient) => {
     }
   };
 
+  const unfinished = endpointOption.endpoint === EModelEndpoint.google ? false : true;
   const { onProgress: progressCallback, getPartialText } = createOnProgress({
     generation,
     onProgress: throttle(
@@ -59,7 +60,7 @@ const EditController = async (req, res, next, initializeClient) => {
           parentMessageId: overrideParentMessageId ?? userMessageId,
           text: partialText,
           model: endpointOption.modelOptions.model,
-          unfinished: true,
+          unfinished,
           isEdited: true,
           error: false,
           user,
@@ -111,11 +112,12 @@ const EditController = async (req, res, next, initializeClient) => {
       getReqData,
       onStart,
       abortController,
-      onProgress: progressCallback.call(null, {
+      progressCallback,
+      progressOptions: {
         res,
         text,
-        parentMessageId: overrideParentMessageId || userMessageId,
-      }),
+        // parentMessageId: overrideParentMessageId || userMessageId,
+      },
     });
 
     const conversation = await getConvo(user, conversationId);

api/server/controllers/EndpointController.js

@@ -16,10 +16,28 @@ async function endpointController(req, res) {
   /** @type {TEndpointsConfig} */
   const mergedConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints };
   if (mergedConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) {
-    const { disableBuilder, retrievalModels, capabilities, ..._rest } =
+    const { disableBuilder, retrievalModels, capabilities, version, ..._rest } =
       req.app.locals[EModelEndpoint.assistants];
+
     mergedConfig[EModelEndpoint.assistants] = {
       ...mergedConfig[EModelEndpoint.assistants],
+      version,
+      retrievalModels,
+      disableBuilder,
+      capabilities,
+    };
+  }
+
+  if (
+    mergedConfig[EModelEndpoint.azureAssistants] &&
+    req.app.locals?.[EModelEndpoint.azureAssistants]
+  ) {
+    const { disableBuilder, retrievalModels, capabilities, version, ..._rest } =
+      req.app.locals[EModelEndpoint.azureAssistants];
+
+    mergedConfig[EModelEndpoint.azureAssistants] = {
+      ...mergedConfig[EModelEndpoint.azureAssistants],
+      version,
       retrievalModels,
       disableBuilder,
       capabilities,

api/server/controllers/PluginController.js

@@ -55,19 +55,27 @@ const getAvailablePluginsController = async (req, res) => {
       return;
     }
 
+    /** @type {{ filteredTools: string[], includedTools: string[] }} */
+    const { filteredTools = [], includedTools = [] } = req.app.locals;
     const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
-
     const jsonData = JSON.parse(pluginManifest);
-    /** @type {TPlugin[]} */
+
     const uniquePlugins = filterUniquePlugins(jsonData);
-    const authenticatedPlugins = uniquePlugins.map((plugin) => {
-      if (isPluginAuthenticated(plugin)) {
-        return { ...plugin, authenticated: true };
-      } else {
-        return plugin;
-      }
-    });
-    const plugins = await addOpenAPISpecs(authenticatedPlugins);
+    let authenticatedPlugins = [];
+    for (const plugin of uniquePlugins) {
+      authenticatedPlugins.push(
+        isPluginAuthenticated(plugin) ? { ...plugin, authenticated: true } : plugin,
+      );
+    }
+
+    let plugins = await addOpenAPISpecs(authenticatedPlugins);
+
+    if (includedTools.length > 0) {
+      plugins = plugins.filter((plugin) => includedTools.includes(plugin.pluginKey));
+    } else {
+      plugins = plugins.filter((plugin) => !filteredTools.includes(plugin.pluginKey));
+    }
+
     await cache.set(CacheKeys.PLUGINS, plugins);
     res.status(200).json(plugins);
   } catch (error) {

api/server/controllers/assistants/chatV1.js

@@ -1,14 +1,13 @@
 const { v4 } = require('uuid');
-const express = require('express');
 const {
   Constants,
   RunStatus,
   CacheKeys,
-  FileSources,
   ContentTypes,
   EModelEndpoint,
   ViolationTypes,
   ImageVisionTool,
+  checkOpenAIStorage,
   AssistantStreamEvents,
 } = require('librechat-data-provider');
 const {
@@ -21,44 +20,36 @@ const {
 } = require('~/server/services/Threads');
 const { sendResponse, sendMessage, sleep, isEnabled, countTokens } = require('~/server/utils');
 const { runAssistant, createOnTextProgress } = require('~/server/services/AssistantService');
-const { addTitle, initializeClient } = require('~/server/services/Endpoints/assistants');
+const validateAuthor = require('~/server/middleware/assistants/validateAuthor');
 const { formatMessage, createVisionPrompt } = require('~/app/clients/prompts');
 const { createRun, StreamRunManager } = require('~/server/services/Runs');
+const { addTitle } = require('~/server/services/Endpoints/assistants');
 const { getTransactions } = require('~/models/Transaction');
 const checkBalance = require('~/models/checkBalance');
 const { getConvo } = require('~/models/Conversation');
 const getLogStores = require('~/cache/getLogStores');
 const { getModelMaxTokens } = require('~/utils');
+const { getOpenAIClient } = require('./helpers');
 const { logger } = require('~/config');
 
-const router = express.Router();
-const {
-  setHeaders,
-  handleAbort,
-  validateModel,
-  handleAbortError,
-  // validateEndpoint,
-  buildEndpointOption,
-} = require('~/server/middleware');
-
-router.post('/abort', handleAbort());
-
 const ten_minutes = 1000 * 60 * 10;
 
 /**
  * @route POST /
  * @desc Chat with an assistant
  * @access Public
- * @param {express.Request} req - The request object, containing the request data.
- * @param {express.Response} res - The response object, used to send back a response.
+ * @param {object} req - The request object, containing the request data.
+ * @param {object} req.body - The request payload.
+ * @param {Express.Response} res - The response object, used to send back a response.
  * @returns {void}
  */
-router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res) => {
+const chatV1 = async (req, res) => {
   logger.debug('[/assistants/chat/] req.body', req.body);
 
   const {
     text,
     model,
+    endpoint,
     files = [],
     promptPrefix,
     assistant_id,
@@ -69,30 +60,6 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
     parentMessageId: _parentId = Constants.NO_PARENT,
   } = req.body;
 
-  /** @type {Partial<TAssistantEndpoint>} */
-  const assistantsConfig = req.app.locals?.[EModelEndpoint.assistants];
-
-  if (assistantsConfig) {
-    const { supportedIds, excludedIds } = assistantsConfig;
-    const error = { message: 'Assistant not supported' };
-    if (supportedIds?.length && !supportedIds.includes(assistant_id)) {
-      return await handleAbortError(res, req, error, {
-        sender: 'System',
-        conversationId: convoId,
-        messageId: v4(),
-        parentMessageId: _messageId,
-        error,
-      });
-    } else if (excludedIds?.length && excludedIds.includes(assistant_id)) {
-      return await handleAbortError(res, req, error, {
-        sender: 'System',
-        conversationId: convoId,
-        messageId: v4(),
-        parentMessageId: _messageId,
-      });
-    }
-  }
-
   /** @type {OpenAIClient} */
   let openai;
   /** @type {string|undefined} - the current thread id */
@@ -138,7 +105,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
       user: req.user.id,
       shouldSaveMessage: false,
       messageId: responseMessageId,
-      endpoint: EModelEndpoint.assistants,
+      endpoint,
     };
 
     if (error.message === 'Run cancelled') {
@@ -149,7 +116,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
       logger.debug('[/assistants/chat/] Request aborted on close');
     } else if (/Files.*are invalid/.test(error.message)) {
       const errorMessage = `Files are invalid, or may not have uploaded yet.${
-        req.app.locals?.[EModelEndpoint.azureOpenAI].assistants
+        endpoint === EModelEndpoint.azureAssistants
           ? ' If using Azure OpenAI, files are only available in the region of the assistant\'s model at the time of upload.'
           : ''
       }`;
@@ -205,6 +172,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
       const runMessages = await checkMessageGaps({
         openai,
         run_id,
+        endpoint,
         thread_id,
         conversationId,
         latestMessageId: responseMessageId,
@@ -247,7 +215,6 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
       }
 
       finalEvent = {
-        title: 'New Chat',
         final: true,
         conversation: await getConvo(req.user.id, conversationId),
         runMessages,
@@ -312,8 +279,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
       });
     };
 
-    /** @type {{ openai: OpenAIClient }} */
-    const { openai: _openai, client } = await initializeClient({
+    const { openai: _openai, client } = await getOpenAIClient({
       req,
       res,
       endpointOption: req.body.endpointOption,
@@ -321,6 +287,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
     });
 
     openai = _openai;
+    await validateAuthor({ req, openai });
 
     if (previousMessages.length) {
       parentMessageId = previousMessages[previousMessages.length - 1].messageId;
@@ -371,10 +338,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
 
       /** @type {MongoFile[]} */
       const attachments = await req.body.endpointOption.attachments;
-      if (
-        attachments &&
-        attachments.every((attachment) => attachment.source === FileSources.openai)
-      ) {
+      if (attachments && attachments.every((attachment) => checkOpenAIStorage(attachment.source))) {
         return;
       }
 
@@ -432,7 +396,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
 
       if (processedFiles) {
         for (const file of processedFiles) {
-          if (file.source !== FileSources.openai) {
+          if (!checkOpenAIStorage(file.source)) {
             attachedFileIds.delete(file.file_id);
             const index = file_ids.indexOf(file.file_id);
             if (index > -1) {
@@ -468,6 +432,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
         assistant_id,
         thread_id,
         model: assistant_id,
+        endpoint,
       };
 
       previousMessages.push(requestMessage);
@@ -477,8 +442,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
 
       conversation = {
         conversationId,
-        title: 'New Chat',
-        endpoint: EModelEndpoint.assistants,
+        endpoint,
         promptPrefix: promptPrefix,
         instructions: instructions,
         assistant_id,
@@ -515,7 +479,8 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
     let response;
 
     const processRun = async (retry = false) => {
-      if (req.app.locals[EModelEndpoint.azureOpenAI]?.assistants) {
+      if (endpoint === EModelEndpoint.azureAssistants) {
+        body.model = openai._options.model;
         openai.attachedFileIds = attachedFileIds;
         openai.visionPromise = visionPromise;
         if (retry) {
@@ -597,17 +562,17 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
 
     /** @type {ResponseMessage} */
     const responseMessage = {
-      ...response.finalMessage,
+      ...(response.responseMessage ?? response.finalMessage),
       parentMessageId: userMessageId,
       conversationId,
       user: req.user.id,
       assistant_id,
       thread_id,
       model: assistant_id,
+      endpoint,
     };
 
     sendMessage(res, {
-      title: 'New Chat',
       final: true,
       conversation,
       requestMessage: {
@@ -657,6 +622,6 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
   } catch (error) {
     await handleError(error);
   }
-});
+};
 
-module.exports = router;
+module.exports = chatV1;

api/server/controllers/assistants/chatV2.js

@@ -0,0 +1,597 @@
+const { v4 } = require('uuid');
+const {
+  Constants,
+  RunStatus,
+  CacheKeys,
+  ContentTypes,
+  ToolCallTypes,
+  EModelEndpoint,
+  ViolationTypes,
+  retrievalMimeTypes,
+  AssistantStreamEvents,
+} = require('librechat-data-provider');
+const {
+  initThread,
+  recordUsage,
+  saveUserMessage,
+  checkMessageGaps,
+  addThreadMetadata,
+  saveAssistantMessage,
+} = require('~/server/services/Threads');
+const { sendResponse, sendMessage, sleep, isEnabled, countTokens } = require('~/server/utils');
+const { runAssistant, createOnTextProgress } = require('~/server/services/AssistantService');
+const validateAuthor = require('~/server/middleware/assistants/validateAuthor');
+const { createRun, StreamRunManager } = require('~/server/services/Runs');
+const { addTitle } = require('~/server/services/Endpoints/assistants');
+const { getTransactions } = require('~/models/Transaction');
+const checkBalance = require('~/models/checkBalance');
+const { getConvo } = require('~/models/Conversation');
+const getLogStores = require('~/cache/getLogStores');
+const { getModelMaxTokens } = require('~/utils');
+const { getOpenAIClient } = require('./helpers');
+const { logger } = require('~/config');
+
+const ten_minutes = 1000 * 60 * 10;
+
+/**
+ * @route POST /
+ * @desc Chat with an assistant
+ * @access Public
+ * @param {Express.Request} req - The request object, containing the request data.
+ * @param {Express.Response} res - The response object, used to send back a response.
+ * @returns {void}
+ */
+const chatV2 = async (req, res) => {
+  logger.debug('[/assistants/chat/] req.body', req.body);
+
+  /** @type {{ files: MongoFile[]}} */
+  const {
+    text,
+    model,
+    endpoint,
+    files = [],
+    promptPrefix,
+    assistant_id,
+    instructions,
+    thread_id: _thread_id,
+    messageId: _messageId,
+    conversationId: convoId,
+    parentMessageId: _parentId = Constants.NO_PARENT,
+  } = req.body;
+
+  /** @type {OpenAIClient} */
+  let openai;
+  /** @type {string|undefined} - the current thread id */
+  let thread_id = _thread_id;
+  /** @type {string|undefined} - the current run id */
+  let run_id;
+  /** @type {string|undefined} - the parent messageId */
+  let parentMessageId = _parentId;
+  /** @type {TMessage[]} */
+  let previousMessages = [];
+  /** @type {import('librechat-data-provider').TConversation | null} */
+  let conversation = null;
+  /** @type {string[]} */
+  let file_ids = [];
+  /** @type {Set<string>} */
+  let attachedFileIds = new Set();
+  /** @type {TMessage | null} */
+  let requestMessage = null;
+
+  const userMessageId = v4();
+  const responseMessageId = v4();
+
+  /** @type {string} - The conversation UUID - created if undefined */
+  const conversationId = convoId ?? v4();
+
+  const cache = getLogStores(CacheKeys.ABORT_KEYS);
+  const cacheKey = `${req.user.id}:${conversationId}`;
+
+  /** @type {Run | undefined} - The completed run, undefined if incomplete */
+  let completedRun;
+
+  const handleError = async (error) => {
+    const defaultErrorMessage =
+      'The Assistant run failed to initialize. Try sending a message in a new conversation.';
+    const messageData = {
+      thread_id,
+      assistant_id,
+      conversationId,
+      parentMessageId,
+      sender: 'System',
+      user: req.user.id,
+      shouldSaveMessage: false,
+      messageId: responseMessageId,
+      endpoint,
+    };
+
+    if (error.message === 'Run cancelled') {
+      return res.end();
+    } else if (error.message === 'Request closed' && completedRun) {
+      return;
+    } else if (error.message === 'Request closed') {
+      logger.debug('[/assistants/chat/] Request aborted on close');
+    } else if (/Files.*are invalid/.test(error.message)) {
+      const errorMessage = `Files are invalid, or may not have uploaded yet.${
+        endpoint === EModelEndpoint.azureAssistants
+          ? ' If using Azure OpenAI, files are only available in the region of the assistant\'s model at the time of upload.'
+          : ''
+      }`;
+      return sendResponse(res, messageData, errorMessage);
+    } else if (error?.message?.includes('string too long')) {
+      return sendResponse(
+        res,
+        messageData,
+        'Message too long. The Assistants API has a limit of 32,768 characters per message. Please shorten it and try again.',
+      );
+    } else if (error?.message?.includes(ViolationTypes.TOKEN_BALANCE)) {
+      return sendResponse(res, messageData, error.message);
+    } else {
+      logger.error('[/assistants/chat/]', error);
+    }
+
+    if (!openai || !thread_id || !run_id) {
+      return sendResponse(res, messageData, defaultErrorMessage);
+    }
+
+    await sleep(2000);
+
+    try {
+      const status = await cache.get(cacheKey);
+      if (status === 'cancelled') {
+        logger.debug('[/assistants/chat/] Run already cancelled');
+        return res.end();
+      }
+      await cache.delete(cacheKey);
+      const cancelledRun = await openai.beta.threads.runs.cancel(thread_id, run_id);
+      logger.debug('[/assistants/chat/] Cancelled run:', cancelledRun);
+    } catch (error) {
+      logger.error('[/assistants/chat/] Error cancelling run', error);
+    }
+
+    await sleep(2000);
+
+    let run;
+    try {
+      run = await openai.beta.threads.runs.retrieve(thread_id, run_id);
+      await recordUsage({
+        ...run.usage,
+        model: run.model,
+        user: req.user.id,
+        conversationId,
+      });
+    } catch (error) {
+      logger.error('[/assistants/chat/] Error fetching or processing run', error);
+    }
+
+    let finalEvent;
+    try {
+      const runMessages = await checkMessageGaps({
+        openai,
+        run_id,
+        endpoint,
+        thread_id,
+        conversationId,
+        latestMessageId: responseMessageId,
+      });
+
+      const errorContentPart = {
+        text: {
+          value:
+            error?.message ?? 'There was an error processing your request. Please try again later.',
+        },
+        type: ContentTypes.ERROR,
+      };
+
+      if (!Array.isArray(runMessages[runMessages.length - 1]?.content)) {
+        runMessages[runMessages.length - 1].content = [errorContentPart];
+      } else {
+        const contentParts = runMessages[runMessages.length - 1].content;
+        for (let i = 0; i < contentParts.length; i++) {
+          const currentPart = contentParts[i];
+          /** @type {CodeToolCall | RetrievalToolCall | FunctionToolCall | undefined} */
+          const toolCall = currentPart?.[ContentTypes.TOOL_CALL];
+          if (
+            toolCall &&
+            toolCall?.function &&
+            !(toolCall?.function?.output || toolCall?.function?.output?.length)
+          ) {
+            contentParts[i] = {
+              ...currentPart,
+              [ContentTypes.TOOL_CALL]: {
+                ...toolCall,
+                function: {
+                  ...toolCall.function,
+                  output: 'error processing tool',
+                },
+              },
+            };
+          }
+        }
+        runMessages[runMessages.length - 1].content.push(errorContentPart);
+      }
+
+      finalEvent = {
+        final: true,
+        conversation: await getConvo(req.user.id, conversationId),
+        runMessages,
+      };
+    } catch (error) {
+      logger.error('[/assistants/chat/] Error finalizing error process', error);
+      return sendResponse(res, messageData, 'The Assistant run failed');
+    }
+
+    return sendResponse(res, finalEvent);
+  };
+
+  try {
+    res.on('close', async () => {
+      if (!completedRun) {
+        await handleError(new Error('Request closed'));
+      }
+    });
+
+    if (convoId && !_thread_id) {
+      completedRun = true;
+      throw new Error('Missing thread_id for existing conversation');
+    }
+
+    if (!assistant_id) {
+      completedRun = true;
+      throw new Error('Missing assistant_id');
+    }
+
+    const checkBalanceBeforeRun = async () => {
+      if (!isEnabled(process.env.CHECK_BALANCE)) {
+        return;
+      }
+      const transactions =
+        (await getTransactions({
+          user: req.user.id,
+          context: 'message',
+          conversationId,
+        })) ?? [];
+
+      const totalPreviousTokens = Math.abs(
+        transactions.reduce((acc, curr) => acc + curr.rawAmount, 0),
+      );
+
+      // TODO: make promptBuffer a config option; buffer for titles, needs buffer for system instructions
+      const promptBuffer = parentMessageId === Constants.NO_PARENT && !_thread_id ? 200 : 0;
+      // 5 is added for labels
+      let promptTokens = (await countTokens(text + (promptPrefix ?? ''))) + 5;
+      promptTokens += totalPreviousTokens + promptBuffer;
+      // Count tokens up to the current context window
+      promptTokens = Math.min(promptTokens, getModelMaxTokens(model));
+
+      await checkBalance({
+        req,
+        res,
+        txData: {
+          model,
+          user: req.user.id,
+          tokenType: 'prompt',
+          amount: promptTokens,
+        },
+      });
+    };
+
+    const { openai: _openai, client } = await getOpenAIClient({
+      req,
+      res,
+      endpointOption: req.body.endpointOption,
+      initAppClient: true,
+    });
+
+    openai = _openai;
+    await validateAuthor({ req, openai });
+
+    if (previousMessages.length) {
+      parentMessageId = previousMessages[previousMessages.length - 1].messageId;
+    }
+
+    let userMessage = {
+      role: 'user',
+      content: [
+        {
+          type: ContentTypes.TEXT,
+          text,
+        },
+      ],
+      metadata: {
+        messageId: userMessageId,
+      },
+    };
+
+    /** @type {CreateRunBody | undefined} */
+    const body = {
+      assistant_id,
+      model,
+    };
+
+    if (promptPrefix) {
+      body.additional_instructions = promptPrefix;
+    }
+
+    if (instructions) {
+      body.instructions = instructions;
+    }
+
+    const getRequestFileIds = async () => {
+      let thread_file_ids = [];
+      if (convoId) {
+        const convo = await getConvo(req.user.id, convoId);
+        if (convo && convo.file_ids) {
+          thread_file_ids = convo.file_ids;
+        }
+      }
+
+      if (files.length || thread_file_ids.length) {
+        attachedFileIds = new Set([...file_ids, ...thread_file_ids]);
+
+        let attachmentIndex = 0;
+        for (const file of files) {
+          file_ids.push(file.file_id);
+          if (file.type.startsWith('image')) {
+            userMessage.content.push({
+              type: ContentTypes.IMAGE_FILE,
+              [ContentTypes.IMAGE_FILE]: { file_id: file.file_id },
+            });
+          }
+
+          if (!userMessage.attachments) {
+            userMessage.attachments = [];
+          }
+
+          userMessage.attachments.push({
+            file_id: file.file_id,
+            tools: [{ type: ToolCallTypes.CODE_INTERPRETER }],
+          });
+
+          if (file.type.startsWith('image')) {
+            continue;
+          }
+
+          const mimeType = file.type;
+          const isSupportedByRetrieval = retrievalMimeTypes.some((regex) => regex.test(mimeType));
+          if (isSupportedByRetrieval) {
+            userMessage.attachments[attachmentIndex].tools.push({
+              type: ToolCallTypes.FILE_SEARCH,
+            });
+          }
+
+          attachmentIndex++;
+        }
+      }
+    };
+
+    const initializeThread = async () => {
+      await getRequestFileIds();
+
+      // TODO: may allow multiple messages to be created beforehand in a future update
+      const initThreadBody = {
+        messages: [userMessage],
+        metadata: {
+          user: req.user.id,
+          conversationId,
+        },
+      };
+
+      const result = await initThread({ openai, body: initThreadBody, thread_id });
+      thread_id = result.thread_id;
+
+      createOnTextProgress({
+        openai,
+        conversationId,
+        userMessageId,
+        messageId: responseMessageId,
+        thread_id,
+      });
+
+      requestMessage = {
+        user: req.user.id,
+        text,
+        messageId: userMessageId,
+        parentMessageId,
+        // TODO: make sure client sends correct format for `files`, use zod
+        files,
+        file_ids,
+        conversationId,
+        isCreatedByUser: true,
+        assistant_id,
+        thread_id,
+        model: assistant_id,
+        endpoint,
+      };
+
+      previousMessages.push(requestMessage);
+
+      /* asynchronous */
+      saveUserMessage({ ...requestMessage, model });
+
+      conversation = {
+        conversationId,
+        endpoint,
+        promptPrefix: promptPrefix,
+        instructions: instructions,
+        assistant_id,
+        // model,
+      };
+
+      if (file_ids.length) {
+        conversation.file_ids = file_ids;
+      }
+    };
+
+    const promises = [initializeThread(), checkBalanceBeforeRun()];
+    await Promise.all(promises);
+
+    const sendInitialResponse = () => {
+      sendMessage(res, {
+        sync: true,
+        conversationId,
+        // messages: previousMessages,
+        requestMessage,
+        responseMessage: {
+          user: req.user.id,
+          messageId: openai.responseMessage.messageId,
+          parentMessageId: userMessageId,
+          conversationId,
+          assistant_id,
+          thread_id,
+          model: assistant_id,
+        },
+      });
+    };
+
+    /** @type {RunResponse | typeof StreamRunManager | undefined} */
+    let response;
+
+    const processRun = async (retry = false) => {
+      if (endpoint === EModelEndpoint.azureAssistants) {
+        body.model = openai._options.model;
+        openai.attachedFileIds = attachedFileIds;
+        if (retry) {
+          response = await runAssistant({
+            openai,
+            thread_id,
+            run_id,
+            in_progress: openai.in_progress,
+          });
+          return;
+        }
+
+        /* NOTE:
+         * By default, a Run will use the model and tools configuration specified in Assistant object,
+         * but you can override most of these when creating the Run for added flexibility:
+         */
+        const run = await createRun({
+          openai,
+          thread_id,
+          body,
+        });
+
+        run_id = run.id;
+        await cache.set(cacheKey, `${thread_id}:${run_id}`, ten_minutes);
+        sendInitialResponse();
+
+        // todo: retry logic
+        response = await runAssistant({ openai, thread_id, run_id });
+        return;
+      }
+
+      /** @type {{[AssistantStreamEvents.ThreadRunCreated]: (event: ThreadRunCreated) => Promise<void>}} */
+      const handlers = {
+        [AssistantStreamEvents.ThreadRunCreated]: async (event) => {
+          await cache.set(cacheKey, `${thread_id}:${event.data.id}`, ten_minutes);
+          run_id = event.data.id;
+          sendInitialResponse();
+        },
+      };
+
+      const streamRunManager = new StreamRunManager({
+        req,
+        res,
+        openai,
+        handlers,
+        thread_id,
+        attachedFileIds,
+        parentMessageId: userMessageId,
+        responseMessage: openai.responseMessage,
+        // streamOptions: {
+
+        // },
+      });
+
+      await streamRunManager.runAssistant({
+        thread_id,
+        body,
+      });
+
+      response = streamRunManager;
+      response.text = streamRunManager.intermediateText;
+    };
+
+    await processRun();
+    logger.debug('[/assistants/chat/] response', {
+      run: response.run,
+      steps: response.steps,
+    });
+
+    if (response.run.status === RunStatus.CANCELLED) {
+      logger.debug('[/assistants/chat/] Run cancelled, handled by `abortRun`');
+      return res.end();
+    }
+
+    if (response.run.status === RunStatus.IN_PROGRESS) {
+      processRun(true);
+    }
+
+    completedRun = response.run;
+
+    /** @type {ResponseMessage} */
+    const responseMessage = {
+      ...(response.responseMessage ?? response.finalMessage),
+      text: response.text,
+      parentMessageId: userMessageId,
+      conversationId,
+      user: req.user.id,
+      assistant_id,
+      thread_id,
+      model: assistant_id,
+      endpoint,
+    };
+
+    sendMessage(res, {
+      final: true,
+      conversation,
+      requestMessage: {
+        parentMessageId,
+        thread_id,
+      },
+    });
+    res.end();
+
+    await saveAssistantMessage({ ...responseMessage, model });
+
+    if (parentMessageId === Constants.NO_PARENT && !_thread_id) {
+      addTitle(req, {
+        text,
+        responseText: response.text,
+        conversationId,
+        client,
+      });
+    }
+
+    await addThreadMetadata({
+      openai,
+      thread_id,
+      messageId: responseMessage.messageId,
+      messages: response.messages,
+    });
+
+    if (!response.run.usage) {
+      await sleep(3000);
+      completedRun = await openai.beta.threads.runs.retrieve(thread_id, response.run.id);
+      if (completedRun.usage) {
+        await recordUsage({
+          ...completedRun.usage,
+          user: req.user.id,
+          model: completedRun.model ?? model,
+          conversationId,
+        });
+      }
+    } else {
+      await recordUsage({
+        ...response.run.usage,
+        user: req.user.id,
+        model: response.run.model ?? model,
+        conversationId,
+      });
+    }
+  } catch (error) {
+    await handleError(error);
+  }
+};
+
+module.exports = chatV2;

api/server/controllers/assistants/helpers.js

@@ -0,0 +1,269 @@
+const {
+  EModelEndpoint,
+  CacheKeys,
+  defaultAssistantsVersion,
+  defaultOrderQuery,
+} = require('librechat-data-provider');
+const {
+  initializeClient: initAzureClient,
+} = require('~/server/services/Endpoints/azureAssistants');
+const { initializeClient } = require('~/server/services/Endpoints/assistants');
+const { getLogStores } = require('~/cache');
+
+/**
+ * @param {Express.Request} req
+ * @param {string} [endpoint]
+ * @returns {Promise<string>}
+ */
+const getCurrentVersion = async (req, endpoint) => {
+  const index = req.baseUrl.lastIndexOf('/v');
+  let version = index !== -1 ? req.baseUrl.substring(index + 1, index + 3) : null;
+  if (!version && req.body.version) {
+    version = `v${req.body.version}`;
+  }
+  if (!version && endpoint) {
+    const cache = getLogStores(CacheKeys.CONFIG_STORE);
+    const cachedEndpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG);
+    version = `v${
+      cachedEndpointsConfig?.[endpoint]?.version ?? defaultAssistantsVersion[endpoint]
+    }`;
+  }
+  if (!version?.startsWith('v') && version.length !== 2) {
+    throw new Error(`[${req.baseUrl}] Invalid version: ${version}`);
+  }
+  return version;
+};
+
+/**
+ * Asynchronously lists assistants based on provided query parameters.
+ *
+ * Initializes the client with the current request and response objects and lists assistants
+ * according to the query parameters. This function abstracts the logic for non-Azure paths.
+ *
+ * @deprecated
+ * @async
+ * @param {object} params - The parameters object.
+ * @param {object} params.req - The request object, used for initializing the client.
+ * @param {object} params.res - The response object, used for initializing the client.
+ * @param {string} params.version - The API version to use.
+ * @param {object} params.query - The query parameters to list assistants (e.g., limit, order).
+ * @returns {Promise<object>} A promise that resolves to the response from the `openai.beta.assistants.list` method call.
+ */
+const _listAssistants = async ({ req, res, version, query }) => {
+  const { openai } = await getOpenAIClient({ req, res, version });
+  return openai.beta.assistants.list(query);
+};
+
+/**
+ * Fetches all assistants based on provided query params, until `has_more` is `false`.
+ *
+ * @async
+ * @param {object} params - The parameters object.
+ * @param {object} params.req - The request object, used for initializing the client.
+ * @param {object} params.res - The response object, used for initializing the client.
+ * @param {string} params.version - The API version to use.
+ * @param {Omit<AssistantListParams, 'endpoint'>} params.query - The query parameters to list assistants (e.g., limit, order).
+ * @returns {Promise<object>} A promise that resolves to the response from the `openai.beta.assistants.list` method call.
+ */
+const listAllAssistants = async ({ req, res, version, query }) => {
+  /** @type {{ openai: OpenAIClient }} */
+  const { openai } = await getOpenAIClient({ req, res, version });
+  const allAssistants = [];
+
+  let first_id;
+  let last_id;
+  let afterToken = query.after;
+  let hasMore = true;
+
+  while (hasMore) {
+    const response = await openai.beta.assistants.list({
+      ...query,
+      after: afterToken,
+    });
+
+    const { body } = response;
+
+    allAssistants.push(...body.data);
+    hasMore = body.has_more;
+
+    if (!first_id) {
+      first_id = body.first_id;
+    }
+
+    if (hasMore) {
+      afterToken = body.last_id;
+    } else {
+      last_id = body.last_id;
+    }
+  }
+
+  return {
+    data: allAssistants,
+    body: {
+      data: allAssistants,
+      has_more: false,
+      first_id,
+      last_id,
+    },
+  };
+};
+
+/**
+ * Asynchronously lists assistants for Azure configured groups.
+ *
+ * Iterates through Azure configured assistant groups, initializes the client with the current request and response objects,
+ * lists assistants based on the provided query parameters, and merges their data alongside the model information into a single array.
+ *
+ * @async
+ * @param {object} params - The parameters object.
+ * @param {object} params.req - The request object, used for initializing the client and manipulating the request body.
+ * @param {object} params.res - The response object, used for initializing the client.
+ * @param {string} params.version - The API version to use.
+ * @param {TAzureConfig} params.azureConfig - The Azure configuration object containing assistantGroups and groupMap.
+ * @param {object} params.query - The query parameters to list assistants (e.g., limit, order).
+ * @returns {Promise<AssistantListResponse>} A promise that resolves to an array of assistant data merged with their respective model information.
+ */
+const listAssistantsForAzure = async ({ req, res, version, azureConfig = {}, query }) => {
+  /** @type {Array<[string, TAzureModelConfig]>} */
+  const groupModelTuples = [];
+  const promises = [];
+  /** @type {Array<TAzureGroup>} */
+  const groups = [];
+
+  const { groupMap, assistantGroups } = azureConfig;
+
+  for (const groupName of assistantGroups) {
+    const group = groupMap[groupName];
+    groups.push(group);
+
+    const currentModelTuples = Object.entries(group?.models);
+    groupModelTuples.push(currentModelTuples);
+
+    /* The specified model is only necessary to
+    fetch assistants for the shared instance */
+    req.body.model = currentModelTuples[0][0];
+    promises.push(listAllAssistants({ req, res, version, query }));
+  }
+
+  const resolvedQueries = await Promise.all(promises);
+  const data = resolvedQueries.flatMap((res, i) =>
+    res.data.map((assistant) => {
+      const deploymentName = assistant.model;
+      const currentGroup = groups[i];
+      const currentModelTuples = groupModelTuples[i];
+      const firstModel = currentModelTuples[0][0];
+
+      if (currentGroup.deploymentName === deploymentName) {
+        return { ...assistant, model: firstModel };
+      }
+
+      for (const [model, modelConfig] of currentModelTuples) {
+        if (modelConfig.deploymentName === deploymentName) {
+          return { ...assistant, model };
+        }
+      }
+
+      return { ...assistant, model: firstModel };
+    }),
+  );
+
+  return {
+    first_id: data[0]?.id,
+    last_id: data[data.length - 1]?.id,
+    object: 'list',
+    has_more: false,
+    data,
+  };
+};
+
+async function getOpenAIClient({ req, res, endpointOption, initAppClient, overrideEndpoint }) {
+  let endpoint = overrideEndpoint ?? req.body.endpoint ?? req.query.endpoint;
+  const version = await getCurrentVersion(req, endpoint);
+  if (!endpoint) {
+    throw new Error(`[${req.baseUrl}] Endpoint is required`);
+  }
+
+  let result;
+  if (endpoint === EModelEndpoint.assistants) {
+    result = await initializeClient({ req, res, version, endpointOption, initAppClient });
+  } else if (endpoint === EModelEndpoint.azureAssistants) {
+    result = await initAzureClient({ req, res, version, endpointOption, initAppClient });
+  }
+
+  return result;
+}
+
+/**
+ * Returns a list of assistants.
+ * @param {object} params
+ * @param {object} params.req - Express Request
+ * @param {AssistantListParams} [params.req.query] - The assistant list parameters for pagination and sorting.
+ * @param {object} params.res - Express Response
+ * @param {string} [params.overrideEndpoint] - The endpoint to override the request endpoint.
+ * @returns {Promise<AssistantListResponse>} 200 - success response - application/json
+ */
+const fetchAssistants = async ({ req, res, overrideEndpoint }) => {
+  const {
+    limit = 100,
+    order = 'desc',
+    after,
+    before,
+    endpoint,
+  } = req.query ?? {
+    endpoint: overrideEndpoint,
+    ...defaultOrderQuery,
+  };
+
+  const version = await getCurrentVersion(req, endpoint);
+  const query = { limit, order, after, before };
+
+  /** @type {AssistantListResponse} */
+  let body;
+
+  if (endpoint === EModelEndpoint.assistants) {
+    ({ body } = await listAllAssistants({ req, res, version, query }));
+  } else if (endpoint === EModelEndpoint.azureAssistants) {
+    const azureConfig = req.app.locals[EModelEndpoint.azureOpenAI];
+    body = await listAssistantsForAzure({ req, res, version, azureConfig, query });
+  }
+
+  if (req.user.role === 'ADMIN') {
+    return body;
+  } else if (!req.app.locals[endpoint]) {
+    return body;
+  }
+
+  body.data = filterAssistants({
+    userId: req.user.id,
+    assistants: body.data,
+    assistantsConfig: req.app.locals[endpoint],
+  });
+  return body;
+};
+
+/**
+ * Filter assistants based on configuration.
+ *
+ * @param {object} params - The parameters object.
+ * @param {string} params.userId -  The user ID to filter private assistants.
+ * @param {Assistant[]} params.assistants - The list of assistants to filter.
+ * @param {Partial<TAssistantEndpoint>} params.assistantsConfig -  The assistant configuration.
+ * @returns {Assistant[]} - The filtered list of assistants.
+ */
+function filterAssistants({ assistants, userId, assistantsConfig }) {
+  const { supportedIds, excludedIds, privateAssistants } = assistantsConfig;
+  if (privateAssistants) {
+    return assistants.filter((assistant) => userId === assistant.metadata?.author);
+  } else if (supportedIds?.length) {
+    return assistants.filter((assistant) => supportedIds.includes(assistant.id));
+  } else if (excludedIds?.length) {
+    return assistants.filter((assistant) => !excludedIds.includes(assistant.id));
+  }
+  return assistants;
+}
+
+module.exports = {
+  getOpenAIClient,
+  fetchAssistants,
+  getCurrentVersion,
+};

api/server/controllers/assistants/v1.js

@@ -1,34 +1,12 @@
-const multer = require('multer');
-const express = require('express');
-const { FileContext, EModelEndpoint } = require('librechat-data-provider');
-const {
-  initializeClient,
-  listAssistantsForAzure,
-  listAssistants,
-} = require('~/server/services/Endpoints/assistants');
+const { FileContext } = require('librechat-data-provider');
+const validateAuthor = require('~/server/middleware/assistants/validateAuthor');
 const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+const { deleteAssistantActions } = require('~/server/services/ActionService');
+const { updateAssistantDoc, getAssistants } = require('~/models/Assistant');
 const { uploadImageBuffer } = require('~/server/services/Files/process');
-const { updateAssistant, getAssistants } = require('~/models/Assistant');
+const { getOpenAIClient, fetchAssistants } = require('./helpers');
 const { deleteFileByFilter } = require('~/models/File');
 const { logger } = require('~/config');
-const actions = require('./actions');
-const tools = require('./tools');
-
-const upload = multer();
-const router = express.Router();
-
-/**
- * Assistant actions route.
- * @route GET|POST /assistants/actions
- */
-router.use('/actions', actions);
-
-/**
- * Create an assistant.
- * @route GET /assistants/tools
- * @returns {TPlugin[]} 200 - application/json
- */
-router.use('/tools', tools);
 
 /**
  * Create an assistant.
@@ -36,12 +14,11 @@ router.use('/tools', tools);
  * @param {AssistantCreateParams} req.body - The assistant creation parameters.
  * @returns {Assistant} 201 - success response - application/json
  */
-router.post('/', async (req, res) => {
+const createAssistant = async (req, res) => {
   try {
-    /** @type {{ openai: OpenAI }} */
-    const { openai } = await initializeClient({ req, res });
+    const { openai } = await getOpenAIClient({ req, res });
 
-    const { tools = [], ...assistantData } = req.body;
+    const { tools = [], endpoint, ...assistantData } = req.body;
     assistantData.tools = tools
       .map((tool) => {
         if (typeof tool !== 'string') {
@@ -52,18 +29,30 @@ router.post('/', async (req, res) => {
       })
       .filter((tool) => tool);
 
+    let azureModelIdentifier = null;
     if (openai.locals?.azureOptions) {
+      azureModelIdentifier = assistantData.model;
       assistantData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName;
     }
 
+    assistantData.metadata = {
+      author: req.user.id,
+      endpoint,
+    };
+
     const assistant = await openai.beta.assistants.create(assistantData);
+    const promise = updateAssistantDoc({ assistant_id: assistant.id }, { user: req.user.id });
+    if (azureModelIdentifier) {
+      assistant.model = azureModelIdentifier;
+    }
+    await promise;
     logger.debug('/assistants/', assistant);
     res.status(201).json(assistant);
   } catch (error) {
     logger.error('[/assistants] Error creating assistant', error);
     res.status(500).json({ error: error.message });
   }
-});
+};
 
 /**
  * Retrieves an assistant.
@@ -71,11 +60,10 @@ router.post('/', async (req, res) => {
  * @param {string} req.params.id - Assistant identifier.
  * @returns {Assistant} 200 - success response - application/json
  */
-router.get('/:id', async (req, res) => {
+const retrieveAssistant = async (req, res) => {
   try {
-    /** @type {{ openai: OpenAI }} */
-    const { openai } = await initializeClient({ req, res });
-
+    /* NOTE: not actually being used right now */
+    const { openai } = await getOpenAIClient({ req, res });
     const assistant_id = req.params.id;
     const assistant = await openai.beta.assistants.retrieve(assistant_id);
     res.json(assistant);
@@ -83,22 +71,24 @@ router.get('/:id', async (req, res) => {
     logger.error('[/assistants/:id] Error retrieving assistant', error);
     res.status(500).json({ error: error.message });
   }
-});
+};
 
 /**
  * Modifies an assistant.
  * @route PATCH /assistants/:id
+ * @param {object} req - Express Request
+ * @param {object} req.params - Request params
  * @param {string} req.params.id - Assistant identifier.
  * @param {AssistantUpdateParams} req.body - The assistant update parameters.
  * @returns {Assistant} 200 - success response - application/json
  */
-router.patch('/:id', async (req, res) => {
+const patchAssistant = async (req, res) => {
   try {
-    /** @type {{ openai: OpenAI }} */
-    const { openai } = await initializeClient({ req, res });
+    const { openai } = await getOpenAIClient({ req, res });
+    await validateAuthor({ req, openai });
 
     const assistant_id = req.params.id;
-    const updateData = req.body;
+    const { endpoint: _e, ...updateData } = req.body;
     updateData.tools = (updateData.tools ?? [])
       .map((tool) => {
         if (typeof tool !== 'string') {
@@ -119,90 +109,76 @@ router.patch('/:id', async (req, res) => {
     logger.error('[/assistants/:id] Error updating assistant', error);
     res.status(500).json({ error: error.message });
   }
-});
+};
 
 /**
  * Deletes an assistant.
  * @route DELETE /assistants/:id
+ * @param {object} req - Express Request
+ * @param {object} req.params - Request params
  * @param {string} req.params.id - Assistant identifier.
  * @returns {Assistant} 200 - success response - application/json
  */
-router.delete('/:id', async (req, res) => {
+const deleteAssistant = async (req, res) => {
   try {
-    /** @type {{ openai: OpenAI }} */
-    const { openai } = await initializeClient({ req, res });
+    const { openai } = await getOpenAIClient({ req, res });
+    await validateAuthor({ req, openai });
 
     const assistant_id = req.params.id;
     const deletionStatus = await openai.beta.assistants.del(assistant_id);
+    if (deletionStatus?.deleted) {
+      await deleteAssistantActions({ req, assistant_id });
+    }
     res.json(deletionStatus);
   } catch (error) {
     logger.error('[/assistants/:id] Error deleting assistant', error);
     res.status(500).json({ error: 'Error deleting assistant' });
   }
-});
+};
 
 /**
  * Returns a list of assistants.
  * @route GET /assistants
+ * @param {object} req - Express Request
  * @param {AssistantListParams} req.query - The assistant list parameters for pagination and sorting.
  * @returns {AssistantListResponse} 200 - success response - application/json
  */
-router.get('/', async (req, res) => {
+const listAssistants = async (req, res) => {
   try {
-    const { limit = 100, order = 'desc', after, before } = req.query;
-    const query = { limit, order, after, before };
-
-    const azureConfig = req.app.locals[EModelEndpoint.azureOpenAI];
-    /** @type {AssistantListResponse} */
-    let body;
-
-    if (azureConfig?.assistants) {
-      body = await listAssistantsForAzure({ req, res, azureConfig, query });
-    } else {
-      ({ body } = await listAssistants({ req, res, query }));
-    }
-
-    if (req.app.locals?.[EModelEndpoint.assistants]) {
-      /** @type {Partial<TAssistantEndpoint>} */
-      const assistantsConfig = req.app.locals[EModelEndpoint.assistants];
-      const { supportedIds, excludedIds } = assistantsConfig;
-      if (supportedIds?.length) {
-        body.data = body.data.filter((assistant) => supportedIds.includes(assistant.id));
-      } else if (excludedIds?.length) {
-        body.data = body.data.filter((assistant) => !excludedIds.includes(assistant.id));
-      }
-    }
-
+    const body = await fetchAssistants({ req, res });
     res.json(body);
   } catch (error) {
     logger.error('[/assistants] Error listing assistants', error);
     res.status(500).json({ message: 'Error listing assistants' });
   }
-});
+};
 
 /**
  * Returns a list of the user's assistant documents (metadata saved to database).
  * @route GET /assistants/documents
  * @returns {AssistantDocument[]} 200 - success response - application/json
  */
-router.get('/documents', async (req, res) => {
+const getAssistantDocuments = async (req, res) => {
   try {
     res.json(await getAssistants({ user: req.user.id }));
   } catch (error) {
     logger.error('[/assistants/documents] Error listing assistant documents', error);
     res.status(500).json({ error: error.message });
   }
-});
+};
 
 /**
  * Uploads and updates an avatar for a specific assistant.
  * @route POST /avatar/:assistant_id
+ * @param {object} req - Express Request
+ * @param {object} req.params - Request params
  * @param {string} req.params.assistant_id - The ID of the assistant.
  * @param {Express.Multer.File} req.file - The avatar image file.
+ * @param {object} req.body - Request body
  * @param {string} [req.body.metadata] - Optional metadata for the assistant's avatar.
  * @returns {Object} 200 - success response - application/json
  */
-router.post('/avatar/:assistant_id', upload.single('file'), async (req, res) => {
+const uploadAssistantAvatar = async (req, res) => {
   try {
     const { assistant_id } = req.params;
     if (!assistant_id) {
@@ -210,10 +186,16 @@ router.post('/avatar/:assistant_id', upload.single('file'), async (req, res) =>
     }
 
     let { metadata: _metadata = '{}' } = req.body;
-    /** @type {{ openai: OpenAI }} */
-    const { openai } = await initializeClient({ req, res });
+    const { openai } = await getOpenAIClient({ req, res });
+    await validateAuthor({ req, openai });
 
-    const image = await uploadImageBuffer({ req, context: FileContext.avatar });
+    const image = await uploadImageBuffer({
+      req,
+      context: FileContext.avatar,
+      metadata: {
+        buffer: req.file.buffer,
+      },
+    });
 
     try {
       _metadata = JSON.parse(_metadata);
@@ -240,7 +222,7 @@ router.post('/avatar/:assistant_id', upload.single('file'), async (req, res) =>
 
     const promises = [];
     promises.push(
-      updateAssistant(
+      updateAssistantDoc(
         { assistant_id },
         {
           avatar: {
@@ -260,6 +242,14 @@ router.post('/avatar/:assistant_id', upload.single('file'), async (req, res) =>
     logger.error(message, error);
     res.status(500).json({ message });
   }
-});
+};
 
-module.exports = router;
+module.exports = {
+  createAssistant,
+  retrieveAssistant,
+  patchAssistant,
+  deleteAssistant,
+  listAssistants,
+  getAssistantDocuments,
+  uploadAssistantAvatar,
+};

api/server/controllers/assistants/v2.js

@@ -0,0 +1,213 @@
+const { ToolCallTypes } = require('librechat-data-provider');
+const validateAuthor = require('~/server/middleware/assistants/validateAuthor');
+const { validateAndUpdateTool } = require('~/server/services/ActionService');
+const { updateAssistantDoc } = require('~/models/Assistant');
+const { getOpenAIClient } = require('./helpers');
+const { logger } = require('~/config');
+
+/**
+ * Create an assistant.
+ * @route POST /assistants
+ * @param {AssistantCreateParams} req.body - The assistant creation parameters.
+ * @returns {Assistant} 201 - success response - application/json
+ */
+const createAssistant = async (req, res) => {
+  try {
+    /** @type {{ openai: OpenAIClient }} */
+    const { openai } = await getOpenAIClient({ req, res });
+
+    const { tools = [], endpoint, ...assistantData } = req.body;
+    assistantData.tools = tools
+      .map((tool) => {
+        if (typeof tool !== 'string') {
+          return tool;
+        }
+
+        return req.app.locals.availableTools[tool];
+      })
+      .filter((tool) => tool);
+
+    let azureModelIdentifier = null;
+    if (openai.locals?.azureOptions) {
+      azureModelIdentifier = assistantData.model;
+      assistantData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName;
+    }
+
+    assistantData.metadata = {
+      author: req.user.id,
+      endpoint,
+    };
+
+    const assistant = await openai.beta.assistants.create(assistantData);
+    const promise = updateAssistantDoc({ assistant_id: assistant.id }, { user: req.user.id });
+    if (azureModelIdentifier) {
+      assistant.model = azureModelIdentifier;
+    }
+    await promise;
+    logger.debug('/assistants/', assistant);
+    res.status(201).json(assistant);
+  } catch (error) {
+    logger.error('[/assistants] Error creating assistant', error);
+    res.status(500).json({ error: error.message });
+  }
+};
+
+/**
+ * Modifies an assistant.
+ * @param {object} params
+ * @param {Express.Request} params.req
+ * @param {OpenAIClient} params.openai
+ * @param {string} params.assistant_id
+ * @param {AssistantUpdateParams} params.updateData
+ * @returns {Promise<Assistant>} The updated assistant.
+ */
+const updateAssistant = async ({ req, openai, assistant_id, updateData }) => {
+  await validateAuthor({ req, openai });
+  const tools = [];
+
+  let hasFileSearch = false;
+  for (const tool of updateData.tools ?? []) {
+    let actualTool = typeof tool === 'string' ? req.app.locals.availableTools[tool] : tool;
+
+    if (!actualTool) {
+      continue;
+    }
+
+    if (actualTool.type === ToolCallTypes.FILE_SEARCH) {
+      hasFileSearch = true;
+    }
+
+    if (!actualTool.function) {
+      tools.push(actualTool);
+      continue;
+    }
+
+    const updatedTool = await validateAndUpdateTool({ req, tool: actualTool, assistant_id });
+    if (updatedTool) {
+      tools.push(updatedTool);
+    }
+  }
+
+  if (hasFileSearch && !updateData.tool_resources) {
+    const assistant = await openai.beta.assistants.retrieve(assistant_id);
+    updateData.tool_resources = assistant.tool_resources ?? null;
+  }
+
+  if (hasFileSearch && !updateData.tool_resources?.file_search) {
+    updateData.tool_resources = {
+      ...(updateData.tool_resources ?? {}),
+      file_search: {
+        vector_store_ids: [],
+      },
+    };
+  }
+
+  updateData.tools = tools;
+
+  if (openai.locals?.azureOptions && updateData.model) {
+    updateData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName;
+  }
+
+  return await openai.beta.assistants.update(assistant_id, updateData);
+};
+
+/**
+ * Modifies an assistant with the resource file id.
+ * @param {object} params
+ * @param {Express.Request} params.req
+ * @param {OpenAIClient} params.openai
+ * @param {string} params.assistant_id
+ * @param {string} params.tool_resource
+ * @param {string} params.file_id
+ * @param {AssistantUpdateParams} params.updateData
+ * @returns {Promise<Assistant>} The updated assistant.
+ */
+const addResourceFileId = async ({ req, openai, assistant_id, tool_resource, file_id }) => {
+  const assistant = await openai.beta.assistants.retrieve(assistant_id);
+  const { tool_resources = {} } = assistant;
+  if (tool_resources[tool_resource]) {
+    tool_resources[tool_resource].file_ids.push(file_id);
+  } else {
+    tool_resources[tool_resource] = { file_ids: [file_id] };
+  }
+
+  delete assistant.id;
+  return await updateAssistant({
+    req,
+    openai,
+    assistant_id,
+    updateData: { tools: assistant.tools, tool_resources },
+  });
+};
+
+/**
+ * Deletes a file ID from an assistant's resource.
+ * @param {object} params
+ * @param {Express.Request} params.req
+ * @param {OpenAIClient} params.openai
+ * @param {string} params.assistant_id
+ * @param {string} [params.tool_resource]
+ * @param {string} params.file_id
+ * @param {AssistantUpdateParams} params.updateData
+ * @returns {Promise<Assistant>} The updated assistant.
+ */
+const deleteResourceFileId = async ({ req, openai, assistant_id, tool_resource, file_id }) => {
+  const assistant = await openai.beta.assistants.retrieve(assistant_id);
+  const { tool_resources = {} } = assistant;
+
+  if (tool_resource && tool_resources[tool_resource]) {
+    const resource = tool_resources[tool_resource];
+    const index = resource.file_ids.indexOf(file_id);
+    if (index !== -1) {
+      resource.file_ids.splice(index, 1);
+    }
+  } else {
+    for (const resourceKey in tool_resources) {
+      const resource = tool_resources[resourceKey];
+      const index = resource.file_ids.indexOf(file_id);
+      if (index !== -1) {
+        resource.file_ids.splice(index, 1);
+        break;
+      }
+    }
+  }
+
+  delete assistant.id;
+  return await updateAssistant({
+    req,
+    openai,
+    assistant_id,
+    updateData: { tools: assistant.tools, tool_resources },
+  });
+};
+
+/**
+ * Modifies an assistant.
+ * @route PATCH /assistants/:id
+ * @param {object} req - Express Request
+ * @param {object} req.params - Request params
+ * @param {string} req.params.id - Assistant identifier.
+ * @param {AssistantUpdateParams} req.body - The assistant update parameters.
+ * @returns {Assistant} 200 - success response - application/json
+ */
+const patchAssistant = async (req, res) => {
+  try {
+    const { openai } = await getOpenAIClient({ req, res });
+    const assistant_id = req.params.id;
+    const { endpoint: _e, ...updateData } = req.body;
+    updateData.tools = updateData.tools ?? [];
+    const updatedAssistant = await updateAssistant({ req, openai, assistant_id, updateData });
+    res.json(updatedAssistant);
+  } catch (error) {
+    logger.error('[/assistants/:id] Error updating assistant', error);
+    res.status(500).json({ error: error.message });
+  }
+};
+
+module.exports = {
+  patchAssistant,
+  createAssistant,
+  updateAssistant,
+  addResourceFileId,
+  deleteResourceFileId,
+};

api/server/index.js

@@ -6,6 +6,7 @@ const axios = require('axios');
 const express = require('express');
 const passport = require('passport');
 const mongoSanitize = require('express-mongo-sanitize');
+const validateImageRequest = require('./middleware/validateImageRequest');
 const errorController = require('./controllers/ErrorController');
 const { jwtLogin, passportLogin } = require('~/strategies');
 const configureSocialLogins = require('./socialLogins');
@@ -14,7 +15,7 @@ const AppService = require('./services/AppService');
 const noIndex = require('./middleware/noIndex');
 const { isEnabled } = require('~/server/utils');
 const { logger } = require('~/config');
-
+const { ldapLogin } = require('~/strategies');
 const routes = require('./routes');
 
 const { PORT, HOST, ALLOW_SOCIAL_LOGIN } = process.env ?? {};
@@ -43,7 +44,8 @@ const startServer = async () => {
   app.use(mongoSanitize());
   app.use(express.urlencoded({ extended: true, limit: '3mb' }));
   app.use(express.static(app.locals.paths.dist));
-  app.use(express.static(app.locals.paths.publicPath));
+  app.use(express.static(app.locals.paths.fonts));
+  app.use(express.static(app.locals.paths.assets));
   app.set('trust proxy', 1); // trust first proxy
   app.use(cors());
 
@@ -58,6 +60,11 @@ const startServer = async () => {
   passport.use(await jwtLogin());
   passport.use(passportLogin());
 
+  // LDAP Auth
+  if (process.env.LDAP_URL && process.env.LDAP_BIND_DN && process.env.LDAP_USER_SEARCH_BASE) {
+    passport.use(ldapLogin);
+  }
+
   if (isEnabled(ALLOW_SOCIAL_LOGIN)) {
     configureSocialLogins(app);
   }
@@ -82,6 +89,8 @@ const startServer = async () => {
   app.use('/api/config', routes.config);
   app.use('/api/assistants', routes.assistants);
   app.use('/api/files', await routes.files.initialize());
+  app.use('/images/', validateImageRequest, routes.staticRoute);
+  app.use('/api/share', routes.share);
 
   app.use((req, res) => {
     res.status(404).sendFile(path.join(app.locals.paths.dist, 'index.html'));

api/server/middleware/abortMiddleware.js

@@ -1,9 +1,9 @@
-const { EModelEndpoint } = require('librechat-data-provider');
+const { isAssistantsEndpoint } = require('librechat-data-provider');
 const { sendMessage, sendError, countTokens, isEnabled } = require('~/server/utils');
+const { truncateText, smartTruncateText } = require('~/app/clients/prompts');
 const { saveMessage, getConvo, getConvoTitle } = require('~/models');
 const clearPendingReq = require('~/cache/clearPendingReq');
 const abortControllers = require('./abortControllers');
-const { redactMessage } = require('~/config/parsers');
 const spendTokens = require('~/models/spendTokens');
 const { abortRun } = require('./abortRun');
 const { logger } = require('~/config');
@@ -15,7 +15,7 @@ async function abortMessage(req, res) {
     abortKey = conversationId;
   }
 
-  if (endpoint === EModelEndpoint.assistants) {
+  if (isAssistantsEndpoint(endpoint)) {
     return await abortRun(req, res);
   }
 
@@ -73,6 +73,8 @@ const createAbortController = (req, res, getAbortData) => {
       ...responseData,
       conversationId,
       finish_reason: 'incomplete',
+      endpoint: endpointOption.endpoint,
+      iconURL: endpointOption.iconURL,
       model: endpointOption.modelOptions.model,
       unfinished: false,
       error: false,
@@ -100,7 +102,15 @@ const createAbortController = (req, res, getAbortData) => {
 };
 
 const handleAbortError = async (res, req, error, data) => {
-  logger.error('[handleAbortError] AI response error; aborting request:', error);
+  if (error?.message?.includes('base64')) {
+    logger.error('[handleAbortError] Error in base64 encoding', {
+      ...error,
+      stack: smartTruncateText(error?.stack, 1000),
+      message: truncateText(error.message, 350),
+    });
+  } else {
+    logger.error('[handleAbortError] AI response error; aborting request:', error);
+  }
   const { sender, conversationId, messageId, parentMessageId, partialText } = data;
 
   if (error.stack && error.stack.includes('google')) {
@@ -109,13 +119,17 @@ const handleAbortError = async (res, req, error, data) => {
     );
   }
 
+  const errorText = error?.message?.includes('"type"')
+    ? error.message
+    : 'An error occurred while processing your request. Please contact the Admin.';
+
   const respondWithError = async (partialText) => {
     let options = {
       sender,
       messageId,
       conversationId,
       parentMessageId,
-      text: redactMessage(error.message),
+      text: errorText,
       shouldSaveMessage: true,
       user: req.user.id,
     };

api/server/middleware/abortRun.js

@@ -1,6 +1,7 @@
 const { CacheKeys, RunStatus, isUUID } = require('librechat-data-provider');
 const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { checkMessageGaps, recordUsage } = require('~/server/services/Threads');
+const { deleteMessages } = require('~/models/Message');
 const { getConvo } = require('~/models/Conversation');
 const getLogStores = require('~/cache/getLogStores');
 const { sendMessage } = require('~/server/utils');
@@ -10,7 +11,7 @@ const three_minutes = 1000 * 60 * 3;
 
 async function abortRun(req, res) {
   res.setHeader('Content-Type', 'application/json');
-  const { abortKey } = req.body;
+  const { abortKey, endpoint } = req.body;
   const [conversationId, latestMessageId] = abortKey.split(':');
   const conversation = await getConvo(req.user.id, conversationId);
 
@@ -66,16 +67,22 @@ async function abortRun(req, res) {
     logger.error('[abortRun] Error fetching or processing run', error);
   }
 
+  /* TODO: a reconciling strategy between the existing intermediate message would be more optimal than deleting it */
+  await deleteMessages({
+    user: req.user.id,
+    unfinished: true,
+    conversationId,
+  });
   runMessages = await checkMessageGaps({
     openai,
-    latestMessageId,
-    thread_id,
     run_id,
+    endpoint,
+    thread_id,
     conversationId,
+    latestMessageId,
   });
 
   const finalEvent = {
-    title: 'New Chat',
     final: true,
     conversation,
     runMessages,

api/server/middleware/assistants/validate.js

@@ -0,0 +1,43 @@
+const { v4 } = require('uuid');
+const { handleAbortError } = require('~/server/middleware/abortMiddleware');
+
+/**
+ * Checks if the assistant is supported or excluded
+ * @param {object} req - Express Request
+ * @param {object} req.body - The request payload.
+ * @param {object} res - Express Response
+ * @param {function} next - Express next middleware function.
+ * @returns {Promise<void>}
+ */
+const validateAssistant = async (req, res, next) => {
+  const { endpoint, conversationId, assistant_id, messageId } = req.body;
+
+  /** @type {Partial<TAssistantEndpoint>} */
+  const assistantsConfig = req.app.locals?.[endpoint];
+  if (!assistantsConfig) {
+    return next();
+  }
+
+  const { supportedIds, excludedIds } = assistantsConfig;
+  const error = { message: 'Assistant not supported' };
+  if (supportedIds?.length && !supportedIds.includes(assistant_id)) {
+    return await handleAbortError(res, req, error, {
+      sender: 'System',
+      conversationId,
+      messageId: v4(),
+      parentMessageId: messageId,
+      error,
+    });
+  } else if (excludedIds?.length && excludedIds.includes(assistant_id)) {
+    return await handleAbortError(res, req, error, {
+      sender: 'System',
+      conversationId,
+      messageId: v4(),
+      parentMessageId: messageId,
+    });
+  }
+
+  return next();
+};
+
+module.exports = validateAssistant;

api/server/middleware/assistants/validateAuthor.js

@@ -0,0 +1,42 @@
+const { getAssistant } = require('~/models/Assistant');
+
+/**
+ * Checks if the assistant is supported or excluded
+ * @param {object} params
+ * @param {object} params.req - Express Request
+ * @param {object} params.req.body - The request payload.
+ * @param {string} params.overrideEndpoint - The override endpoint
+ * @param {string} params.overrideAssistantId - The override assistant ID
+ * @param {OpenAIClient} params.openai - OpenAI API Client
+ * @returns {Promise<void>}
+ */
+const validateAuthor = async ({ req, openai, overrideEndpoint, overrideAssistantId }) => {
+  if (req.user.role === 'ADMIN') {
+    return;
+  }
+
+  const endpoint = overrideEndpoint ?? req.body.endpoint ?? req.query.endpoint;
+  const assistant_id =
+    overrideAssistantId ?? req.params.id ?? req.body.assistant_id ?? req.query.assistant_id;
+
+  /** @type {Partial<TAssistantEndpoint>} */
+  const assistantsConfig = req.app.locals?.[endpoint];
+  if (!assistantsConfig) {
+    return;
+  }
+
+  if (!assistantsConfig.privateAssistants) {
+    return;
+  }
+
+  const assistantDoc = await getAssistant({ assistant_id, user: req.user.id });
+  if (assistantDoc) {
+    return;
+  }
+  const assistant = await openai.beta.assistants.retrieve(assistant_id);
+  if (req.user.id !== assistant?.metadata?.author) {
+    throw new Error(`Assistant ${assistant_id} is not authored by the user.`);
+  }
+};
+
+module.exports = validateAuthor;

api/server/middleware/buildEndpointOption.js

@@ -1,5 +1,6 @@
 const { parseConvo, EModelEndpoint } = require('librechat-data-provider');
 const { getModelsConfig } = require('~/server/controllers/ModelController');
+const azureAssistants = require('~/server/services/Endpoints/azureAssistants');
 const assistants = require('~/server/services/Endpoints/assistants');
 const gptPlugins = require('~/server/services/Endpoints/gptPlugins');
 const { processFiles } = require('~/server/services/Files/process');
@@ -7,6 +8,8 @@ const anthropic = require('~/server/services/Endpoints/anthropic');
 const openAI = require('~/server/services/Endpoints/openAI');
 const custom = require('~/server/services/Endpoints/custom');
 const google = require('~/server/services/Endpoints/google');
+const enforceModelSpec = require('./enforceModelSpec');
+const { handleError } = require('~/server/utils');
 
 const buildFunction = {
   [EModelEndpoint.openAI]: openAI.buildOptions,
@@ -16,11 +19,46 @@ const buildFunction = {
   [EModelEndpoint.anthropic]: anthropic.buildOptions,
   [EModelEndpoint.gptPlugins]: gptPlugins.buildOptions,
   [EModelEndpoint.assistants]: assistants.buildOptions,
+  [EModelEndpoint.azureAssistants]: azureAssistants.buildOptions,
 };
 
 async function buildEndpointOption(req, res, next) {
   const { endpoint, endpointType } = req.body;
   const parsedBody = parseConvo({ endpoint, endpointType, conversation: req.body });
+
+  if (req.app.locals.modelSpecs?.list && req.app.locals.modelSpecs?.enforce) {
+    /** @type {{ list: TModelSpec[] }}*/
+    const { list } = req.app.locals.modelSpecs;
+    const { spec } = parsedBody;
+
+    if (!spec) {
+      return handleError(res, { text: 'No model spec selected' });
+    }
+
+    const currentModelSpec = list.find((s) => s.name === spec);
+    if (!currentModelSpec) {
+      return handleError(res, { text: 'Invalid model spec' });
+    }
+
+    if (endpoint !== currentModelSpec.preset.endpoint) {
+      return handleError(res, { text: 'Model spec mismatch' });
+    }
+
+    if (
+      currentModelSpec.preset.endpoint !== EModelEndpoint.gptPlugins &&
+      currentModelSpec.preset.tools
+    ) {
+      return handleError(res, {
+        text: `Only the "${EModelEndpoint.gptPlugins}" endpoint can have tools defined in the preset`,
+      });
+    }
+
+    const isValidModelSpec = enforceModelSpec(currentModelSpec, parsedBody);
+    if (!isValidModelSpec) {
+      return handleError(res, { text: 'Model spec mismatch' });
+    }
+  }
+
   req.body.endpointOption = buildFunction[endpointType ?? endpoint](
     endpoint,
     parsedBody,

api/server/middleware/checkBan.js

@@ -1,14 +1,13 @@
 const Keyv = require('keyv');
 const uap = require('ua-parser-js');
-const denyRequest = require('./denyRequest');
-const { getLogStores } = require('../../cache');
+const { ViolationTypes } = require('librechat-data-provider');
 const { isEnabled, removePorts } = require('../utils');
-const keyvRedis = require('../../cache/keyvRedis');
-const User = require('../../models/User');
+const keyvMongo = require('~/cache/keyvMongo');
+const denyRequest = require('./denyRequest');
+const { getLogStores } = require('~/cache');
+const User = require('~/models/User');
 
-const banCache = isEnabled(process.env.USE_REDIS)
-  ? new Keyv({ store: keyvRedis })
-  : new Keyv({ namespace: 'bans', ttl: 0 });
+const banCache = new Keyv({ store: keyvMongo, namespace: ViolationTypes.BAN, ttl: 0 });
 const message = 'Your account has been temporarily banned due to violations of our service.';
 
 /**
@@ -28,7 +27,7 @@ const banResponse = async (req, res) => {
   if (!ua.browser.name) {
     return res.status(403).json({ message });
   } else if (baseUrl === '/api/ask' || baseUrl === '/api/edit') {
-    return await denyRequest(req, res, { type: 'ban' });
+    return await denyRequest(req, res, { type: ViolationTypes.BAN });
   }
 
   return res.status(403).json({ message });
@@ -87,7 +86,7 @@ const checkBan = async (req, res, next = () => {}) => {
     return await banResponse(req, res);
   }
 
-  const banLogs = getLogStores('ban');
+  const banLogs = getLogStores(ViolationTypes.BAN);
   const duration = banLogs.opts.ttl;
 
   if (duration <= 0) {

api/server/middleware/checkDomainAllowed.js

@@ -0,0 +1,25 @@
+const { isDomainAllowed } = require('~/server/services/AuthService');
+const { logger } = require('~/config');
+
+/**
+ * Checks the domain's social login is allowed
+ *
+ * @async
+ * @function
+ * @param {Object} req - Express request object.
+ * @param {Object} res - Express response object.
+ * @param {Function} next - Next middleware function.
+ *
+ * @returns {Promise<function|Object>} - Returns a Promise which when resolved calls next middleware if the domain's email is allowed
+ */
+const checkDomainAllowed = async (req, res, next = () => {}) => {
+  const email = req?.user?.email;
+  if (email && !(await isDomainAllowed(email))) {
+    logger.error(`[Social Login] [Social Login not allowed] [Email: ${email}]`);
+    return res.redirect('/login');
+  } else {
+    return next();
+  }
+};
+
+module.exports = checkDomainAllowed;

api/server/middleware/enforceModelSpec.js

@@ -0,0 +1,58 @@
+const interchangeableKeys = new Map([
+  ['chatGptLabel', ['modelLabel']],
+  ['modelLabel', ['chatGptLabel']],
+]);
+
+/**
+ * Middleware to enforce the model spec for a conversation
+ * @param {TModelSpec} modelSpec - The model spec to enforce
+ * @param {TConversation} parsedBody - The parsed body of the conversation
+ * @returns {boolean} - Whether the model spec is enforced
+ */
+const enforceModelSpec = (modelSpec, parsedBody) => {
+  for (const [key, value] of Object.entries(modelSpec.preset)) {
+    if (key === 'endpoint') {
+      continue;
+    }
+
+    if (!checkMatch(key, value, parsedBody)) {
+      return false;
+    }
+  }
+  return true;
+};
+
+/**
+ * Checks if there is a match for the given key and value in the parsed body
+ * or any of its interchangeable keys, including deep comparison for objects and arrays.
+ * @param {string} key
+ * @param {any} value
+ * @param {object} parsedBody
+ * @returns {boolean}
+ */
+const checkMatch = (key, value, parsedBody) => {
+  const isEqual = (a, b) => {
+    if (Array.isArray(a) && Array.isArray(b)) {
+      return a.length === b.length && a.every((val, index) => isEqual(val, b[index]));
+    } else if (typeof a === 'object' && typeof b === 'object' && a !== null && b !== null) {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+      return keysA.length === keysB.length && keysA.every((k) => isEqual(a[k], b[k]));
+    }
+    return a === b;
+  };
+
+  if (isEqual(parsedBody[key], value)) {
+    return true;
+  }
+
+  if (interchangeableKeys.has(key)) {
+    return interchangeableKeys
+      .get(key)
+      .some((interchangeableKey) => isEqual(parsedBody[interchangeableKey], value));
+  }
+
+  return false;
+};
+
+module.exports = enforceModelSpec;

api/server/middleware/enforceModelSpec.spec.js

@@ -0,0 +1,47 @@
+// enforceModelSpec.test.js
+
+const enforceModelSpec = require('./enforceModelSpec');
+
+describe('enforceModelSpec function', () => {
+  test('returns true when all model specs match parsed body directly', () => {
+    const modelSpec = { preset: { title: 'Dialog', status: 'Active' } };
+    const parsedBody = { title: 'Dialog', status: 'Active' };
+    expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true);
+  });
+
+  test('returns true when model specs match via interchangeable keys', () => {
+    const modelSpec = { preset: { chatGptLabel: 'GPT-4' } };
+    const parsedBody = { modelLabel: 'GPT-4' };
+    expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true);
+  });
+
+  test('returns false if any key value does not match', () => {
+    const modelSpec = { preset: { language: 'English', level: 'Advanced' } };
+    const parsedBody = { language: 'Spanish', level: 'Advanced' };
+    expect(enforceModelSpec(modelSpec, parsedBody)).toBe(false);
+  });
+
+  test('ignores the \'endpoint\' key in model spec', () => {
+    const modelSpec = { preset: { endpoint: 'ignored', feature: 'Special' } };
+    const parsedBody = { feature: 'Special' };
+    expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true);
+  });
+
+  test('handles nested objects correctly', () => {
+    const modelSpec = { preset: { details: { time: 'noon', location: 'park' } } };
+    const parsedBody = { details: { time: 'noon', location: 'park' } };
+    expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true);
+  });
+
+  test('handles arrays within objects', () => {
+    const modelSpec = { preset: { tags: ['urgent', 'important'] } };
+    const parsedBody = { tags: ['urgent', 'important'] };
+    expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true);
+  });
+
+  test('fails when arrays in objects do not match', () => {
+    const modelSpec = { preset: { tags: ['urgent', 'important'] } };
+    const parsedBody = { tags: ['important', 'urgent'] }; // Different order
+    expect(enforceModelSpec(modelSpec, parsedBody)).toBe(false);
+  });
+});

api/server/middleware/importLimiters.js

@@ -0,0 +1,69 @@
+const rateLimit = require('express-rate-limit');
+const { ViolationTypes } = require('librechat-data-provider');
+const logViolation = require('~/cache/logViolation');
+
+const getEnvironmentVariables = () => {
+  const IMPORT_IP_MAX = parseInt(process.env.IMPORT_IP_MAX) || 100;
+  const IMPORT_IP_WINDOW = parseInt(process.env.IMPORT_IP_WINDOW) || 15;
+  const IMPORT_USER_MAX = parseInt(process.env.IMPORT_USER_MAX) || 50;
+  const IMPORT_USER_WINDOW = parseInt(process.env.IMPORT_USER_WINDOW) || 15;
+
+  const importIpWindowMs = IMPORT_IP_WINDOW * 60 * 1000;
+  const importIpMax = IMPORT_IP_MAX;
+  const importIpWindowInMinutes = importIpWindowMs / 60000;
+
+  const importUserWindowMs = IMPORT_USER_WINDOW * 60 * 1000;
+  const importUserMax = IMPORT_USER_MAX;
+  const importUserWindowInMinutes = importUserWindowMs / 60000;
+
+  return {
+    importIpWindowMs,
+    importIpMax,
+    importIpWindowInMinutes,
+    importUserWindowMs,
+    importUserMax,
+    importUserWindowInMinutes,
+  };
+};
+
+const createImportHandler = (ip = true) => {
+  const { importIpMax, importIpWindowInMinutes, importUserMax, importUserWindowInMinutes } =
+    getEnvironmentVariables();
+
+  return async (req, res) => {
+    const type = ViolationTypes.FILE_UPLOAD_LIMIT;
+    const errorMessage = {
+      type,
+      max: ip ? importIpMax : importUserMax,
+      limiter: ip ? 'ip' : 'user',
+      windowInMinutes: ip ? importIpWindowInMinutes : importUserWindowInMinutes,
+    };
+
+    await logViolation(req, res, type, errorMessage);
+    res.status(429).json({ message: 'Too many conversation import requests. Try again later' });
+  };
+};
+
+const createImportLimiters = () => {
+  const { importIpWindowMs, importIpMax, importUserWindowMs, importUserMax } =
+    getEnvironmentVariables();
+
+  const importIpLimiter = rateLimit({
+    windowMs: importIpWindowMs,
+    max: importIpMax,
+    handler: createImportHandler(),
+  });
+
+  const importUserLimiter = rateLimit({
+    windowMs: importUserWindowMs,
+    max: importUserMax,
+    handler: createImportHandler(false),
+    keyGenerator: function (req) {
+      return req.user?.id; // Use the user ID or NULL if not available
+    },
+  });
+
+  return { importIpLimiter, importUserLimiter };
+};
+
+module.exports = { createImportLimiters };

api/server/middleware/index.js

@@ -1,10 +1,12 @@
 const abortMiddleware = require('./abortMiddleware');
 const checkBan = require('./checkBan');
+const checkDomainAllowed = require('./checkDomainAllowed');
 const uaParser = require('./uaParser');
 const setHeaders = require('./setHeaders');
 const loginLimiter = require('./loginLimiter');
 const validateModel = require('./validateModel');
 const requireJwtAuth = require('./requireJwtAuth');
+const requireLdapAuth = require('./requireLdapAuth');
 const uploadLimiters = require('./uploadLimiters');
 const registerLimiter = require('./registerLimiter');
 const messageLimiters = require('./messageLimiters');
@@ -14,8 +16,10 @@ const concurrentLimiter = require('./concurrentLimiter');
 const validateMessageReq = require('./validateMessageReq');
 const buildEndpointOption = require('./buildEndpointOption');
 const validateRegistration = require('./validateRegistration');
+const validateImageRequest = require('./validateImageRequest');
 const moderateText = require('./moderateText');
 const noIndex = require('./noIndex');
+const importLimiters = require('./importLimiters');
 
 module.exports = {
   ...uploadLimiters,
@@ -26,6 +30,7 @@ module.exports = {
   setHeaders,
   loginLimiter,
   requireJwtAuth,
+  requireLdapAuth,
   registerLimiter,
   requireLocalAuth,
   validateEndpoint,
@@ -33,7 +38,10 @@ module.exports = {
   validateMessageReq,
   buildEndpointOption,
   validateRegistration,
+  validateImageRequest,
   validateModel,
   moderateText,
   noIndex,
+  ...importLimiters,
+  checkDomainAllowed,
 };

api/server/middleware/moderateText.js

@@ -1,5 +1,7 @@
 const axios = require('axios');
+const { ErrorTypes } = require('librechat-data-provider');
 const denyRequest = require('./denyRequest');
+const { logger } = require('~/config');
 
 async function moderateText(req, res, next) {
   if (process.env.OPENAI_MODERATION === 'true') {
@@ -23,12 +25,12 @@ async function moderateText(req, res, next) {
       const flagged = results.some((result) => result.flagged);
 
       if (flagged) {
-        const type = 'moderation';
+        const type = ErrorTypes.MODERATION;
         const errorMessage = { type };
         return await denyRequest(req, res, errorMessage);
       }
     } catch (error) {
-      console.error('Error in moderateText:', error);
+      logger.error('Error in moderateText:', error);
       const errorMessage = 'error in moderation check';
       return await denyRequest(req, res, errorMessage);
     }

api/server/middleware/requireLdapAuth.js

@@ -0,0 +1,22 @@
+const passport = require('passport');
+
+const requireLdapAuth = (req, res, next) => {
+  passport.authenticate('ldapauth', (err, user, info) => {
+    if (err) {
+      console.log({
+        title: '(requireLdapAuth) Error at passport.authenticate',
+        parameters: [{ name: 'error', value: err }],
+      });
+      return next(err);
+    }
+    if (!user) {
+      console.log({
+        title: '(requireLdapAuth) Error: No user',
+      });
+      return res.status(422).send(info);
+    }
+    req.user = user;
+    next();
+  })(req, res, next);
+};
+module.exports = requireLdapAuth;

api/server/middleware/speech/index.js

@@ -0,0 +1,7 @@
+const createTTSLimiters = require('./ttsLimiters');
+const createSTTLimiters = require('./sttLimiters');
+
+module.exports = {
+  createTTSLimiters,
+  createSTTLimiters,
+};

api/server/middleware/speech/sttLimiters.js

@@ -0,0 +1,68 @@
+const rateLimit = require('express-rate-limit');
+const { ViolationTypes } = require('librechat-data-provider');
+const logViolation = require('~/cache/logViolation');
+
+const getEnvironmentVariables = () => {
+  const STT_IP_MAX = parseInt(process.env.STT_IP_MAX) || 100;
+  const STT_IP_WINDOW = parseInt(process.env.STT_IP_WINDOW) || 1;
+  const STT_USER_MAX = parseInt(process.env.STT_USER_MAX) || 50;
+  const STT_USER_WINDOW = parseInt(process.env.STT_USER_WINDOW) || 1;
+
+  const sttIpWindowMs = STT_IP_WINDOW * 60 * 1000;
+  const sttIpMax = STT_IP_MAX;
+  const sttIpWindowInMinutes = sttIpWindowMs / 60000;
+
+  const sttUserWindowMs = STT_USER_WINDOW * 60 * 1000;
+  const sttUserMax = STT_USER_MAX;
+  const sttUserWindowInMinutes = sttUserWindowMs / 60000;
+
+  return {
+    sttIpWindowMs,
+    sttIpMax,
+    sttIpWindowInMinutes,
+    sttUserWindowMs,
+    sttUserMax,
+    sttUserWindowInMinutes,
+  };
+};
+
+const createSTTHandler = (ip = true) => {
+  const { sttIpMax, sttIpWindowInMinutes, sttUserMax, sttUserWindowInMinutes } =
+    getEnvironmentVariables();
+
+  return async (req, res) => {
+    const type = ViolationTypes.STT_LIMIT;
+    const errorMessage = {
+      type,
+      max: ip ? sttIpMax : sttUserMax,
+      limiter: ip ? 'ip' : 'user',
+      windowInMinutes: ip ? sttIpWindowInMinutes : sttUserWindowInMinutes,
+    };
+
+    await logViolation(req, res, type, errorMessage);
+    res.status(429).json({ message: 'Too many STT requests. Try again later' });
+  };
+};
+
+const createSTTLimiters = () => {
+  const { sttIpWindowMs, sttIpMax, sttUserWindowMs, sttUserMax } = getEnvironmentVariables();
+
+  const sttIpLimiter = rateLimit({
+    windowMs: sttIpWindowMs,
+    max: sttIpMax,
+    handler: createSTTHandler(),
+  });
+
+  const sttUserLimiter = rateLimit({
+    windowMs: sttUserWindowMs,
+    max: sttUserMax,
+    handler: createSTTHandler(false),
+    keyGenerator: function (req) {
+      return req.user?.id; // Use the user ID or NULL if not available
+    },
+  });
+
+  return { sttIpLimiter, sttUserLimiter };
+};
+
+module.exports = createSTTLimiters;

api/server/middleware/speech/ttsLimiters.js

@@ -0,0 +1,68 @@
+const rateLimit = require('express-rate-limit');
+const { ViolationTypes } = require('librechat-data-provider');
+const logViolation = require('~/cache/logViolation');
+
+const getEnvironmentVariables = () => {
+  const TTS_IP_MAX = parseInt(process.env.TTS_IP_MAX) || 100;
+  const TTS_IP_WINDOW = parseInt(process.env.TTS_IP_WINDOW) || 1;
+  const TTS_USER_MAX = parseInt(process.env.TTS_USER_MAX) || 50;
+  const TTS_USER_WINDOW = parseInt(process.env.TTS_USER_WINDOW) || 1;
+
+  const ttsIpWindowMs = TTS_IP_WINDOW * 60 * 1000;
+  const ttsIpMax = TTS_IP_MAX;
+  const ttsIpWindowInMinutes = ttsIpWindowMs / 60000;
+
+  const ttsUserWindowMs = TTS_USER_WINDOW * 60 * 1000;
+  const ttsUserMax = TTS_USER_MAX;
+  const ttsUserWindowInMinutes = ttsUserWindowMs / 60000;
+
+  return {
+    ttsIpWindowMs,
+    ttsIpMax,
+    ttsIpWindowInMinutes,
+    ttsUserWindowMs,
+    ttsUserMax,
+    ttsUserWindowInMinutes,
+  };
+};
+
+const createTTSHandler = (ip = true) => {
+  const { ttsIpMax, ttsIpWindowInMinutes, ttsUserMax, ttsUserWindowInMinutes } =
+    getEnvironmentVariables();
+
+  return async (req, res) => {
+    const type = ViolationTypes.TTS_LIMIT;
+    const errorMessage = {
+      type,
+      max: ip ? ttsIpMax : ttsUserMax,
+      limiter: ip ? 'ip' : 'user',
+      windowInMinutes: ip ? ttsIpWindowInMinutes : ttsUserWindowInMinutes,
+    };
+
+    await logViolation(req, res, type, errorMessage);
+    res.status(429).json({ message: 'Too many TTS requests. Try again later' });
+  };
+};
+
+const createTTSLimiters = () => {
+  const { ttsIpWindowMs, ttsIpMax, ttsUserWindowMs, ttsUserMax } = getEnvironmentVariables();
+
+  const ttsIpLimiter = rateLimit({
+    windowMs: ttsIpWindowMs,
+    max: ttsIpMax,
+    handler: createTTSHandler(),
+  });
+
+  const ttsUserLimiter = rateLimit({
+    windowMs: ttsUserWindowMs,
+    max: ttsUserMax,
+    handler: createTTSHandler(false),
+    keyGenerator: function (req) {
+      return req.user?.id; // Use the user ID or NULL if not available
+    },
+  });
+
+  return { ttsIpLimiter, ttsUserLimiter };
+};
+
+module.exports = createTTSLimiters;

api/server/middleware/validateImageRequest.js

@@ -0,0 +1,42 @@
+const cookies = require('cookie');
+const jwt = require('jsonwebtoken');
+const { logger } = require('~/config');
+
+/**
+ * Middleware to validate image request.
+ * Must be set by `secureImageLinks` via custom config file.
+ */
+function validateImageRequest(req, res, next) {
+  if (!req.app.locals.secureImageLinks) {
+    return next();
+  }
+
+  const refreshToken = req.headers.cookie ? cookies.parse(req.headers.cookie).refreshToken : null;
+  if (!refreshToken) {
+    logger.warn('[validateImageRequest] Refresh token not provided');
+    return res.status(401).send('Unauthorized');
+  }
+
+  let payload;
+  try {
+    payload = jwt.verify(refreshToken, process.env.JWT_REFRESH_SECRET);
+  } catch (err) {
+    logger.warn('[validateImageRequest]', err);
+    return res.status(403).send('Access Denied');
+  }
+
+  const currentTimeInSeconds = Math.floor(Date.now() / 1000);
+  if (payload.exp < currentTimeInSeconds) {
+    logger.warn('[validateImageRequest] Refresh token expired');
+    return res.status(403).send('Access Denied');
+  }
+
+  if (req.path.includes(payload.id)) {
+    logger.debug('[validateImageRequest] Image request validated');
+    next();
+  } else {
+    res.status(403).send('Access Denied');
+  }
+}
+
+module.exports = validateImageRequest;

api/server/routes/__tests__/config.spec.js

@@ -25,6 +25,11 @@ afterEach(() => {
   delete process.env.DOMAIN_SERVER;
   delete process.env.ALLOW_REGISTRATION;
   delete process.env.ALLOW_SOCIAL_LOGIN;
+  delete process.env.LDAP_URL;
+  delete process.env.LDAP_BIND_DN;
+  delete process.env.LDAP_BIND_CREDENTIALS;
+  delete process.env.LDAP_USER_SEARCH_BASE;
+  delete process.env.LDAP_SEARCH_FILTER;
 });
 
 //TODO: This works/passes locally but http request tests fail with 404 in CI. Need to figure out why.
@@ -50,6 +55,11 @@ describe.skip('GET /', () => {
     process.env.DOMAIN_SERVER = 'http://test-server.com';
     process.env.ALLOW_REGISTRATION = 'true';
     process.env.ALLOW_SOCIAL_LOGIN = 'true';
+    process.env.LDAP_URL = 'Test LDAP URL';
+    process.env.LDAP_BIND_DN = 'Test LDAP Bind DN';
+    process.env.LDAP_BIND_CREDENTIALS = 'Test LDAP Bind Credentials';
+    process.env.LDAP_USER_SEARCH_BASE = 'Test LDAP User Search Base';
+    process.env.LDAP_SEARCH_FILTER = 'Test LDAP Search Filter';
 
     const response = await request(app).get('/');
 
@@ -64,6 +74,7 @@ describe.skip('GET /', () => {
       openidLoginEnabled: true,
       openidLabel: 'Test OpenID',
       openidImageUrl: 'http://test-server.com',
+      ldapLoginEnabled: true,
       serverDomain: 'http://test-server.com',
       emailLoginEnabled: 'true',
       registrationEnabled: 'true',

api/server/routes/ask/gptPlugins.js

@@ -106,7 +106,11 @@ router.post(
     const pluginMap = new Map();
     const onAgentAction = async (action, runId) => {
       pluginMap.set(runId, action.tool);
-      sendIntermediateMessage(res, { plugins });
+      sendIntermediateMessage(res, {
+        plugins,
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
     };
 
     const onToolStart = async (tool, input, runId, parentRunId) => {
@@ -124,7 +128,11 @@ router.post(
       }
       const extraTokens = ':::plugin:::\n';
       plugins.push(latestPlugin);
-      sendIntermediateMessage(res, { plugins }, extraTokens);
+      sendIntermediateMessage(
+        res,
+        { plugins, parentMessageId: userMessage.messageId, messageId: responseMessageId },
+        extraTokens,
+      );
     };
 
     const onToolEnd = async (output, runId) => {
@@ -142,7 +150,11 @@ router.post(
 
     const onChainEnd = () => {
       saveMessage({ ...userMessage, user });
-      sendIntermediateMessage(res, { plugins });
+      sendIntermediateMessage(res, {
+        plugins,
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
     };
 
     const getAbortData = () => ({
@@ -174,12 +186,13 @@ router.post(
         onStart,
         getPartialText,
         ...endpointOption,
-        onProgress: progressCallback.call(null, {
+        progressCallback,
+        progressOptions: {
           res,
           text,
-          parentMessageId: overrideParentMessageId || userMessageId,
+          // parentMessageId: overrideParentMessageId || userMessageId,
           plugins,
-        }),
+        },
         abortController,
       });
 

api/server/routes/assistants/actions.js

@@ -1,10 +1,10 @@
 const { v4 } = require('uuid');
 const express = require('express');
-const { actionDelimiter } = require('librechat-data-provider');
-const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { encryptMetadata, domainParser } = require('~/server/services/ActionService');
+const { actionDelimiter, EModelEndpoint } = require('librechat-data-provider');
+const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
 const { updateAction, getActions, deleteAction } = require('~/models/Action');
-const { updateAssistant, getAssistant } = require('~/models/Assistant');
+const { updateAssistantDoc, getAssistant } = require('~/models/Assistant');
 const { logger } = require('~/config');
 
 const router = express.Router();
@@ -45,8 +45,7 @@ router.post('/:assistant_id', async (req, res) => {
     let metadata = encryptMetadata(_metadata);
 
     let { domain } = metadata;
-    /* Azure doesn't support periods in function names */
-    domain = domainParser(req, domain, true);
+    domain = await domainParser(req, domain, true);
 
     if (!domain) {
       return res.status(400).json({ message: 'No domain provided' });
@@ -55,8 +54,7 @@ router.post('/:assistant_id', async (req, res) => {
     const action_id = _action_id ?? v4();
     const initialPromises = [];
 
-    /** @type {{ openai: OpenAI }} */
-    const { openai } = await initializeClient({ req, res });
+    const { openai } = await getOpenAIClient({ req, res });
 
     initialPromises.push(getAssistant({ assistant_id }));
     initialPromises.push(openai.beta.assistants.retrieve(assistant_id));
@@ -108,9 +106,10 @@ router.post('/:assistant_id', async (req, res) => {
         })),
       );
 
+    let updatedAssistant = await openai.beta.assistants.update(assistant_id, { tools });
     const promises = [];
     promises.push(
-      updateAssistant(
+      updateAssistantDoc(
         { assistant_id },
         {
           actions,
@@ -118,18 +117,26 @@ router.post('/:assistant_id', async (req, res) => {
         },
       ),
     );
-    promises.push(openai.beta.assistants.update(assistant_id, { tools }));
     promises.push(updateAction({ action_id }, { metadata, assistant_id, user: req.user.id }));
 
-    /** @type {[AssistantDocument, Assistant, Action]} */
-    const resolved = await Promise.all(promises);
+    /** @type {[AssistantDocument, Action]} */
+    let [assistantDocument, updatedAction] = await Promise.all(promises);
     const sensitiveFields = ['api_key', 'oauth_client_id', 'oauth_client_secret'];
     for (let field of sensitiveFields) {
-      if (resolved[2].metadata[field]) {
-        delete resolved[2].metadata[field];
+      if (updatedAction.metadata[field]) {
+        delete updatedAction.metadata[field];
       }
     }
-    res.json(resolved);
+
+    /* Map Azure OpenAI model to the assistant as defined by config */
+    if (req.app.locals[EModelEndpoint.azureOpenAI]?.assistants) {
+      updatedAssistant = {
+        ...updatedAssistant,
+        model: req.body.model,
+      };
+    }
+
+    res.json([assistantDocument, updatedAssistant, updatedAction]);
   } catch (error) {
     const message = 'Trouble updating the Assistant Action';
     logger.error(message, error);
@@ -148,9 +155,7 @@ router.delete('/:assistant_id/:action_id/:model', async (req, res) => {
   try {
     const { assistant_id, action_id, model } = req.params;
     req.body.model = model;
-
-    /** @type {{ openai: OpenAI }} */
-    const { openai } = await initializeClient({ req, res });
+    const { openai } = await getOpenAIClient({ req, res });
 
     const initialPromises = [];
     initialPromises.push(getAssistant({ assistant_id }));
@@ -171,15 +176,17 @@ router.delete('/:assistant_id/:action_id/:model', async (req, res) => {
       return true;
     });
 
-    domain = domainParser(req, domain, true);
+    domain = await domainParser(req, domain, true);
 
     const updatedTools = tools.filter(
       (tool) => !(tool.function && tool.function.name.includes(domain)),
     );
 
+    await openai.beta.assistants.update(assistant_id, { tools: updatedTools });
+
     const promises = [];
     promises.push(
-      updateAssistant(
+      updateAssistantDoc(
         { assistant_id },
         {
           actions: updatedActions,
@@ -187,7 +194,6 @@ router.delete('/:assistant_id/:action_id/:model', async (req, res) => {
         },
       ),
     );
-    promises.push(openai.beta.assistants.update(assistant_id, { tools: updatedTools }));
     promises.push(deleteAction({ action_id }));
 
     await Promise.all(promises);

api/server/routes/assistants/chatV1.js

@@ -0,0 +1,26 @@
+const express = require('express');
+
+const router = express.Router();
+const {
+  setHeaders,
+  handleAbort,
+  validateModel,
+  // validateEndpoint,
+  buildEndpointOption,
+} = require('~/server/middleware');
+const validateAssistant = require('~/server/middleware/assistants/validate');
+const chatController = require('~/server/controllers/assistants/chatV1');
+
+router.post('/abort', handleAbort());
+
+/**
+ * @route POST /
+ * @desc Chat with an assistant
+ * @access Public
+ * @param {express.Request} req - The request object, containing the request data.
+ * @param {express.Response} res - The response object, used to send back a response.
+ * @returns {void}
+ */
+router.post('/', validateModel, buildEndpointOption, validateAssistant, setHeaders, chatController);
+
+module.exports = router;

api/server/routes/assistants/chatV2.js

@@ -0,0 +1,26 @@
+const express = require('express');
+
+const router = express.Router();
+const {
+  setHeaders,
+  handleAbort,
+  validateModel,
+  // validateEndpoint,
+  buildEndpointOption,
+} = require('~/server/middleware');
+const validateAssistant = require('~/server/middleware/assistants/validate');
+const chatController = require('~/server/controllers/assistants/chatV2');
+
+router.post('/abort', handleAbort());
+
+/**
+ * @route POST /
+ * @desc Chat with an assistant
+ * @access Public
+ * @param {express.Request} req - The request object, containing the request data.
+ * @param {express.Response} res - The response object, used to send back a response.
+ * @returns {void}
+ */
+router.post('/', validateModel, buildEndpointOption, validateAssistant, setHeaders, chatController);
+
+module.exports = router;