chore: add missing audio mimetype

* refactor: move audio encode over to TS

* refactor: audio encoding now functional in LC again

* refactor: move video encode over to TS

* refactor: move document encode over to TS

* refactor: video encoding now functional in LC again

* refactor: document encoding now functional in LC again

* fix: extend file type options in AttachFileMenu to include 'google_multimodal' and update dependency array to include agent?.provider

* feat: only accept pdfs if responses api is enabled for openai convos

api/server/controllers/agents/client.js

@@ -11,6 +11,9 @@ const {
   memoryInstructions,
   formatContentStrings,
   createMemoryProcessor,
+  encodeAndFormatAudios,
+  encodeAndFormatVideos,
+  encodeAndFormatDocuments,
 } = require('@librechat/api');
 const {
   Callback,
@@ -33,6 +36,7 @@ const {
   AgentCapabilities,
   bedrockInputSchema,
   removeNullishValues,
+  isDocumentSupportedEndpoint,
 } = require('librechat-data-provider');
 const { addCacheControl, createContextHandlers } = require('~/app/clients/prompts');
 const { initializeAgent } = require('~/server/services/Endpoints/agents/agent');
@@ -40,11 +44,13 @@ const { spendTokens, spendStructuredTokens } = require('~/models/spendTokens');
 const { getFormattedMemories, deleteMemory, setMemory } = require('~/models');
 const { encodeAndFormat } = require('~/server/services/Files/images/encode');
 const { getProviderConfig } = require('~/server/services/Endpoints');
+const { getStrategyFunctions } = require('~/server/services/Files');
 const { checkCapability } = require('~/server/services/Config');
 const BaseClient = require('~/app/clients/BaseClient');
 const { getRoleByName } = require('~/models/Role');
 const { loadAgent } = require('~/models/Agent');
 const { getMCPManager } = require('~/config');
+const { getFiles } = require('~/models');
 
 const omitTitleOptions = new Set([
   'stream',
@@ -222,6 +228,168 @@ class AgentClient extends BaseClient {
     return files;
   }
 
+  async addDocuments(message, attachments) {
+    const documentResult = await encodeAndFormatDocuments(
+      this.options.req,
+      attachments,
+      this.options.agent.provider,
+      getStrategyFunctions,
+    );
+    message.documents =
+      documentResult.documents && documentResult.documents.length
+        ? documentResult.documents
+        : undefined;
+    return documentResult.files;
+  }
+
+  async addVideos(message, attachments) {
+    const videoResult = await encodeAndFormatVideos(
+      this.options.req,
+      attachments,
+      this.options.agent.provider,
+      getStrategyFunctions,
+    );
+    message.videos =
+      videoResult.videos && videoResult.videos.length ? videoResult.videos : undefined;
+    return videoResult.files;
+  }
+
+  async addAudios(message, attachments) {
+    const audioResult = await encodeAndFormatAudios(
+      this.options.req,
+      attachments,
+      this.options.agent.provider,
+      getStrategyFunctions,
+    );
+    message.audios =
+      audioResult.audios && audioResult.audios.length ? audioResult.audios : undefined;
+    return audioResult.files;
+  }
+
+  /**
+   * Override addPreviousAttachments to handle all file types, not just images
+   * @param {TMessage[]} _messages
+   * @returns {Promise<TMessage[]>}
+   */
+  async addPreviousAttachments(_messages) {
+    if (!this.options.resendFiles) {
+      return _messages;
+    }
+
+    const seen = new Set();
+    const attachmentsProcessed =
+      this.options.attachments && !(this.options.attachments instanceof Promise);
+    if (attachmentsProcessed) {
+      for (const attachment of this.options.attachments) {
+        seen.add(attachment.file_id);
+      }
+    }
+
+    /**
+     *
+     * @param {TMessage} message
+     */
+    const processMessage = async (message) => {
+      if (!this.message_file_map) {
+        /** @type {Record<string, MongoFile[]> */
+        this.message_file_map = {};
+      }
+
+      const fileIds = [];
+      for (const file of message.files) {
+        if (seen.has(file.file_id)) {
+          continue;
+        }
+        fileIds.push(file.file_id);
+        seen.add(file.file_id);
+      }
+
+      if (fileIds.length === 0) {
+        return message;
+      }
+
+      const files = await getFiles(
+        {
+          file_id: { $in: fileIds },
+        },
+        {},
+        {},
+      );
+
+      await this.processAttachments(message, files);
+
+      this.message_file_map[message.messageId] = files;
+      return message;
+    };
+
+    const promises = [];
+
+    for (const message of _messages) {
+      if (!message.files) {
+        promises.push(message);
+        continue;
+      }
+
+      promises.push(processMessage(message));
+    }
+
+    const messages = await Promise.all(promises);
+
+    this.checkVisionRequest(Object.values(this.message_file_map ?? {}).flat());
+    return messages;
+  }
+
+  async processAttachments(message, attachments) {
+    const categorizedAttachments = {
+      images: [],
+      documents: [],
+      videos: [],
+      audios: [],
+    };
+
+    for (const file of attachments) {
+      if (file.type.startsWith('image/')) {
+        categorizedAttachments.images.push(file);
+      } else if (file.type === 'application/pdf') {
+        categorizedAttachments.documents.push(file);
+      } else if (file.type.startsWith('video/')) {
+        categorizedAttachments.videos.push(file);
+      } else if (file.type.startsWith('audio/')) {
+        categorizedAttachments.audios.push(file);
+      }
+    }
+
+    const [imageFiles, documentFiles, videoFiles, audioFiles] = await Promise.all([
+      categorizedAttachments.images.length > 0
+        ? this.addImageURLs(message, categorizedAttachments.images)
+        : Promise.resolve([]),
+      categorizedAttachments.documents.length > 0
+        ? this.addDocuments(message, categorizedAttachments.documents)
+        : Promise.resolve([]),
+      categorizedAttachments.videos.length > 0
+        ? this.addVideos(message, categorizedAttachments.videos)
+        : Promise.resolve([]),
+      categorizedAttachments.audios.length > 0
+        ? this.addAudios(message, categorizedAttachments.audios)
+        : Promise.resolve([]),
+    ]);
+
+    const allFiles = [...imageFiles, ...documentFiles, ...videoFiles, ...audioFiles];
+    const seenFileIds = new Set();
+    const uniqueFiles = [];
+
+    for (const file of allFiles) {
+      if (file.file_id && !seenFileIds.has(file.file_id)) {
+        seenFileIds.add(file.file_id);
+        uniqueFiles.push(file);
+      } else if (!file.file_id) {
+        uniqueFiles.push(file);
+      }
+    }
+
+    return uniqueFiles;
+  }
+
   async buildMessages(
     messages,
     parentMessageId,
@@ -255,7 +423,7 @@ class AgentClient extends BaseClient {
         };
       }
 
-      const files = await this.addImageURLs(
+      const files = await this.processAttachments(
         orderedMessages[orderedMessages.length - 1],
         attachments,
       );
@@ -278,6 +446,47 @@ class AgentClient extends BaseClient {
         assistantName: this.options?.modelLabel,
       });
 
+      const hasFiles =
+        (message.documents && message.documents.length > 0) ||
+        (message.videos && message.videos.length > 0) ||
+        (message.audios && message.audios.length > 0) ||
+        (message.image_urls && message.image_urls.length > 0);
+
+      if (
+        hasFiles &&
+        message.isCreatedByUser &&
+        isDocumentSupportedEndpoint(this.options.agent.provider)
+      ) {
+        const contentParts = [];
+
+        if (message.documents && message.documents.length > 0) {
+          contentParts.push(...message.documents);
+        }
+
+        if (message.videos && message.videos.length > 0) {
+          contentParts.push(...message.videos);
+        }
+
+        if (message.audios && message.audios.length > 0) {
+          contentParts.push(...message.audios);
+        }
+
+        if (message.image_urls && message.image_urls.length > 0) {
+          contentParts.push(...message.image_urls);
+        }
+
+        if (typeof formattedMessage.content === 'string') {
+          contentParts.push({ type: 'text', text: formattedMessage.content });
+        } else {
+          const textPart = formattedMessage.content.find((part) => part.type === 'text');
+          if (textPart) {
+            contentParts.push(textPart);
+          }
+        }
+
+        formattedMessage.content = contentParts;
+      }
+
       if (message.ocr && i !== orderedMessages.length - 1) {
         if (typeof formattedMessage.content === 'string') {
           formattedMessage.content = message.ocr + '\n' + formattedMessage.content;
@@ -793,6 +1002,7 @@ class AgentClient extends BaseClient {
       };
 
       const toolSet = new Set((this.options.agent.tools ?? []).map((tool) => tool && tool.name));
+
       let { messages: initialMessages, indexTokenCountMap } = formatAgentMessages(
         payload,
         this.indexTokenCountMap,

api/server/services/Files/Local/crud.js

@@ -4,6 +4,7 @@ const axios = require('axios');
 const { logger } = require('@librechat/data-schemas');
 const { EModelEndpoint } = require('librechat-data-provider');
 const { generateShortLivedToken } = require('@librechat/api');
+const { resizeImageBuffer } = require('~/server/services/Files/images/resize');
 const { getBufferMetadata } = require('~/server/utils');
 const paths = require('~/config/paths');
 
@@ -286,7 +287,18 @@ async function uploadLocalFile({ req, file, file_id }) {
   await fs.promises.writeFile(newPath, inputBuffer);
   const filepath = path.posix.join('/', 'uploads', req.user.id, path.basename(newPath));
 
-  return { filepath, bytes };
+  let height, width;
+  if (file.mimetype && file.mimetype.startsWith('image/')) {
+    try {
+      const { width: imgWidth, height: imgHeight } = await resizeImageBuffer(inputBuffer, 'high');
+      height = imgHeight;
+      width = imgWidth;
+    } catch (error) {
+      logger.warn('[uploadLocalFile] Could not get image dimensions:', error.message);
+    }
+  }
+
+  return { filepath, bytes, height, width };
 }
 
 /**

api/server/services/Files/index.js

@@ -2,11 +2,13 @@ const { processCodeFile } = require('./Code/process');
 const { processFileUpload } = require('./process');
 const { uploadImageBuffer } = require('./images');
 const { hasAccessToFilesViaAgent, filterFilesByAgentAccess } = require('./permissions');
+const { getStrategyFunctions } = require('./strategies');
 
 module.exports = {
   processCodeFile,
   processFileUpload,
   uploadImageBuffer,
+  getStrategyFunctions,
   hasAccessToFilesViaAgent,
   filterFilesByAgentAccess,
 };

api/server/services/Files/process.js

@@ -419,11 +419,11 @@ const processFileUpload = async ({ req, res, metadata }) => {
   const isAssistantUpload = isAssistantsEndpoint(metadata.endpoint);
   const assistantSource =
     metadata.endpoint === EModelEndpoint.azureAssistants ? FileSources.azure : FileSources.openai;
+
   // Use the configured file strategy for regular file uploads (not vectordb)
   const source = isAssistantUpload ? assistantSource : appConfig.fileStrategy;
   const { handleFileUpload } = getStrategyFunctions(source);
   const { file_id, temp_file_id = null } = metadata;
-
   /** @type {OpenAI | undefined} */
   let openai;
   if (checkOpenAIStorage(source)) {

client/src/common/agents-types.ts

@@ -21,6 +21,7 @@ export type TAgentCapabilities = {
   [AgentCapabilities.execute_code]: boolean;
   [AgentCapabilities.end_after_tools]?: boolean;
   [AgentCapabilities.hide_sequential_outputs]?: boolean;
+  [AgentCapabilities.direct_attach]?: boolean;
 };
 
 export type AgentForm = {

client/src/components/Chat/Input/Files/AttachFileChat.tsx

@@ -36,6 +36,7 @@ function AttachFileChat({ disableInputs }: { disableInputs: boolean }) {
         disabled={disableInputs}
         conversationId={conversationId}
         endpointFileConfig={endpointFileConfig}
+        endpoint={endpoint}
       />
     );
   }

client/src/components/Chat/Input/Files/AttachFileMenu.tsx

@@ -1,8 +1,19 @@
 import React, { useRef, useState, useMemo } from 'react';
 import * as Ariakit from '@ariakit/react';
 import { useSetRecoilState } from 'recoil';
-import { FileSearch, ImageUpIcon, TerminalSquareIcon, FileType2Icon } from 'lucide-react';
-import { EToolResources, EModelEndpoint, defaultAgentCapabilities } from 'librechat-data-provider';
+import {
+  FileSearch,
+  ImageUpIcon,
+  TerminalSquareIcon,
+  FileType2Icon,
+  FileImageIcon,
+} from 'lucide-react';
+import {
+  EToolResources,
+  EModelEndpoint,
+  defaultAgentCapabilities,
+  isDocumentSupportedEndpoint,
+} from 'librechat-data-provider';
 import {
   FileUpload,
   TooltipAnchor,
@@ -14,8 +25,9 @@ import type { EndpointFileConfig } from 'librechat-data-provider';
 import { useLocalize, useGetAgentsConfig, useFileHandling, useAgentCapabilities } from '~/hooks';
 import useSharePointFileHandling from '~/hooks/Files/useSharePointFileHandling';
 import { SharePointPickerDialog } from '~/components/SharePoint';
-import { useGetStartupConfig } from '~/data-provider';
+import { useGetStartupConfig, useGetAgentByIdQuery } from '~/data-provider';
 import { ephemeralAgentByConvoId } from '~/store';
+import { useChatContext } from '~/Providers/ChatContext';
 import { MenuItemProps } from '~/common';
 import { cn } from '~/utils';
 
@@ -23,9 +35,15 @@ interface AttachFileMenuProps {
   conversationId: string;
   disabled?: boolean | null;
   endpointFileConfig?: EndpointFileConfig;
+  endpoint?: string | null;
 }
 
-const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: AttachFileMenuProps) => {
+const AttachFileMenu = ({
+  disabled,
+  conversationId,
+  endpointFileConfig,
+  endpoint,
+}: AttachFileMenuProps) => {
   const localize = useLocalize();
   const isUploadDisabled = disabled ?? false;
   const inputRef = useRef<HTMLInputElement>(null);
@@ -46,34 +64,79 @@ const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: Attach
 
   const [isSharePointDialogOpen, setIsSharePointDialogOpen] = useState(false);
   const { agentsConfig } = useGetAgentsConfig();
+  const { conversation } = useChatContext();
+
+  // Get agent details if using an agent
+  const { data: agent } = useGetAgentByIdQuery(conversation?.agent_id ?? '', {
+    enabled: !!conversation?.agent_id && conversation?.agent_id !== 'ephemeral',
+  });
+
   /** TODO: Ephemeral Agent Capabilities
    * Allow defining agent capabilities on a per-endpoint basis
    * Use definition for agents endpoint for ephemeral agents
    * */
   const capabilities = useAgentCapabilities(agentsConfig?.capabilities ?? defaultAgentCapabilities);
 
-  const handleUploadClick = (isImage?: boolean) => {
+  const handleUploadClick = (
+    fileType?: 'image' | 'document' | 'multimodal' | 'google_multimodal',
+  ) => {
     if (!inputRef.current) {
       return;
     }
     inputRef.current.value = '';
-    inputRef.current.accept = isImage === true ? 'image/*' : '';
+    if (fileType === 'image') {
+      inputRef.current.accept = 'image/*';
+    } else if (fileType === 'document') {
+      inputRef.current.accept = '.pdf,application/pdf';
+    } else if (fileType === 'multimodal') {
+      inputRef.current.accept = 'image/*,.pdf,application/pdf';
+    } else if (fileType === 'google_multimodal') {
+      inputRef.current.accept = 'image/*,.pdf,application/pdf,video/*,audio/*';
+    } else {
+      inputRef.current.accept = '';
+    }
     inputRef.current.click();
     inputRef.current.accept = '';
   };
 
   const dropdownItems = useMemo(() => {
-    const createMenuItems = (onAction: (isImage?: boolean) => void) => {
-      const items: MenuItemProps[] = [
-        {
+    const createMenuItems = (
+      onAction: (fileType?: 'image' | 'document' | 'multimodal' | 'google_multimodal') => void,
+    ) => {
+      const items: MenuItemProps[] = [];
+
+      const currentProvider = agent?.provider ?? endpoint;
+      const isOpenAIOrAzure =
+        currentProvider === EModelEndpoint.openAI || currentProvider === EModelEndpoint.azureOpenAI;
+      const useResponsesApiEnabled = conversation?.useResponsesApi ?? false;
+
+      const shouldShowDirectAttach =
+        isDocumentSupportedEndpoint(currentProvider) &&
+        (!isOpenAIOrAzure || useResponsesApiEnabled);
+
+      if (shouldShowDirectAttach) {
+        items.push({
+          label: localize('com_ui_upload_provider'),
+          onClick: () => {
+            setToolResource(EToolResources.direct_attach);
+            onAction(
+              (agent?.provider ?? endpoint) === EModelEndpoint.google
+                ? 'google_multimodal'
+                : 'multimodal',
+            );
+          },
+          icon: <FileImageIcon className="icon-md" />,
+        });
+      } else {
+        items.push({
           label: localize('com_ui_upload_image_input'),
           onClick: () => {
             setToolResource(undefined);
-            onAction(true);
+            onAction('image');
           },
           icon: <ImageUpIcon className="icon-md" />,
-        },
-      ];
+        });
+      }
 
       if (capabilities.ocrEnabled) {
         items.push({
@@ -139,6 +202,8 @@ const AttachFileMenu = ({ disabled, conversationId, endpointFileConfig }: Attach
     setEphemeralAgent,
     sharePointEnabled,
     setIsSharePointDialogOpen,
+    endpoint,
+    agent?.provider,
   ]);
 
   const menuTrigger = (

client/src/hooks/Agents/useAgentCapabilities.ts

@@ -9,6 +9,7 @@ interface AgentCapabilitiesResult {
   fileSearchEnabled: boolean;
   webSearchEnabled: boolean;
   codeEnabled: boolean;
+  directAttachEnabled: boolean;
 }
 
 export default function useAgentCapabilities(
@@ -49,6 +50,11 @@ export default function useAgentCapabilities(
     [capabilities],
   );
 
+  const directAttachEnabled = useMemo(
+    () => capabilities?.includes(AgentCapabilities.direct_attach) ?? false,
+    [capabilities],
+  );
+
   return {
     ocrEnabled,
     codeEnabled,
@@ -57,5 +63,6 @@ export default function useAgentCapabilities(
     artifactsEnabled,
     webSearchEnabled,
     fileSearchEnabled,
+    directAttachEnabled,
   };
 }

client/src/locales/en/translation.json

@@ -1218,6 +1218,7 @@
   "com_ui_upload_invalid": "Invalid file for upload. Must be an image not exceeding the limit",
   "com_ui_upload_invalid_var": "Invalid file for upload. Must be an image not exceeding {{0}} MB",
   "com_ui_upload_ocr_text": "Upload as Text",
+  "com_ui_upload_provider": "Upload to Provider",
   "com_ui_upload_success": "Successfully uploaded file",
   "com_ui_upload_type": "Select Upload Type",
   "com_ui_usage": "Usage",

client/src/utils/files.ts

@@ -1,4 +1,11 @@
-import { SheetPaths, TextPaths, FilePaths, CodePaths } from '@librechat/client';
+import {
+  SheetPaths,
+  TextPaths,
+  FilePaths,
+  CodePaths,
+  AudioPaths,
+  VideoPaths,
+} from '@librechat/client';
 import {
   megabyte,
   QueryKeys,
@@ -38,6 +45,18 @@ const artifact = {
   title: 'Code',
 };
 
+const audioFile = {
+  paths: AudioPaths,
+  fill: '#FF6B35',
+  title: 'Audio',
+};
+
+const videoFile = {
+  paths: VideoPaths,
+  fill: '#8B5CF6',
+  title: 'Video',
+};
+
 export const fileTypes = {
   /* Category matches */
   file: {
@@ -47,6 +66,8 @@ export const fileTypes = {
   },
   text: textDocument,
   txt: textDocument,
+  audio: audioFile,
+  video: videoFile,
   // application:,
 
   /* Partial matches */

packages/api/src/files/audio/encode.ts

@@ -0,0 +1,116 @@
+import { Readable } from 'stream';
+import getStream from 'get-stream';
+import { EModelEndpoint, isDocumentSupportedEndpoint } from 'librechat-data-provider';
+import type { IMongoFile } from '@librechat/data-schemas';
+import type { Request } from 'express';
+import { validateAudio } from '~/files/validation';
+
+interface StrategyFunctions {
+  getDownloadStream: (req: Request, filepath: string) => Promise<Readable>;
+}
+
+interface AudioResult {
+  audios: Array<{
+    type: string;
+    mimeType: string;
+    data: string;
+  }>;
+  files: Array<{
+    file_id?: string;
+    temp_file_id?: string;
+    filepath: string;
+    source?: string;
+    filename: string;
+    type: string;
+  }>;
+}
+
+/**
+ * Encodes and formats audio files for different endpoints
+ * @param req - The request object
+ * @param files - Array of audio files
+ * @param endpoint - The endpoint to format for (currently only google is supported)
+ * @returns Promise that resolves to audio and file metadata
+ */
+export async function encodeAndFormatAudios(
+  req: Request,
+  files: IMongoFile[],
+  endpoint: EModelEndpoint,
+  getStrategyFunctions: (source: string) => StrategyFunctions,
+): Promise<AudioResult> {
+  if (!files?.length) {
+    return { audios: [], files: [] };
+  }
+
+  const encodingMethods: Record<string, StrategyFunctions> = {};
+  const result: AudioResult = { audios: [], files: [] };
+
+  const processFile = async (file: IMongoFile) => {
+    if (!file?.filepath) return null;
+
+    const source = file.source ?? 'local';
+    if (!encodingMethods[source]) {
+      encodingMethods[source] = getStrategyFunctions(source);
+    }
+
+    const { getDownloadStream } = encodingMethods[source];
+    const stream = await getDownloadStream(req, file.filepath);
+    const buffer = await getStream.buffer(stream);
+
+    return {
+      file,
+      content: buffer.toString('base64'),
+      metadata: {
+        file_id: file.file_id,
+        temp_file_id: file.temp_file_id,
+        filepath: file.filepath,
+        source: file.source,
+        filename: file.filename,
+        type: file.type,
+      },
+    };
+  };
+
+  const results = await Promise.allSettled(files.map(processFile));
+
+  for (const settledResult of results) {
+    if (settledResult.status === 'rejected') {
+      console.error('Audio processing failed:', settledResult.reason);
+      continue;
+    }
+
+    const processed = settledResult.value;
+    if (!processed) continue;
+
+    const { file, content, metadata } = processed;
+
+    if (!content || !file) {
+      if (metadata) result.files.push(metadata);
+      continue;
+    }
+
+    if (!file.type.startsWith('audio/') || !isDocumentSupportedEndpoint(endpoint)) {
+      result.files.push(metadata);
+      continue;
+    }
+
+    const audioBuffer = Buffer.from(content, 'base64');
+    const validation = await validateAudio(audioBuffer, audioBuffer.length, endpoint);
+
+    if (!validation.isValid) {
+      throw new Error(`Audio validation failed: ${validation.error}`);
+    }
+
+    if (endpoint === EModelEndpoint.google) {
+      result.audios.push({
+        type: 'audio',
+        mimeType: file.type,
+        data: content,
+      });
+    }
+
+    result.files.push(metadata);
+  }
+
+  return result;
+}

packages/api/src/files/document/encode.ts

@@ -0,0 +1,150 @@
+import { EModelEndpoint, isDocumentSupportedEndpoint } from 'librechat-data-provider';
+import { validatePdf } from '@librechat/api';
+import getStream from 'get-stream';
+import type { Request } from 'express';
+import type { IMongoFile } from '@librechat/data-schemas';
+import { Readable } from 'stream';
+
+interface StrategyFunctions {
+  getDownloadStream: (req: Request, filepath: string) => Promise<Readable>;
+}
+
+interface DocumentResult {
+  documents: Array<{
+    type: string;
+    source?: {
+      type: string;
+      media_type: string;
+      data: string;
+    };
+    cache_control?: { type: string };
+    citations?: { enabled: boolean };
+    filename?: string;
+    file_data?: string;
+    mimeType?: string;
+    data?: string;
+  }>;
+  files: Array<{
+    file_id?: string;
+    temp_file_id?: string;
+    filepath: string;
+    source?: string;
+    filename: string;
+    type: string;
+  }>;
+}
+
+/**
+ * Processes and encodes document files for various endpoints
+ * @param req - Express request object
+ * @param files - Array of file objects to process
+ * @param endpoint - The endpoint identifier (e.g., EModelEndpoint.anthropic)
+ * @param getStrategyFunctions - Function to get strategy functions
+ * @returns Promise that resolves to documents and file metadata
+ */
+export async function encodeAndFormatDocuments(
+  req: Request,
+  files: IMongoFile[],
+  endpoint: EModelEndpoint,
+  getStrategyFunctions: (source: string) => StrategyFunctions,
+): Promise<DocumentResult> {
+  if (!files?.length) {
+    return { documents: [], files: [] };
+  }
+
+  const encodingMethods: Record<string, StrategyFunctions> = {};
+  const result: DocumentResult = { documents: [], files: [] };
+
+  const documentFiles = files.filter(
+    (file) => file.type === 'application/pdf' || file.type?.startsWith('application/'),
+  );
+
+  if (!documentFiles.length) {
+    return result;
+  }
+
+  const processFile = async (file: IMongoFile) => {
+    if (file.type !== 'application/pdf' || !isDocumentSupportedEndpoint(endpoint)) {
+      return null;
+    }
+
+    const source = file.source ?? 'local';
+    if (!encodingMethods[source]) {
+      encodingMethods[source] = getStrategyFunctions(source);
+    }
+
+    const { getDownloadStream } = encodingMethods[source];
+    const stream = await getDownloadStream(req, file.filepath);
+    const buffer = await getStream.buffer(stream);
+
+    return {
+      file,
+      content: buffer.toString('base64'),
+      metadata: {
+        file_id: file.file_id,
+        temp_file_id: file.temp_file_id,
+        filepath: file.filepath,
+        source: file.source,
+        filename: file.filename,
+        type: file.type,
+      },
+    };
+  };
+
+  const results = await Promise.allSettled(documentFiles.map(processFile));
+
+  for (const settledResult of results) {
+    if (settledResult.status === 'rejected') {
+      console.error('Document processing failed:', settledResult.reason);
+      continue;
+    }
+
+    const processed = settledResult.value;
+    if (!processed) continue;
+
+    const { file, content, metadata } = processed;
+
+    if (!content || !file) {
+      if (metadata) result.files.push(metadata);
+      continue;
+    }
+
+    if (file.type === 'application/pdf' && isDocumentSupportedEndpoint(endpoint)) {
+      const pdfBuffer = Buffer.from(content, 'base64');
+      const validation = await validatePdf(pdfBuffer, pdfBuffer.length, endpoint);
+
+      if (!validation.isValid) {
+        throw new Error(`PDF validation failed: ${validation.error}`);
+      }
+
+      if (endpoint === EModelEndpoint.anthropic) {
+        result.documents.push({
+          type: 'document',
+          source: {
+            type: 'base64',
+            media_type: 'application/pdf',
+            data: content,
+          },
+          cache_control: { type: 'ephemeral' },
+          citations: { enabled: true },
+        });
+      } else if (endpoint === EModelEndpoint.openAI) {
+        result.documents.push({
+          type: 'input_file',
+          filename: file.filename,
+          file_data: `data:application/pdf;base64,${content}`,
+        });
+      } else if (endpoint === EModelEndpoint.google) {
+        result.documents.push({
+          type: 'document',
+          mimeType: 'application/pdf',
+          data: content,
+        });
+      }
+
+      result.files.push(metadata);
+    }
+  }
+
+  return result;
+}

packages/api/src/files/index.ts

@@ -2,3 +2,7 @@ export * from './mistral/crud';
 export * from './audio';
 export * from './text';
 export * from './parse';
+export * from './validation';
+export * from './audio/encode';
+export * from './video/encode';
+export * from './document/encode';

packages/api/src/files/validation.ts

@@ -0,0 +1,185 @@
+import { anthropicPdfSizeLimit, EModelEndpoint } from 'librechat-data-provider';
+
+export interface PDFValidationResult {
+  isValid: boolean;
+  error?: string;
+}
+
+export interface VideoValidationResult {
+  isValid: boolean;
+  error?: string;
+}
+
+export interface AudioValidationResult {
+  isValid: boolean;
+  error?: string;
+}
+
+export async function validatePdf(
+  pdfBuffer: Buffer,
+  fileSize: number,
+  endpoint: EModelEndpoint,
+): Promise<PDFValidationResult> {
+  if (endpoint === EModelEndpoint.anthropic) {
+    return validateAnthropicPdf(pdfBuffer, fileSize);
+  }
+
+  if (endpoint === EModelEndpoint.openAI || endpoint === EModelEndpoint.azureOpenAI) {
+    return validateOpenAIPdf(fileSize);
+  }
+
+  if (endpoint === EModelEndpoint.google) {
+    return validateGooglePdf(fileSize);
+  }
+
+  return { isValid: true };
+}
+
+/**
+ * Validates if a PDF meets Anthropic's requirements
+ * @param pdfBuffer - The PDF file as a buffer
+ * @param fileSize - The file size in bytes
+ * @returns Promise that resolves to validation result
+ */
+async function validateAnthropicPdf(
+  pdfBuffer: Buffer,
+  fileSize: number,
+): Promise<PDFValidationResult> {
+  try {
+    if (fileSize > anthropicPdfSizeLimit) {
+      return {
+        isValid: false,
+        error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Anthropic's 32MB limit`,
+      };
+    }
+
+    if (!pdfBuffer || pdfBuffer.length < 5) {
+      return {
+        isValid: false,
+        error: 'Invalid PDF file: too small or corrupted',
+      };
+    }
+
+    const pdfHeader = pdfBuffer.subarray(0, 5).toString();
+    if (!pdfHeader.startsWith('%PDF-')) {
+      return {
+        isValid: false,
+        error: 'Invalid PDF file: missing PDF header',
+      };
+    }
+
+    const pdfContent = pdfBuffer.toString('binary');
+    if (
+      pdfContent.includes('/Encrypt ') ||
+      pdfContent.includes('/U (') ||
+      pdfContent.includes('/O (')
+    ) {
+      return {
+        isValid: false,
+        error: 'PDF is password-protected or encrypted. Anthropic requires unencrypted PDFs.',
+      };
+    }
+
+    const pageMatches = pdfContent.match(/\/Type[\s]*\/Page[^s]/g);
+    const estimatedPages = pageMatches ? pageMatches.length : 1;
+
+    if (estimatedPages > 100) {
+      return {
+        isValid: false,
+        error: `PDF has approximately ${estimatedPages} pages, exceeding Anthropic's 100-page limit`,
+      };
+    }
+
+    return { isValid: true };
+  } catch (error) {
+    console.error('PDF validation error:', error);
+    return {
+      isValid: false,
+      error: 'Failed to validate PDF file',
+    };
+  }
+}
+
+async function validateOpenAIPdf(fileSize: number): Promise<PDFValidationResult> {
+  if (fileSize > 10 * 1024 * 1024) {
+    return {
+      isValid: false,
+      error: "PDF file size exceeds OpenAI's 10MB limit",
+    };
+  }
+
+  return { isValid: true };
+}
+
+async function validateGooglePdf(fileSize: number): Promise<PDFValidationResult> {
+  if (fileSize > 20 * 1024 * 1024) {
+    return {
+      isValid: false,
+      error: "PDF file size exceeds Google's 20MB limit",
+    };
+  }
+
+  return { isValid: true };
+}
+
+/**
+ * Validates video files for different endpoints
+ * @param videoBuffer - The video file as a buffer
+ * @param fileSize - The file size in bytes
+ * @param endpoint - The endpoint to validate for
+ * @returns Promise that resolves to validation result
+ */
+export async function validateVideo(
+  videoBuffer: Buffer,
+  fileSize: number,
+  endpoint: EModelEndpoint,
+): Promise<VideoValidationResult> {
+  if (endpoint === EModelEndpoint.google) {
+    if (fileSize > 20 * 1024 * 1024) {
+      return {
+        isValid: false,
+        error: `Video file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Google's 20MB limit`,
+      };
+    }
+  }
+
+  if (!videoBuffer || videoBuffer.length < 10) {
+    return {
+      isValid: false,
+      error: 'Invalid video file: too small or corrupted',
+    };
+  }
+
+  return { isValid: true };
+}
+
+/**
+ * Validates audio files for different endpoints
+ * @param audioBuffer - The audio file as a buffer
+ * @param fileSize - The file size in bytes
+ * @param endpoint - The endpoint to validate for
+ * @returns Promise that resolves to validation result
+ */
+export async function validateAudio(
+  audioBuffer: Buffer,
+  fileSize: number,
+  endpoint: EModelEndpoint,
+): Promise<AudioValidationResult> {
+  if (endpoint === EModelEndpoint.google) {
+    if (fileSize > 20 * 1024 * 1024) {
+      return {
+        isValid: false,
+        error: `Audio file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Google's 20MB limit`,
+      };
+    }
+  }
+
+  if (!audioBuffer || audioBuffer.length < 10) {
+    return {
+      isValid: false,
+      error: 'Invalid audio file: too small or corrupted',
+    };
+  }
+
+  return { isValid: true };
+}

packages/api/src/files/video/encode.ts

@@ -0,0 +1,117 @@
+import { EModelEndpoint, isDocumentSupportedEndpoint } from 'librechat-data-provider';
+import { validateVideo } from '@librechat/api';
+import getStream from 'get-stream';
+import type { Request } from 'express';
+import type { IMongoFile } from '@librechat/data-schemas';
+import { Readable } from 'stream';
+
+interface StrategyFunctions {
+  getDownloadStream: (req: Request, filepath: string) => Promise<Readable>;
+}
+
+interface VideoResult {
+  videos: Array<{
+    type: string;
+    mimeType: string;
+    data: string;
+  }>;
+  files: Array<{
+    file_id?: string;
+    temp_file_id?: string;
+    filepath: string;
+    source?: string;
+    filename: string;
+    type: string;
+  }>;
+}
+
+/**
+ * Encodes and formats video files for different endpoints
+ * @param req - The request object
+ * @param files - Array of video files
+ * @param endpoint - The endpoint to format for
+ * @param getStrategyFunctions - Function to get strategy functions
+ * @returns Promise that resolves to videos and file metadata
+ */
+export async function encodeAndFormatVideos(
+  req: Request,
+  files: IMongoFile[],
+  endpoint: EModelEndpoint,
+  getStrategyFunctions: (source: string) => StrategyFunctions,
+): Promise<VideoResult> {
+  if (!files?.length) {
+    return { videos: [], files: [] };
+  }
+
+  const encodingMethods: Record<string, StrategyFunctions> = {};
+  const result: VideoResult = { videos: [], files: [] };
+
+  const processFile = async (file: IMongoFile) => {
+    if (!file?.filepath) return null;
+
+    const source = file.source ?? 'local';
+    if (!encodingMethods[source]) {
+      encodingMethods[source] = getStrategyFunctions(source);
+    }
+
+    const { getDownloadStream } = encodingMethods[source];
+    const stream = await getDownloadStream(req, file.filepath);
+    const buffer = await getStream.buffer(stream);
+
+    return {
+      file,
+      content: buffer.toString('base64'),
+      metadata: {
+        file_id: file.file_id,
+        temp_file_id: file.temp_file_id,
+        filepath: file.filepath,
+        source: file.source,
+        filename: file.filename,
+        type: file.type,
+      },
+    };
+  };
+
+  const results = await Promise.allSettled(files.map(processFile));
+
+  for (const settledResult of results) {
+    if (settledResult.status === 'rejected') {
+      console.error('Video processing failed:', settledResult.reason);
+      continue;
+    }
+
+    const processed = settledResult.value;
+    if (!processed) continue;
+
+    const { file, content, metadata } = processed;
+
+    if (!content || !file) {
+      if (metadata) result.files.push(metadata);
+      continue;
+    }
+
+    if (!file.type.startsWith('video/') || !isDocumentSupportedEndpoint(endpoint)) {
+      result.files.push(metadata);
+      continue;
+    }
+
+    const videoBuffer = Buffer.from(content, 'base64');
+    const validation = await validateVideo(videoBuffer, videoBuffer.length, endpoint);
+
+    if (!validation.isValid) {
+      throw new Error(`Video validation failed: ${validation.error}`);
+    }
+
+    if (endpoint === EModelEndpoint.google) {
+      result.videos.push({
+        type: 'video',
+        mimeType: file.type,
+        data: content,
+      });
+    }
+
+    result.files.push(metadata);
+  }
+
+  return result;
+}

packages/client/src/svgs/AudioPaths.tsx

@@ -0,0 +1,41 @@
+export default function AudioPaths() {
+  return (
+    <>
+      <path
+        d="M8 15v6"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M13 8v20"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M18 10v16"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M23 6v24"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M28 12v12"
+        stroke="white"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </>
+  );
+}

packages/client/src/svgs/VideoPaths.tsx

@@ -0,0 +1,10 @@
+export default function VideoPaths() {
+  return (
+    <>
+      {/* Video container - rounded rectangle (not filled) */}
+      <rect x="8" y="10" width="20" height="16" rx="3" stroke="white" strokeWidth="2" fill="none" />
+      {/* Play button - centered and pointing right */}
+      <path d="M22 18l-6 4v-8L22 18z" fill="white" />
+    </>
+  );
+}

packages/client/src/svgs/index.ts

@@ -65,9 +65,11 @@ export { default as PersonalizationIcon } from './PersonalizationIcon';
 export { default as MCPIcon } from './MCPIcon';
 export { default as VectorIcon } from './VectorIcon';
 export { default as SquirclePlusIcon } from './SquirclePlusIcon';
+export { default as AudioPaths } from './AudioPaths';
 export { default as CodePaths } from './CodePaths';
 export { default as FileIcon } from './FileIcon';
 export { default as FilePaths } from './FilePaths';
 export { default as SheetPaths } from './SheetPaths';
 export { default as TextPaths } from './TextPaths';
+export { default as VideoPaths } from './VideoPaths';
 export { default as SharePointIcon } from './SharePointIcon';

packages/data-provider/src/config.ts

@@ -175,6 +175,7 @@ export enum Capabilities {
 export enum AgentCapabilities {
   hide_sequential_outputs = 'hide_sequential_outputs',
   end_after_tools = 'end_after_tools',
+  direct_attach = 'direct_attach',
   execute_code = 'execute_code',
   file_search = 'file_search',
   web_search = 'web_search',
@@ -248,6 +249,7 @@ export const assistantEndpointSchema = baseEndpointSchema.merge(
 export type TAssistantEndpoint = z.infer<typeof assistantEndpointSchema>;
 
 export const defaultAgentCapabilities = [
+  AgentCapabilities.direct_attach,
   AgentCapabilities.execute_code,
   AgentCapabilities.file_search,
   AgentCapabilities.web_search,

packages/data-provider/src/file-config.ts

@@ -57,6 +57,27 @@ export const fullMimeTypesList = [
   'application/zip',
   'image/svg',
   'image/svg+xml',
+  // Video formats
+  'video/mp4',
+  'video/avi',
+  'video/mov',
+  'video/wmv',
+  'video/flv',
+  'video/webm',
+  'video/mkv',
+  'video/m4v',
+  'video/3gp',
+  'video/ogv',
+  // Audio formats
+  'audio/mp3',
+  'audio/wav',
+  'audio/ogg',
+  'audio/m4a',
+  'audio/aac',
+  'audio/flac',
+  'audio/wma',
+  'audio/opus',
+  'audio/mpeg',
   ...excelFileTypes,
 ];
 
@@ -123,7 +144,9 @@ export const applicationMimeTypes =
 export const imageMimeTypes = /^image\/(jpeg|gif|png|webp|heic|heif)$/;
 
 export const audioMimeTypes =
-  /^audio\/(mp3|mpeg|mpeg3|wav|wave|x-wav|ogg|vorbis|mp4|x-m4a|flac|x-flac|webm)$/;
+  /^audio\/(mp3|mpeg|mpeg3|wav|wave|x-wav|ogg|vorbis|mp4|m4a|x-m4a|flac|x-flac|webm|aac|wma|opus)$/;
+
+export const videoMimeTypes = /^video\/(mp4|avi|mov|wmv|flv|webm|mkv|m4v|3gp|ogv)$/;
 
 export const defaultOCRMimeTypes = [
   imageMimeTypes,
@@ -142,8 +165,9 @@ export const supportedMimeTypes = [
   excelMimeTypes,
   applicationMimeTypes,
   imageMimeTypes,
+  videoMimeTypes,
   audioMimeTypes,
-  /** Supported by LC Code Interpreter PAI */
+  /** Supported by LC Code Interpreter API */
   /^image\/(svg|svg\+xml)$/,
 ];
 
@@ -186,6 +210,10 @@ export const mbToBytes = (mb: number): number => mb * megabyte;
 
 const defaultSizeLimit = mbToBytes(512);
 const defaultTokenLimit = 100000;
+
+// Anthropic PDF limits: 32MB max, 100 pages max
+export const anthropicPdfSizeLimit = mbToBytes(32);
+
 const assistantsFileConfig = {
   fileLimit: 10,
   fileSizeLimit: defaultSizeLimit,
@@ -199,6 +227,14 @@ export const fileConfig = {
     [EModelEndpoint.assistants]: assistantsFileConfig,
     [EModelEndpoint.azureAssistants]: assistantsFileConfig,
     [EModelEndpoint.agents]: assistantsFileConfig,
+    [EModelEndpoint.anthropic]: {
+      fileLimit: 10,
+      fileSizeLimit: defaultSizeLimit,
+      totalSizeLimit: defaultSizeLimit,
+      supportedMimeTypes,
+      disabled: false,
+      pdfSizeLimit: anthropicPdfSizeLimit,
+    },
     default: {
       fileLimit: 10,
       fileSizeLimit: defaultSizeLimit,

packages/data-provider/src/schemas.ts

@@ -31,6 +31,20 @@ export enum EModelEndpoint {
   gptPlugins = 'gptPlugins',
 }
 
+/**
+ * Endpoints that support direct PDF processing in the agent system
+ */
+export const documentSupportedEndpoints = new Set<EModelEndpoint>([
+  EModelEndpoint.anthropic,
+  EModelEndpoint.openAI,
+  EModelEndpoint.azureOpenAI,
+  EModelEndpoint.google,
+]);
+
+export const isDocumentSupportedEndpoint = (endpoint: EModelEndpoint): boolean => {
+  return documentSupportedEndpoints.has(endpoint);
+};
+
 export const paramEndpoints = new Set<EModelEndpoint | string>([
   EModelEndpoint.agents,
   EModelEndpoint.openAI,

packages/data-provider/src/types/assistants.ts

@@ -27,6 +27,7 @@ export enum Tools {
 
 export enum EToolResources {
   code_interpreter = 'code_interpreter',
+  direct_attach = 'direct_attach',
   execute_code = 'execute_code',
   file_search = 'file_search',
   image_edit = 'image_edit',