From 352565c9a62c0e47b71d7b78d08c838a313c0f41 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 31 Jan 2025 19:11:04 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A5=20feat:=20YouTube=20Tool=20(#5582)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adding youtube tool * refactor: use short `url` param instead of `videoUrl` * refactor: move API key retrieval to a separate credentials module * refactor: remove unnecessary `isEdited` message property * refactor: remove unnecessary `isEdited` message property pt. 2 * refactor: YouTube Tool with new `tool()` generator, handle tools already created by new `tool` generator * fix: only reset request data for multi-convo messages * refactor: enhance YouTube tool by adding transcript parsing and returning structured JSON responses * refactor: update transcript parsing to handle raw response and clean up text output * feat: support toolkits and refactor YouTube tool as a toolkit for better LLM usage * refactor: remove unused OpenAPI specs and streamline tools transformation in loadAsyncEndpoints * refactor: implement manifestToolMap for better tool management and streamline authentication handling * feat: support toolkits for assistants * refactor: rename loadedTools to toolDefinitions for clarity in PluginController and assistant controllers * feat: complete support of toolkits for assistants --------- Co-authored-by: Danilo Pejakovic --- .env.example | 4 + api/app/clients/PluginsClient.js | 2 - .../clients/prompts/formatMessages.spec.js | 1 - api/app/clients/tools/index.js | 21 +- api/app/clients/tools/manifest.json | 14 ++ .../clients/tools/structured/TavilySearch.js | 10 +- api/app/clients/tools/structured/YouTube.js | 203 ++++++++++++++++++ .../clients/tools/structured/credentials.js | 13 ++ api/app/clients/tools/util/handleTools.js | 37 ++-- api/models/Message.js | 5 +- api/models/Message.spec.js | 1 - api/models/schema/messageSchema.js | 4 - api/package.json | 2 + api/server/controllers/PluginController.js | 21 +- api/server/controllers/assistants/v1.js | 33 ++- api/server/controllers/assistants/v2.js | 42 +++- api/server/middleware/abortMiddleware.js | 3 +- .../services/Config/loadAsyncEndpoints.js | 12 -- api/server/services/ToolService.js | 31 ++- .../import/__data__/librechat-export.json | 6 - .../librechat-opts-nonr-branches.json | 5 - .../utils/import/__data__/librechat-tree.json | 2 - .../Chat/Messages/Content/EditMessage.tsx | 1 - .../Messages/Content/Parts/EditTextPart.tsx | 1 - .../SidePanel/Builder/AssistantPanel.tsx | 16 +- .../SidePanel/Builder/AssistantSelect.tsx | 39 +++- package-lock.json | 25 ++- packages/data-provider/package.json | 2 +- packages/data-provider/src/schemas.ts | 2 +- 29 files changed, 456 insertions(+), 102 deletions(-) create mode 100644 api/app/clients/tools/structured/YouTube.js create mode 100644 api/app/clients/tools/structured/credentials.js diff --git a/.env.example b/.env.example index 4f6fbe091..93408df7b 100644 --- a/.env.example +++ b/.env.example @@ -255,6 +255,10 @@ AZURE_AI_SEARCH_SEARCH_OPTION_SELECT= GOOGLE_SEARCH_API_KEY= GOOGLE_CSE_ID= +# YOUTUBE +#----------------- +YOUTUBE_API_KEY= + # SerpAPI #----------------- SERPAPI_API_KEY= diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js index c15258f98..bfe222e24 100644 --- a/api/app/clients/PluginsClient.js +++ b/api/app/clients/PluginsClient.js @@ -280,7 +280,6 @@ class PluginsClient extends OpenAIClient { logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts }); const { user, - isEdited, conversationId, responseMessageId, saveOptions, @@ -359,7 +358,6 @@ class PluginsClient extends OpenAIClient { conversationId, parentMessageId: userMessage.messageId, isCreatedByUser: false, - isEdited, model: this.modelOptions.model, sender: this.sender, promptTokens, diff --git a/api/app/clients/prompts/formatMessages.spec.js b/api/app/clients/prompts/formatMessages.spec.js index 712a6d962..97e40b0ca 100644 --- a/api/app/clients/prompts/formatMessages.spec.js +++ b/api/app/clients/prompts/formatMessages.spec.js @@ -60,7 +60,6 @@ describe('formatMessage', () => { error: false, finish_reason: null, isCreatedByUser: true, - isEdited: false, model: null, parentMessageId: Constants.NO_PARENT, sender: 'User', diff --git a/api/app/clients/tools/index.js b/api/app/clients/tools/index.js index 176c26dd7..b8df50c77 100644 --- a/api/app/clients/tools/index.js +++ b/api/app/clients/tools/index.js @@ -2,23 +2,40 @@ const availableTools = require('./manifest.json'); // Structured Tools const DALLE3 = require('./structured/DALLE3'); +const OpenWeather = require('./structured/OpenWeather'); +const createYouTubeTools = require('./structured/YouTube'); const StructuredWolfram = require('./structured/Wolfram'); const StructuredACS = require('./structured/AzureAISearch'); const StructuredSD = require('./structured/StableDiffusion'); const GoogleSearchAPI = require('./structured/GoogleSearch'); const TraversaalSearch = require('./structured/TraversaalSearch'); const TavilySearchResults = require('./structured/TavilySearchResults'); -const OpenWeather = require('./structured/OpenWeather'); + +/** @type {Record} */ +const manifestToolMap = {}; + +/** @type {Array} */ +const toolkits = []; + +availableTools.forEach((tool) => { + manifestToolMap[tool.pluginKey] = tool; + if (tool.toolkit === true) { + toolkits.push(tool); + } +}); module.exports = { + toolkits, availableTools, + manifestToolMap, // Structured Tools DALLE3, + OpenWeather, StructuredSD, StructuredACS, GoogleSearchAPI, TraversaalSearch, StructuredWolfram, + createYouTubeTools, TavilySearchResults, - OpenWeather, }; diff --git a/api/app/clients/tools/manifest.json b/api/app/clients/tools/manifest.json index 4b5aabd1a..7cb92b8d8 100644 --- a/api/app/clients/tools/manifest.json +++ b/api/app/clients/tools/manifest.json @@ -30,6 +30,20 @@ } ] }, + { + "name": "YouTube", + "pluginKey": "youtube", + "toolkit": true, + "description": "Get YouTube video information, retrieve comments, analyze transcripts and search for videos.", + "icon": "https://www.youtube.com/s/desktop/7449ebf7/img/favicon_144x144.png", + "authConfig": [ + { + "authField": "YOUTUBE_API_KEY", + "label": "YouTube API Key", + "description": "Your YouTube Data API v3 key." + } + ] + }, { "name": "Wolfram", "pluginKey": "wolfram", diff --git a/api/app/clients/tools/structured/TavilySearch.js b/api/app/clients/tools/structured/TavilySearch.js index 3bb8f34f3..b5478d0fc 100644 --- a/api/app/clients/tools/structured/TavilySearch.js +++ b/api/app/clients/tools/structured/TavilySearch.js @@ -1,6 +1,6 @@ const { z } = require('zod'); const { tool } = require('@langchain/core/tools'); -const { getEnvironmentVariable } = require('@langchain/core/utils/env'); +const { getApiKey } = require('./credentials'); function createTavilySearchTool(fields = {}) { const envVar = 'TAVILY_API_KEY'; @@ -8,14 +8,6 @@ function createTavilySearchTool(fields = {}) { const apiKey = fields.apiKey ?? getApiKey(envVar, override); const kwargs = fields?.kwargs ?? {}; - function getApiKey(envVar, override) { - const key = getEnvironmentVariable(envVar); - if (!key && !override) { - throw new Error(`Missing ${envVar} environment variable.`); - } - return key; - } - return tool( async (input) => { const { query, ...rest } = input; diff --git a/api/app/clients/tools/structured/YouTube.js b/api/app/clients/tools/structured/YouTube.js new file mode 100644 index 000000000..aa19fc211 --- /dev/null +++ b/api/app/clients/tools/structured/YouTube.js @@ -0,0 +1,203 @@ +const { z } = require('zod'); +const { tool } = require('@langchain/core/tools'); +const { youtube } = require('@googleapis/youtube'); +const { YoutubeTranscript } = require('youtube-transcript'); +const { getApiKey } = require('./credentials'); +const { logger } = require('~/config'); + +function extractVideoId(url) { + const rawIdRegex = /^[a-zA-Z0-9_-]{11}$/; + if (rawIdRegex.test(url)) { + return url; + } + + const regex = new RegExp( + '(?:youtu\\.be/|youtube(?:\\.com)?/(?:' + + '(?:watch\\?v=)|(?:embed/)|(?:shorts/)|(?:live/)|(?:v/)|(?:/))?)' + + '([a-zA-Z0-9_-]{11})(?:\\S+)?$', + ); + const match = url.match(regex); + return match ? match[1] : null; +} + +function parseTranscript(transcriptResponse) { + if (!Array.isArray(transcriptResponse)) { + return ''; + } + + return transcriptResponse + .map((entry) => entry.text.trim()) + .filter((text) => text) + .join(' ') + .replaceAll(''', '\''); +} + +function createYouTubeTools(fields = {}) { + const envVar = 'YOUTUBE_API_KEY'; + const override = fields.override ?? false; + const apiKey = fields.apiKey ?? fields[envVar] ?? getApiKey(envVar, override); + + const youtubeClient = youtube({ + version: 'v3', + auth: apiKey, + }); + + const searchTool = tool( + async ({ query, maxResults = 5 }) => { + const response = await youtubeClient.search.list({ + part: 'snippet', + q: query, + type: 'video', + maxResults: maxResults || 5, + }); + const result = response.data.items.map((item) => ({ + title: item.snippet.title, + description: item.snippet.description, + url: `https://www.youtube.com/watch?v=${item.id.videoId}`, + })); + return JSON.stringify(result, null, 2); + }, + { + name: 'youtube_search', + description: `Search for YouTube videos by keyword or phrase. +- Required: query (search terms to find videos) +- Optional: maxResults (number of videos to return, 1-50, default: 5) +- Returns: List of videos with titles, descriptions, and URLs +- Use for: Finding specific videos, exploring content, research +Example: query="cooking pasta tutorials" maxResults=3`, + schema: z.object({ + query: z.string().describe('Search query terms'), + maxResults: z.number().int().min(1).max(50).optional().describe('Number of results (1-50)'), + }), + }, + ); + + const infoTool = tool( + async ({ url }) => { + const videoId = extractVideoId(url); + if (!videoId) { + throw new Error('Invalid YouTube URL or video ID'); + } + + const response = await youtubeClient.videos.list({ + part: 'snippet,statistics', + id: videoId, + }); + + if (!response.data.items?.length) { + throw new Error('Video not found'); + } + const video = response.data.items[0]; + + const result = { + title: video.snippet.title, + description: video.snippet.description, + views: video.statistics.viewCount, + likes: video.statistics.likeCount, + comments: video.statistics.commentCount, + }; + return JSON.stringify(result, null, 2); + }, + { + name: 'youtube_info', + description: `Get detailed metadata and statistics for a specific YouTube video. +- Required: url (full YouTube URL or video ID) +- Returns: Video title, description, view count, like count, comment count +- Use for: Getting video metrics and basic metadata +- DO NOT USE FOR VIDEO SUMMARIES, USE TRANSCRIPTS FOR COMPREHENSIVE ANALYSIS +- Accepts both full URLs and video IDs +Example: url="https://youtube.com/watch?v=abc123" or url="abc123"`, + schema: z.object({ + url: z.string().describe('YouTube video URL or ID'), + }), + }, + ); + + const commentsTool = tool( + async ({ url, maxResults = 10 }) => { + const videoId = extractVideoId(url); + if (!videoId) { + throw new Error('Invalid YouTube URL or video ID'); + } + + const response = await youtubeClient.commentThreads.list({ + part: 'snippet', + videoId, + maxResults: maxResults || 10, + }); + + const result = response.data.items.map((item) => ({ + author: item.snippet.topLevelComment.snippet.authorDisplayName, + text: item.snippet.topLevelComment.snippet.textDisplay, + likes: item.snippet.topLevelComment.snippet.likeCount, + })); + return JSON.stringify(result, null, 2); + }, + { + name: 'youtube_comments', + description: `Retrieve top-level comments from a YouTube video. +- Required: url (full YouTube URL or video ID) +- Optional: maxResults (number of comments, 1-50, default: 10) +- Returns: Comment text, author names, like counts +- Use for: Sentiment analysis, audience feedback, engagement review +Example: url="abc123" maxResults=20`, + schema: z.object({ + url: z.string().describe('YouTube video URL or ID'), + maxResults: z + .number() + .int() + .min(1) + .max(50) + .optional() + .describe('Number of comments to retrieve'), + }), + }, + ); + + const transcriptTool = tool( + async ({ url }) => { + const videoId = extractVideoId(url); + if (!videoId) { + throw new Error('Invalid YouTube URL or video ID'); + } + + try { + try { + const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'en' }); + return parseTranscript(transcript); + } catch (e) { + logger.error(e); + } + + try { + const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'de' }); + return parseTranscript(transcript); + } catch (e) { + logger.error(e); + } + + const transcript = await YoutubeTranscript.fetchTranscript(videoId); + return parseTranscript(transcript); + } catch (error) { + throw new Error(`Failed to fetch transcript: ${error.message}`); + } + }, + { + name: 'youtube_transcript', + description: `Fetch and parse the transcript/captions of a YouTube video. +- Required: url (full YouTube URL or video ID) +- Returns: Full video transcript as plain text +- Use for: Content analysis, summarization, translation reference +- This is the "Go-to" tool for analyzing actual video content +- Attempts to fetch English first, then German, then any available language +Example: url="https://youtube.com/watch?v=abc123"`, + schema: z.object({ + url: z.string().describe('YouTube video URL or ID'), + }), + }, + ); + + return [searchTool, infoTool, commentsTool, transcriptTool]; +} + +module.exports = createYouTubeTools; diff --git a/api/app/clients/tools/structured/credentials.js b/api/app/clients/tools/structured/credentials.js new file mode 100644 index 000000000..fbcce6fbf --- /dev/null +++ b/api/app/clients/tools/structured/credentials.js @@ -0,0 +1,13 @@ +const { getEnvironmentVariable } = require('@langchain/core/utils/env'); + +function getApiKey(envVar, override) { + const key = getEnvironmentVariable(envVar); + if (!key && !override) { + throw new Error(`Missing ${envVar} environment variable.`); + } + return key; +} + +module.exports = { + getApiKey, +}; diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index 39e710d94..f1dfa24a4 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -5,16 +5,18 @@ const { createCodeExecutionTool, EnvVar } = require('@librechat/agents'); const { getUserPluginAuthValue } = require('~/server/services/PluginService'); const { availableTools, + manifestToolMap, // Basic Tools GoogleSearchAPI, // Structured Tools DALLE3, + OpenWeather, StructuredSD, StructuredACS, TraversaalSearch, StructuredWolfram, + createYouTubeTools, TavilySearchResults, - OpenWeather, } = require('../'); const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process'); const { createFileSearchTool, primeFiles: primeSearchFiles } = require('./fileSearch'); @@ -146,6 +148,14 @@ const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => }; }; +/** + * @param {string} toolKey + * @returns {Array} + */ +const getAuthFields = (toolKey) => { + return manifestToolMap[toolKey]?.authConfig.map((auth) => auth.authField) ?? []; +}; + /** * * @param {object} object @@ -174,19 +184,21 @@ const loadTools = async ({ const toolConstructors = { calculator: Calculator, google: GoogleSearchAPI, + open_weather: OpenWeather, wolfram: StructuredWolfram, 'stable-diffusion': StructuredSD, 'azure-ai-search': StructuredACS, traversaal_search: TraversaalSearch, tavily_search_results_json: TavilySearchResults, - open_weather: OpenWeather, }; const customConstructors = { serpapi: async () => { - let apiKey = process.env.SERPAPI_API_KEY; + const authFields = getAuthFields('serpapi'); + let envVar = authFields[0] ?? ''; + let apiKey = process.env[envVar]; if (!apiKey) { - apiKey = await getUserPluginAuthValue(user, 'SERPAPI_API_KEY'); + apiKey = await getUserPluginAuthValue(user, envVar); } return new SerpAPI(apiKey, { location: 'Austin,Texas,United States', @@ -194,6 +206,11 @@ const loadTools = async ({ gl: 'us', }); }, + youtube: async () => { + const authFields = getAuthFields('youtube'); + const authValues = await loadAuthValues({ userId: user, authFields }); + return createYouTubeTools(authValues); + }, }; const requestedTools = {}; @@ -218,16 +235,6 @@ const loadTools = async ({ 'stable-diffusion': imageGenOptions, }; - const toolAuthFields = {}; - - availableTools.forEach((tool) => { - if (customConstructors[tool.pluginKey]) { - return; - } - - toolAuthFields[tool.pluginKey] = tool.authConfig.map((auth) => auth.authField); - }); - const toolContextMap = {}; const remainingTools = []; const appTools = options.req?.app?.locals?.availableTools ?? {}; @@ -282,7 +289,7 @@ const loadTools = async ({ const options = toolOptions[tool] || {}; const toolInstance = loadToolWithAuth( user, - toolAuthFields[tool], + getAuthFields(tool), toolConstructors[tool], options, ); diff --git a/api/models/Message.js b/api/models/Message.js index f8f4fa7bc..22e1fa655 100644 --- a/api/models/Message.js +++ b/api/models/Message.js @@ -23,7 +23,6 @@ const idSchema = z.string().uuid(); * @param {string} [params.error] - Any error associated with the message. * @param {boolean} [params.unfinished] - Indicates if the message is unfinished. * @param {Object[]} [params.files] - An array of files associated with the message. - * @param {boolean} [params.isEdited] - Indicates if the message was edited. * @param {string} [params.finish_reason] - Reason for finishing the message. * @param {number} [params.tokenCount] - The number of tokens in the message. * @param {string} [params.plugin] - Plugin associated with the message. @@ -77,7 +76,7 @@ async function saveMessage(req, params, metadata) { * @returns {Promise} The result of the bulk write operation. * @throws {Error} If there is an error in saving messages in bulk. */ -async function bulkSaveMessages(messages, overrideTimestamp=false) { +async function bulkSaveMessages(messages, overrideTimestamp = false) { try { const bulkOps = messages.map((message) => ({ updateOne: { @@ -182,7 +181,6 @@ async function updateMessageText(req, { messageId, text }) { async function updateMessage(req, message, metadata) { try { const { messageId, ...update } = message; - update.isEdited = true; const updatedMessage = await Message.findOneAndUpdate( { messageId, user: req.user.id }, update, @@ -203,7 +201,6 @@ async function updateMessage(req, message, metadata) { text: updatedMessage.text, isCreatedByUser: updatedMessage.isCreatedByUser, tokenCount: updatedMessage.tokenCount, - isEdited: true, }; } catch (err) { logger.error('Error updating message:', err); diff --git a/api/models/Message.spec.js b/api/models/Message.spec.js index a38458a68..a542130b5 100644 --- a/api/models/Message.spec.js +++ b/api/models/Message.spec.js @@ -100,7 +100,6 @@ describe('Message Operations', () => { expect.objectContaining({ messageId: 'msg123', text: 'Hello, world!', - isEdited: true, }), ); }); diff --git a/api/models/schema/messageSchema.js b/api/models/schema/messageSchema.js index 5d2e43a3e..000f83e8c 100644 --- a/api/models/schema/messageSchema.js +++ b/api/models/schema/messageSchema.js @@ -62,10 +62,6 @@ const messageSchema = mongoose.Schema( required: true, default: false, }, - isEdited: { - type: Boolean, - default: false, - }, unfinished: { type: Boolean, default: false, diff --git a/api/package.json b/api/package.json index 7a2941301..1480b847d 100644 --- a/api/package.json +++ b/api/package.json @@ -37,6 +37,7 @@ "@anthropic-ai/sdk": "^0.32.1", "@azure/search-documents": "^12.0.0", "@google/generative-ai": "^0.21.0", + "@googleapis/youtube": "^20.0.0", "@keyv/mongo": "^2.1.8", "@keyv/redis": "^2.8.1", "@langchain/community": "^0.3.14", @@ -105,6 +106,7 @@ "ua-parser-js": "^1.0.36", "winston": "^3.11.0", "winston-daily-rotate-file": "^4.7.1", + "youtube-transcript": "^1.2.1", "zod": "^3.22.4" }, "devDependencies": { diff --git a/api/server/controllers/PluginController.js b/api/server/controllers/PluginController.js index 2cdbd1549..9e87b4628 100644 --- a/api/server/controllers/PluginController.js +++ b/api/server/controllers/PluginController.js @@ -1,7 +1,7 @@ -const { promises: fs } = require('fs'); const { CacheKeys, AuthType } = require('librechat-data-provider'); const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs'); const { getCustomConfig } = require('~/server/services/Config'); +const { availableTools } = require('~/app/clients/tools'); const { getMCPManager } = require('~/config'); const { getLogStores } = require('~/cache'); @@ -59,10 +59,9 @@ const getAvailablePluginsController = async (req, res) => { /** @type {{ filteredTools: string[], includedTools: string[] }} */ const { filteredTools = [], includedTools = [] } = req.app.locals; - const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8'); - const jsonData = JSON.parse(pluginManifest); + const pluginManifest = availableTools; - const uniquePlugins = filterUniquePlugins(jsonData); + const uniquePlugins = filterUniquePlugins(pluginManifest); let authenticatedPlugins = []; for (const plugin of uniquePlugins) { authenticatedPlugins.push( @@ -106,17 +105,15 @@ const getAvailableTools = async (req, res) => { return; } - const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8'); - - const jsonData = JSON.parse(pluginManifest); + const pluginManifest = availableTools; const customConfig = await getCustomConfig(); if (customConfig?.mcpServers != null) { const mcpManager = await getMCPManager(); - await mcpManager.loadManifestTools(jsonData); + await mcpManager.loadManifestTools(pluginManifest); } /** @type {TPlugin[]} */ - const uniquePlugins = filterUniquePlugins(jsonData); + const uniquePlugins = filterUniquePlugins(pluginManifest); const authenticatedPlugins = uniquePlugins.map((plugin) => { if (checkPluginAuth(plugin)) { @@ -126,8 +123,12 @@ const getAvailableTools = async (req, res) => { } }); + const toolDefinitions = req.app.locals.availableTools; const tools = authenticatedPlugins.filter( - (plugin) => req.app.locals.availableTools[plugin.pluginKey] !== undefined, + (plugin) => + toolDefinitions[plugin.pluginKey] !== undefined || + (plugin.toolkit === true && + Object.keys(toolDefinitions).some((key) => key.startsWith(`${plugin.pluginKey}_`))), ); await cache.set(CacheKeys.TOOLS, tools); diff --git a/api/server/controllers/assistants/v1.js b/api/server/controllers/assistants/v1.js index 05fda6895..8fb73167c 100644 --- a/api/server/controllers/assistants/v1.js +++ b/api/server/controllers/assistants/v1.js @@ -6,6 +6,7 @@ const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { deleteAssistantActions } = require('~/server/services/ActionService'); const { updateAssistantDoc, getAssistants } = require('~/models/Assistant'); const { getOpenAIClient, fetchAssistants } = require('./helpers'); +const { manifestToolMap } = require('~/app/clients/tools'); const { deleteFileByFilter } = require('~/models/File'); const { logger } = require('~/config'); @@ -35,9 +36,21 @@ const createAssistant = async (req, res) => { return tool; } - return req.app.locals.availableTools[tool]; + const toolDefinitions = req.app.locals.availableTools; + const toolDef = toolDefinitions[tool]; + if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) { + return ( + Object.entries(toolDefinitions) + .filter(([key]) => key.startsWith(`${tool}_`)) + // eslint-disable-next-line no-unused-vars + .map(([_, val]) => val) + ); + } + + return toolDef; }) - .filter((tool) => tool); + .filter((tool) => tool) + .flat(); let azureModelIdentifier = null; if (openai.locals?.azureOptions) { @@ -128,9 +141,21 @@ const patchAssistant = async (req, res) => { return tool; } - return req.app.locals.availableTools[tool]; + const toolDefinitions = req.app.locals.availableTools; + const toolDef = toolDefinitions[tool]; + if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) { + return ( + Object.entries(toolDefinitions) + .filter(([key]) => key.startsWith(`${tool}_`)) + // eslint-disable-next-line no-unused-vars + .map(([_, val]) => val) + ); + } + + return toolDef; }) - .filter((tool) => tool); + .filter((tool) => tool) + .flat(); if (openai.locals?.azureOptions && updateData.model) { updateData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName; diff --git a/api/server/controllers/assistants/v2.js b/api/server/controllers/assistants/v2.js index 54f9a6fbc..3bf83a626 100644 --- a/api/server/controllers/assistants/v2.js +++ b/api/server/controllers/assistants/v2.js @@ -2,6 +2,7 @@ const { ToolCallTypes } = require('librechat-data-provider'); const validateAuthor = require('~/server/middleware/assistants/validateAuthor'); const { validateAndUpdateTool } = require('~/server/services/ActionService'); const { updateAssistantDoc } = require('~/models/Assistant'); +const { manifestToolMap } = require('~/app/clients/tools'); const { getOpenAIClient } = require('./helpers'); const { logger } = require('~/config'); @@ -32,9 +33,21 @@ const createAssistant = async (req, res) => { return tool; } - return req.app.locals.availableTools[tool]; + const toolDefinitions = req.app.locals.availableTools; + const toolDef = toolDefinitions[tool]; + if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) { + return ( + Object.entries(toolDefinitions) + .filter(([key]) => key.startsWith(`${tool}_`)) + // eslint-disable-next-line no-unused-vars + .map(([_, val]) => val) + ); + } + + return toolDef; }) - .filter((tool) => tool); + .filter((tool) => tool) + .flat(); let azureModelIdentifier = null; if (openai.locals?.azureOptions) { @@ -112,9 +125,30 @@ const updateAssistant = async ({ req, openai, assistant_id, updateData }) => { let hasFileSearch = false; for (const tool of updateData.tools ?? []) { - let actualTool = typeof tool === 'string' ? req.app.locals.availableTools[tool] : tool; + const toolDefinitions = req.app.locals.availableTools; + let actualTool = typeof tool === 'string' ? toolDefinitions[tool] : tool; - if (!actualTool) { + if (!actualTool && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) { + actualTool = Object.entries(toolDefinitions) + .filter(([key]) => key.startsWith(`${tool}_`)) + // eslint-disable-next-line no-unused-vars + .map(([_, val]) => val); + } else if (!actualTool) { + continue; + } + + if (Array.isArray(actualTool)) { + for (const subTool of actualTool) { + if (!subTool.function) { + tools.push(subTool); + continue; + } + + const updatedTool = await validateAndUpdateTool({ req, tool: subTool, assistant_id }); + if (updatedTool) { + tools.push(updatedTool); + } + } continue; } diff --git a/api/server/middleware/abortMiddleware.js b/api/server/middleware/abortMiddleware.js index 6e608e1cc..2137523ef 100644 --- a/api/server/middleware/abortMiddleware.js +++ b/api/server/middleware/abortMiddleware.js @@ -75,8 +75,9 @@ const createAbortController = (req, res, getAbortData, getReqData) => { const abortKey = userMessage?.conversationId ?? req.user.id; const prevRequest = abortControllers.get(abortKey); + const { overrideUserMessageId } = req?.body ?? {}; - if (prevRequest && prevRequest?.abortController) { + if (overrideUserMessageId != null && prevRequest && prevRequest?.abortController) { const data = prevRequest.abortController.getAbortData(); getReqData({ userMessage: data?.userMessage }); const addedAbortKey = `${abortKey}:${responseMessageId}`; diff --git a/api/server/services/Config/loadAsyncEndpoints.js b/api/server/services/Config/loadAsyncEndpoints.js index 409b9485d..0282146cd 100644 --- a/api/server/services/Config/loadAsyncEndpoints.js +++ b/api/server/services/Config/loadAsyncEndpoints.js @@ -1,6 +1,4 @@ const { EModelEndpoint } = require('librechat-data-provider'); -const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs'); -const { availableTools } = require('~/app/clients/tools'); const { isUserProvided } = require('~/server/utils'); const { config } = require('./EndpointService'); @@ -28,22 +26,12 @@ async function loadAsyncEndpoints(req) { } } - const tools = await addOpenAPISpecs(availableTools); - function transformToolsToMap(tools) { - return tools.reduce((map, obj) => { - map[obj.pluginKey] = obj.name; - return map; - }, {}); - } - const plugins = transformToolsToMap(tools); - const google = serviceKey || googleKey ? { userProvide: googleUserProvides } : false; const useAzure = req.app.locals[EModelEndpoint.azureOpenAI]?.plugins; const gptPlugins = useAzure || openAIApiKey || azureOpenAIApiKey ? { - plugins, availableAgents: ['classic', 'functions'], userProvide: useAzure ? false : userProvidedOpenAI, userProvideURL: useAzure diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 306b2e99f..08891c99b 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -1,7 +1,7 @@ const fs = require('fs'); const path = require('path'); const { zodToJsonSchema } = require('zod-to-json-schema'); -const { tool: toolFn, Tool } = require('@langchain/core/tools'); +const { tool: toolFn, Tool, DynamicStructuredTool } = require('@langchain/core/tools'); const { Calculator } = require('@langchain/community/tools/calculator'); const { Tools, @@ -16,6 +16,7 @@ const { validateAndParseOpenAPISpec, } = require('librechat-data-provider'); const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process'); +const { createYouTubeTools, manifestToolMap, toolkits } = require('~/app/clients/tools'); const { loadActionSets, createActionTool, domainParser } = require('./ActionService'); const { getEndpointsConfig } = require('~/server/services/Config'); const { recordUsage } = require('~/server/services/Threads'); @@ -97,7 +98,7 @@ function loadAndFormatTools({ directory, adminFilter = [], adminIncluded = [] }) } /** Basic Tools; schema: { input: string } */ - const basicToolInstances = [new Calculator()]; + const basicToolInstances = [new Calculator(), ...createYouTubeTools({ override: true })]; for (const toolInstance of basicToolInstances) { const formattedTool = formatToOpenAIAssistantTool(toolInstance); tools.push(formattedTool); @@ -173,7 +174,26 @@ async function processRequiredActions(client, requiredActions) { `[required actions] user: ${client.req.user.id} | thread_id: ${requiredActions[0].thread_id} | run_id: ${requiredActions[0].run_id}`, requiredActions, ); - const tools = requiredActions.map((action) => action.tool); + const toolDefinitions = client.req.app.locals.availableTools; + const seenToolkits = new Set(); + const tools = requiredActions + .map((action) => { + const toolName = action.tool; + const toolDef = toolDefinitions[toolName]; + if (toolDef && !manifestToolMap[toolName]) { + for (const toolkit of toolkits) { + if (seenToolkits.has(toolkit.pluginKey)) { + return; + } else if (toolName.startsWith(`${toolkit.pluginKey}_`)) { + seenToolkits.add(toolkit.pluginKey); + return toolkit.pluginKey; + } + } + } + return toolName; + }) + .filter((toolName) => !!toolName); + const { loadedTools } = await loadTools({ user: client.req.user.id, model: client.req.body.model ?? 'gpt-4o-mini', @@ -441,6 +461,11 @@ async function loadAgentTools({ req, agent, tool_resources, openAIApiKey }) { continue; } + if (tool instanceof DynamicStructuredTool) { + agentTools.push(tool); + continue; + } + const toolDefinition = { name: tool.name, schema: tool.schema, diff --git a/api/server/utils/import/__data__/librechat-export.json b/api/server/utils/import/__data__/librechat-export.json index 001d83140..4cec7f339 100644 --- a/api/server/utils/import/__data__/librechat-export.json +++ b/api/server/utils/import/__data__/librechat-export.json @@ -29,7 +29,6 @@ "endpoint": "openAI", "error": false, "isCreatedByUser": true, - "isEdited": false, "model": null, "parentMessageId": "00000000-0000-0000-0000-000000000000", "sender": "user", @@ -47,7 +46,6 @@ "endpoint": "openAI", "error": false, "isCreatedByUser": false, - "isEdited": false, "model": null, "parentMessageId": "b123942f-ca1a-4b16-9e1f-ea4af5171168", "sender": "GPT-3.5", @@ -65,7 +63,6 @@ "endpoint": "openAI", "error": false, "isCreatedByUser": true, - "isEdited": false, "model": null, "parentMessageId": "549a4f45-cf93-4e3b-ae62-1abf02afbfc8", "sender": "user", @@ -83,7 +80,6 @@ "endpoint": "openAI", "error": false, "isCreatedByUser": false, - "isEdited": false, "model": null, "parentMessageId": "880e5357-3e0c-4218-b351-fd3fc184adef", "sender": "GPT-3.5", @@ -101,7 +97,6 @@ "endpoint": "openAI", "error": false, "isCreatedByUser": true, - "isEdited": false, "model": null, "parentMessageId": "e9796d11-3bdf-4e25-9f0e-4802bbbb8c6d", "sender": "user", @@ -119,7 +114,6 @@ "endpoint": "openAI", "error": false, "isCreatedByUser": false, - "isEdited": false, "model": null, "parentMessageId": "04408c06-62dc-4961-8ef5-4336b68e7a0a", "sender": "GPT-3.5", diff --git a/api/server/utils/import/__data__/librechat-opts-nonr-branches.json b/api/server/utils/import/__data__/librechat-opts-nonr-branches.json index cc8a5e05b..eb02a7ebd 100644 --- a/api/server/utils/import/__data__/librechat-opts-nonr-branches.json +++ b/api/server/utils/import/__data__/librechat-opts-nonr-branches.json @@ -27,7 +27,6 @@ "endpoint": "azureOpenAI", "error": false, "isCreatedByUser": true, - "isEdited": false, "model": null, "parentMessageId": "00000000-0000-0000-0000-000000000000", "sender": "User", @@ -42,7 +41,6 @@ "createdAt": "2024-05-28T18:08:55.390Z", "error": false, "isCreatedByUser": false, - "isEdited": false, "model": "gpt-4o", "parentMessageId": "115a6247-8fb0-4937-a536-12956669098d", "sender": "GPT-4", @@ -59,7 +57,6 @@ "createdAt": "2024-05-28T18:09:27.444Z", "error": false, "isCreatedByUser": false, - "isEdited": false, "model": "gpt-4o", "parentMessageId": "115a6247-8fb0-4937-a536-12956669098d", "sender": "GPT-4", @@ -74,7 +71,6 @@ "endpoint": "azureOpenAI", "error": false, "isCreatedByUser": true, - "isEdited": false, "model": null, "parentMessageId": "00000000-0000-0000-0000-000000000000", "sender": "User", @@ -89,7 +85,6 @@ "createdAt": "2024-05-28T18:14:08.403Z", "error": false, "isCreatedByUser": false, - "isEdited": true, "model": "gpt-4o", "parentMessageId": "599e1908-8c52-4a73-ba6b-f6dffbd79ba0", "sender": "GPT-4", diff --git a/api/server/utils/import/__data__/librechat-tree.json b/api/server/utils/import/__data__/librechat-tree.json index 50130da2d..8e94cb1dd 100644 --- a/api/server/utils/import/__data__/librechat-tree.json +++ b/api/server/utils/import/__data__/librechat-tree.json @@ -29,7 +29,6 @@ "endpoint": "openAI", "error": false, "isCreatedByUser": true, - "isEdited": false, "model": null, "parentMessageId": "00000000-0000-0000-0000-000000000000", "sender": "User", @@ -47,7 +46,6 @@ "createdAt": "2024-05-01T16:35:12.604Z", "error": false, "isCreatedByUser": false, - "isEdited": false, "model": "gpt-4-turbo", "parentMessageId": "9501f99d-9bbb-40cb-bbb2-16d79aeceb72", "sender": "Software Engineer", diff --git a/client/src/components/Chat/Messages/Content/EditMessage.tsx b/client/src/components/Chat/Messages/Content/EditMessage.tsx index 7c86010d1..aa2eca2d1 100644 --- a/client/src/components/Chat/Messages/Content/EditMessage.tsx +++ b/client/src/components/Chat/Messages/Content/EditMessage.tsx @@ -114,7 +114,6 @@ const EditMessage = ({ ? { ...msg, text: data.text, - isEdited: true, } : msg, ), diff --git a/client/src/components/Chat/Messages/Content/Parts/EditTextPart.tsx b/client/src/components/Chat/Messages/Content/Parts/EditTextPart.tsx index d4ce5dff4..e6736b192 100644 --- a/client/src/components/Chat/Messages/Content/Parts/EditTextPart.tsx +++ b/client/src/components/Chat/Messages/Content/Parts/EditTextPart.tsx @@ -119,7 +119,6 @@ const EditTextPart = ({ ? { ...msg, content: updatedContent, - isEdited: true, } : msg, ), diff --git a/client/src/components/SidePanel/Builder/AssistantPanel.tsx b/client/src/components/SidePanel/Builder/AssistantPanel.tsx index 63f2587da..c78d456ff 100644 --- a/client/src/components/SidePanel/Builder/AssistantPanel.tsx +++ b/client/src/components/SidePanel/Builder/AssistantPanel.tsx @@ -1,25 +1,27 @@ import { useState, useMemo } from 'react'; -import { useQueryClient } from '@tanstack/react-query'; import { useForm, FormProvider, Controller, useWatch } from 'react-hook-form'; import { useGetModelsQuery } from 'librechat-data-provider/react-query'; import { Tools, - QueryKeys, Capabilities, actionDelimiter, ImageVisionTool, defaultAssistantFormValues, } from 'librechat-data-provider'; -import type { FunctionTool, TConfig, TPlugin } from 'librechat-data-provider'; +import type { FunctionTool, TConfig } from 'librechat-data-provider'; import type { AssistantForm, AssistantPanelProps } from '~/common'; -import { useCreateAssistantMutation, useUpdateAssistantMutation } from '~/data-provider'; +import { + useCreateAssistantMutation, + useUpdateAssistantMutation, + useAvailableAgentToolsQuery, +} from '~/data-provider'; import { cn, cardStyle, defaultTextProps, removeFocusOutlines } from '~/utils'; import AssistantConversationStarters from './AssistantConversationStarters'; import { useAssistantsMapContext, useToastContext } from '~/Providers'; import { useSelectAssistant, useLocalize } from '~/hooks'; import { ToolSelectDialog } from '~/components/Tools'; -import CapabilitiesForm from './CapabilitiesForm'; import AppendDateCheckbox from './AppendDateCheckbox'; +import CapabilitiesForm from './CapabilitiesForm'; import { SelectDropDown } from '~/components/ui'; import AssistantAvatar from './AssistantAvatar'; import AssistantSelect from './AssistantSelect'; @@ -49,11 +51,10 @@ export default function AssistantPanel({ assistantsConfig, version, }: AssistantPanelProps & { assistantsConfig?: TConfig | null }) { - const queryClient = useQueryClient(); const modelsQuery = useGetModelsQuery(); const assistantMap = useAssistantsMapContext(); - const allTools = queryClient.getQueryData([QueryKeys.tools]) ?? []; + const { data: allTools = [] } = useAvailableAgentToolsQuery(); const { onSelect: onSelectAssistant } = useSelectAssistant(endpoint); const { showToast } = useToastContext(); const localize = useLocalize(); @@ -227,6 +228,7 @@ export default function AssistantPanel({ value={field.value} endpoint={endpoint} documentsMap={documentsMap} + allTools={allTools} setCurrentAssistantId={setCurrentAssistantId} selectedAssistant={current_assistant_id ?? null} createMutation={create} diff --git a/client/src/components/SidePanel/Builder/AssistantSelect.tsx b/client/src/components/SidePanel/Builder/AssistantSelect.tsx index d3202cdd2..99f840e6d 100644 --- a/client/src/components/SidePanel/Builder/AssistantSelect.tsx +++ b/client/src/components/SidePanel/Builder/AssistantSelect.tsx @@ -1,5 +1,5 @@ import { Plus } from 'lucide-react'; -import { useCallback, useEffect, useRef } from 'react'; +import { useMemo, useCallback, useEffect, useRef } from 'react'; import { Tools, FileSources, @@ -12,6 +12,7 @@ import { import type { UseFormReset } from 'react-hook-form'; import type { UseMutationResult } from '@tanstack/react-query'; import type { + TPlugin, Assistant, AssistantDocument, AssistantsEndpoint, @@ -48,6 +49,7 @@ export default function AssistantSelect({ selectedAssistant, setCurrentAssistantId, createMutation, + allTools, }: { reset: UseFormReset; value: TAssistantOption; @@ -56,6 +58,7 @@ export default function AssistantSelect({ documentsMap: Map | null; setCurrentAssistantId: React.Dispatch>; createMutation: UseMutationResult; + allTools?: TPlugin[]; }) { const localize = useLocalize(); const fileMap = useFileMapContext(); @@ -65,6 +68,11 @@ export default function AssistantSelect({ {} as LastSelectedModels, ); + const toolkits = useMemo( + () => new Set(allTools?.filter((tool) => tool.toolkit === true).map((tool) => tool.pluginKey)), + [allTools], + ); + const query = useListAssistantsQuery(endpoint, undefined, { select: (res) => res.data.map((_assistant) => { @@ -153,7 +161,7 @@ export default function AssistantSelect({ const update = { ...assistant, label: assistant.name ?? '', - value: assistant.id ?? '', + value: assistant.id || '', }; const actions: Actions = { @@ -164,7 +172,7 @@ export default function AssistantSelect({ (assistant.tools ?? []) .filter((tool) => tool.type !== 'function' || isImageVisionTool(tool)) - .map((tool) => tool.function?.name || tool.type) + .map((tool) => (tool.function?.name ?? '') || tool.type) .forEach((tool) => { if (tool === Tools.file_search) { actions[Capabilities.retrieval] = true; @@ -172,9 +180,22 @@ export default function AssistantSelect({ actions[tool] = true; }); + const seenToolkits = new Set(); const functions = (assistant.tools ?? []) .filter((tool) => tool.type === 'function' && !isImageVisionTool(tool)) - .map((tool) => tool.function?.name ?? ''); + .map((tool) => tool.function?.name ?? '') + .filter((fnName) => { + const fnPrefix = fnName.split('_')[0]; + const seenToolkit = toolkits.has(fnPrefix); + if (seenToolkit) { + seenToolkits.add(fnPrefix); + } + return !seenToolkit; + }); + + if (seenToolkits.size > 0) { + functions.push(...Array.from(seenToolkits)); + } const formValues: Partial = { functions, @@ -210,7 +231,15 @@ export default function AssistantSelect({ reset(formValues); setCurrentAssistantId(assistant.id); }, - [query.data, reset, setCurrentAssistantId, createMutation, endpoint, lastSelectedModels], + [ + query.data, + reset, + setCurrentAssistantId, + createMutation, + endpoint, + lastSelectedModels, + toolkits, + ], ); useEffect(() => { diff --git a/package-lock.json b/package-lock.json index c5a5e1af0..5f12f7bdc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,6 +46,7 @@ "@anthropic-ai/sdk": "^0.32.1", "@azure/search-documents": "^12.0.0", "@google/generative-ai": "^0.21.0", + "@googleapis/youtube": "^20.0.0", "@keyv/mongo": "^2.1.8", "@keyv/redis": "^2.8.1", "@langchain/community": "^0.3.14", @@ -114,6 +115,7 @@ "ua-parser-js": "^1.0.36", "winston": "^3.11.0", "winston-daily-rotate-file": "^4.7.1", + "youtube-transcript": "^1.2.1", "zod": "^3.22.4" }, "devDependencies": { @@ -8391,6 +8393,18 @@ "node": ">=18.0.0" } }, + "node_modules/@googleapis/youtube": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@googleapis/youtube/-/youtube-20.0.0.tgz", + "integrity": "sha512-wdt1J0JoKYhvpoS2XIRHX0g/9ul/B0fQeeJAhuuBIdYINuuLt6/oZYZZCBmkuhtkA3IllXgqgAXOjLtLRAnR2g==", + "license": "Apache-2.0", + "dependencies": { + "googleapis-common": "^7.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.9.15", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.9.15.tgz", @@ -35501,6 +35515,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/youtube-transcript": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/youtube-transcript/-/youtube-transcript-1.2.1.tgz", + "integrity": "sha512-TvEGkBaajKw+B6y91ziLuBLsa5cawgowou+Bk0ciGpjELDfAzSzTGXaZmeSSkUeknCPpEr/WGApOHDwV7V+Y9Q==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/zod": { "version": "3.23.8", "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", @@ -35528,7 +35551,7 @@ }, "packages/data-provider": { "name": "librechat-data-provider", - "version": "0.7.697", + "version": "0.7.698", "license": "ISC", "dependencies": { "axios": "^1.7.7", diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index da33713a3..e2d63ce1f 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.7.697", + "version": "0.7.698", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js", diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index 7cf930343..6bfe788ab 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -386,6 +386,7 @@ export const tPluginSchema = z.object({ authConfig: z.array(tPluginAuthConfigSchema).optional(), authenticated: z.boolean().optional(), isButton: z.boolean().optional(), + toolkit: z.boolean().optional(), }); export type TPlugin = z.infer; @@ -462,7 +463,6 @@ export const tMessageSchema = z.object({ sender: z.string().optional(), text: z.string(), generation: z.string().nullable().optional(), - isEdited: z.boolean().optional(), isCreatedByUser: z.boolean(), error: z.boolean().optional(), clientTimestamp: z.string().optional(),