diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js index 853d0e12d..5e9bde143 100644 --- a/api/app/clients/PluginsClient.js +++ b/api/app/clients/PluginsClient.js @@ -357,6 +357,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s const trimmedPartial = opts.getPartialText().replaceAll(':::plugin:::\n', ''); responseMessage.text = trimmedPartial.length === 0 ? `${partialText}${this.result.output}` : partialText; + addImages(this.result.intermediateSteps, responseMessage); await this.generateTextStream(this.result.output, opts.onProgress, { delay: 5 }); return await this.handleResponseMessage(responseMessage, saveOptions, user); } diff --git a/api/app/clients/output_parsers/addImages.js b/api/app/clients/output_parsers/addImages.js index 02bf05dbe..b64dc16d4 100644 --- a/api/app/clients/output_parsers/addImages.js +++ b/api/app/clients/output_parsers/addImages.js @@ -1,23 +1,68 @@ +/** + * The `addImages` function corrects any erroneous image URLs in the `responseMessage.text` + * and appends image observations from `intermediateSteps` if they are not already present. + * + * @function + * @module addImages + * + * @param {Array.} intermediateSteps - An array of objects, each containing an observation. + * @param {Object} responseMessage - An object containing the text property which might have image URLs. + * + * @property {string} intermediateSteps[].observation - The observation string which might contain an image markdown. + * @property {string} responseMessage.text - The text which might contain image URLs. + * + * @example + * + * const intermediateSteps = [ + * { observation: '![desc](/images/test.png)' } + * ]; + * const responseMessage = { text: 'Some text with ![desc](sandbox:/images/test.png)' }; + * + * addImages(intermediateSteps, responseMessage); + * + * console.log(responseMessage.text); + * // Outputs: 'Some text with ![desc](/images/test.png)\n![desc](/images/test.png)' + * + * @returns {void} + */ function addImages(intermediateSteps, responseMessage) { if (!intermediateSteps || !responseMessage) { return; } + // Correct any erroneous URLs in the responseMessage.text first intermediateSteps.forEach((step) => { const { observation } = step; if (!observation || !observation.includes('![')) { return; } - // Extract the image file path from the observation - const observedImagePath = observation.match(/\(\/images\/.*\.\w*\)/g)[0]; + const match = observation.match(/\/images\/.*\.\w*/); + if (!match) { + return; + } + const essentialImagePath = match[0]; - // Check if the responseMessage already includes the image file path - if (!responseMessage.text.includes(observedImagePath)) { - // If the image file path is not found, append the whole observation + const regex = /!\[.*?\]\((.*?)\)/g; + let matchErroneous; + while ((matchErroneous = regex.exec(responseMessage.text)) !== null) { + if (matchErroneous[1] && !matchErroneous[1].startsWith('/images/')) { + responseMessage.text = responseMessage.text.replace(matchErroneous[1], essentialImagePath); + } + } + }); + + // Now, check if the responseMessage already includes the correct image file path and append if not + intermediateSteps.forEach((step) => { + const { observation } = step; + if (!observation || !observation.includes('![')) { + return; + } + const observedImagePath = observation.match(/\(\/images\/.*\.\w*\)/g); + if (observedImagePath && !responseMessage.text.includes(observedImagePath[0])) { responseMessage.text += '\n' + observation; - if (this.options.debug) { - console.debug('added image from intermediateSteps'); + if (process.env.DEBUG_PLUGINS) { + console.debug('[addImages] added image from intermediateSteps'); } } }); diff --git a/api/app/clients/output_parsers/addImages.spec.js b/api/app/clients/output_parsers/addImages.spec.js new file mode 100644 index 000000000..eb4d87d65 --- /dev/null +++ b/api/app/clients/output_parsers/addImages.spec.js @@ -0,0 +1,84 @@ +let addImages = require('./addImages'); + +describe('addImages', () => { + let intermediateSteps; + let responseMessage; + let options; + + beforeEach(() => { + intermediateSteps = []; + responseMessage = { text: '' }; + options = { debug: false }; + this.options = options; + addImages = addImages.bind(this); + }); + + it('should handle null or undefined parameters', () => { + addImages(null, responseMessage); + expect(responseMessage.text).toBe(''); + + addImages(intermediateSteps, null); + expect(responseMessage.text).toBe(''); + + addImages(null, null); + expect(responseMessage.text).toBe(''); + }); + + it('should append correct image markdown if not present in responseMessage', () => { + intermediateSteps.push({ observation: '![desc](/images/test.png)' }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe('\n![desc](/images/test.png)'); + }); + + it('should not append image markdown if already present in responseMessage', () => { + responseMessage.text = '![desc](/images/test.png)'; + intermediateSteps.push({ observation: '![desc](/images/test.png)' }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe('![desc](/images/test.png)'); + }); + + it('should correct and append image markdown with erroneous URL', () => { + responseMessage.text = '![desc](sandbox:/images/test.png)'; + intermediateSteps.push({ observation: '![desc](/images/test.png)' }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe('![desc](/images/test.png)'); + }); + + it('should correct multiple erroneous URLs in responseMessage', () => { + responseMessage.text = + '![desc1](sandbox:/images/test1.png) ![desc2](version:/images/test2.png)'; + intermediateSteps.push({ observation: '![desc1](/images/test1.png)' }); + intermediateSteps.push({ observation: '![desc2](/images/test2.png)' }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe('![desc1](/images/test1.png) ![desc2](/images/test2.png)'); + }); + + it('should not append non-image markdown observations', () => { + intermediateSteps.push({ observation: '[desc](/images/test.png)' }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe(''); + }); + + it('should handle multiple observations', () => { + intermediateSteps.push({ observation: '![desc1](/images/test1.png)' }); + intermediateSteps.push({ observation: '![desc2](/images/test2.png)' }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe('\n![desc1](/images/test1.png)\n![desc2](/images/test2.png)'); + }); + + it('should not append if observation does not contain image markdown', () => { + intermediateSteps.push({ observation: 'This is a test observation without image markdown.' }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe(''); + }); + + it('should append correctly from a real scenario', () => { + responseMessage.text = + 'Here is the generated image based on your request. It depicts a surreal landscape filled with floating musical notes. The style is impressionistic, with vibrant sunset hues dominating the scene. At the center, there\'s a silhouette of a grand piano, adding a dreamy emotion to the overall image. This could serve as a unique and creative music album cover. Would you like to make any changes or generate another image?'; + const originalText = responseMessage.text; + const imageMarkdown = '![generated image](/images/img-RnVWaYo2Yg4x3e0isICiMuf5.png)'; + intermediateSteps.push({ observation: imageMarkdown }); + addImages(intermediateSteps, responseMessage); + expect(responseMessage.text).toBe(`${originalText}\n${imageMarkdown}`); + }); +});