From d0ef0f84c85ff16201c45863eaed5edd490c6246 Mon Sep 17 00:00:00 2001
From: Danny Avila <messagedaniel@protonmail.com>
Date: Tue, 14 Mar 2023 16:05:46 -0400
Subject: [PATCH] chore: delegate text handling to one place, html sanitization
 in progress

---
 api/server/routes/ask.js       |  7 ++++---
 api/server/routes/askBing.js   |  3 ++-
 api/server/routes/askSydney.js |  3 ++-
 api/server/routes/handlers.js  | 24 +++++++++++++++++++-----
 4 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/api/server/routes/ask.js b/api/server/routes/ask.js
index 5f94d7d4c..8d20b808d 100644
--- a/api/server/routes/ask.js
+++ b/api/server/routes/ask.js
@@ -8,10 +8,10 @@ const {
   askClient,
   browserClient,
   customClient,
-  detectCode
+  // detectCode
 } = require('../../app/');
 const { getConvo, saveMessage, getConvoTitle, saveConvo } = require('../../models');
-const { handleError, sendMessage, createOnProgress } = require('./handlers');
+const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
 const { getMessages } = require('../../models/Message');
 
 router.use('/bing', askBing);
@@ -177,7 +177,8 @@ const ask = async ({
 
     gptResponse.sender = model === 'chatgptCustom' ? convo.chatGptLabel : model;
     // gptResponse.final = true;
-    gptResponse.text = await detectCode(gptResponse.text);
+    // gptResponse.text = await detectCode(gptResponse.text);
+    gptResponse.text = await handleText(gptResponse.text);
 
     if (convo.chatGptLabel?.length > 0 && model === 'chatgptCustom') {
       gptResponse.chatGptLabel = convo.chatGptLabel;
diff --git a/api/server/routes/askBing.js b/api/server/routes/askBing.js
index add804a43..5bb8abd4f 100644
--- a/api/server/routes/askBing.js
+++ b/api/server/routes/askBing.js
@@ -3,7 +3,7 @@ const crypto = require('crypto');
 const router = express.Router();
 const { titleConvo, getCitations, citeText, askBing } = require('../../app/');
 const { saveMessage, getConvoTitle, saveConvo } = require('../../models');
-const { handleError, sendMessage, createOnProgress } = require('./handlers');
+const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
 const citationRegex = /\[\^\d+?\^]/g;
 
 router.post('/', async (req, res) => {
@@ -113,6 +113,7 @@ const ask = async ({
     response.text =
       citeText(response) +
       (links?.length > 0 && hasCitations ? `\n<small>${links}</small>` : '');
+    response.text = await handleText(response.text);
 
     await saveMessage(response);
     await saveConvo({...response, model, ...convo});
diff --git a/api/server/routes/askSydney.js b/api/server/routes/askSydney.js
index 1f04ad416..a0a3bd32d 100644
--- a/api/server/routes/askSydney.js
+++ b/api/server/routes/askSydney.js
@@ -3,7 +3,7 @@ const crypto = require('crypto');
 const router = express.Router();
 const { titleConvo, getCitations, citeText, askSydney } = require('../../app/');
 const { saveMessage, saveConvo, getConvoTitle } = require('../../models');
-const { handleError, sendMessage, createOnProgress } = require('./handlers');
+const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
 const citationRegex = /\[\^\d+?\^]/g;
 
 router.post('/', async (req, res) => {
@@ -114,6 +114,7 @@ const ask = async ({
     response.text =
       citeText(response) +
       (links?.length > 0 && hasCitations ? `\n<small>${links}</small>` : '');
+    response.text = await handleText(response.text);
 
     // Save user message
     userMessage.conversationId = response.conversationId || conversationId;
diff --git a/api/server/routes/handlers.js b/api/server/routes/handlers.js
index 7672ef41a..d3e613bcc 100644
--- a/api/server/routes/handlers.js
+++ b/api/server/routes/handlers.js
@@ -1,4 +1,4 @@
-const { citeText } = require('../../app/');
+const { citeText, detectCode } = require('../../app/');
 const _ = require('lodash');
 const sanitizeHtml = require('sanitize-html');
 
@@ -20,11 +20,14 @@ const createOnProgress = () => {
 
   const progressCallback = async (partial, { res, text, bing = false, ...rest }) => {
     tokens += partial === text ? '' : partial;
-    tokens = tokens.trim();
     tokens = tokens.replaceAll('[DONE]', '');
-    if (tokens.includes('```')) {
-      tokens = sanitizeHtml(tokens);
+
+    if (tokens.match(/^\n/)) {
+      tokens = tokens.replace(/^\n/, '');
     }
+    // if (tokens.includes('```')) {
+    //   tokens = sanitizeHtml(tokens);
+    // }
 
     if (bing) {
       tokens = citeText(tokens, true);
@@ -42,4 +45,15 @@ const createOnProgress = () => {
   return onProgress;
 };
 
-module.exports = { handleError, sendMessage, createOnProgress };
\ No newline at end of file
+const handleText = async (input) => {
+  let text = input;
+  text = await detectCode(text);
+  // if (text.includes('```')) {
+  //   text = sanitizeHtml(text);
+  //   text = text.replaceAll(') =&gt;', ') =>');
+  // }
+
+  return text;
+};
+
+module.exports = { handleError, sendMessage, createOnProgress, handleText };
\ No newline at end of file