From 1d021d3ef3407845ea09a9cd5f7bc56042b4c39d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 24 Oct 2024 15:50:48 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Minor=20Vulnerab?= =?UTF-8?q?ilities=20(#4543)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: ReDoS in ChatGPT Import * ci: should correctly process citations from real ChatGPT data * ci: Add ReDoS vulnerability test for processAssistantMessage * refactor: Update thread management and citation handling * refactor(validateImageRequest): robust validation * refactor(Prompt.js): update name search regex to escape special characters * refactor(Preset): exclude user from preset update to prevent mass assignment * refactor(files.js): Improve file deletion process * ci: updated validateImageRequest.spec.js * a11y: plugin pagination * refactor(CreatePromptForm.tsx): Improve input field styling * chore(Prompts): typing and accessibility * fix: prompt creation access role check * chore: remove duplicate jsdocs --- api/models/Preset.js | 3 +- api/models/Prompt.js | 5 +- .../middleware/spec/validateImages.spec.js | 37 +- api/server/middleware/validateImageRequest.js | 23 ++ api/server/routes/files/files.js | 8 +- api/server/routes/prompts.js | 2 +- api/server/services/Threads/manage.js | 10 +- api/server/utils/citations.js | 10 +- .../import/__data__/chatgpt-citations.json | 345 ++++++++++++++++++ api/server/utils/import/importers.js | 33 +- api/server/utils/import/importers.spec.js | 233 +++++++++++- .../Plugins/Store/PluginPagination.tsx | 28 +- .../Prompts/Groups/CreatePromptForm.tsx | 6 +- .../Prompts/Groups/FilterPrompts.tsx | 6 +- client/src/hooks/Prompts/useCategories.tsx | 2 +- 15 files changed, 698 insertions(+), 53 deletions(-) create mode 100644 api/server/utils/import/__data__/chatgpt-citations.json diff --git a/api/models/Preset.js b/api/models/Preset.js index 8a300614954..0a00dfbfcfb 100644 --- a/api/models/Preset.js +++ b/api/models/Preset.js @@ -78,7 +78,8 @@ module.exports = { } const setter = { $set: {} }; - const update = { presetId, ...preset, user }; + const { user: _, ...cleanPreset } = preset; + const update = { presetId, ...cleanPreset }; if (preset.tools && Array.isArray(preset.tools)) { update.tools = preset.tools diff --git a/api/models/Prompt.js b/api/models/Prompt.js index 548589b4d7e..742ced0f728 100644 --- a/api/models/Prompt.js +++ b/api/models/Prompt.js @@ -7,6 +7,7 @@ const { removeGroupFromAllProjects, } = require('./Project'); const { Prompt, PromptGroup } = require('./schema/promptSchema'); +const { escapeRegExp } = require('~/server/utils'); const { logger } = require('~/config'); /** @@ -106,7 +107,7 @@ const getAllPromptGroups = async (req, filter) => { let searchShared = true; let searchSharedOnly = false; if (name) { - query.name = new RegExp(name, 'i'); + query.name = new RegExp(escapeRegExp(name), 'i'); } if (!query.category) { delete query.category; @@ -159,7 +160,7 @@ const getPromptGroups = async (req, filter) => { let searchShared = true; let searchSharedOnly = false; if (name) { - query.name = new RegExp(name, 'i'); + query.name = new RegExp(escapeRegExp(name), 'i'); } if (!query.category) { delete query.category; diff --git a/api/server/middleware/spec/validateImages.spec.js b/api/server/middleware/spec/validateImages.spec.js index ebc6fa60a30..8b04ac931f2 100644 --- a/api/server/middleware/spec/validateImages.spec.js +++ b/api/server/middleware/spec/validateImages.spec.js @@ -3,6 +3,7 @@ const validateImageRequest = require('~/server/middleware/validateImageRequest') describe('validateImageRequest middleware', () => { let req, res, next; + const validObjectId = '65cfb246f7ecadb8b1e8036b'; beforeEach(() => { req = { @@ -43,7 +44,7 @@ describe('validateImageRequest middleware', () => { test('should return 403 if refresh token is expired', () => { const expiredToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) - 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) - 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${expiredToken}`; @@ -54,22 +55,34 @@ describe('validateImageRequest middleware', () => { test('should call next() for valid image path', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; - req.originalUrl = '/images/123/example.jpg'; + req.originalUrl = `/images/${validObjectId}/example.jpg`; validateImageRequest(req, res, next); expect(next).toHaveBeenCalled(); }); test('should return 403 for invalid image path', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; - req.originalUrl = '/images/456/example.jpg'; + req.originalUrl = '/images/65cfb246f7ecadb8b1e8036c/example.jpg'; // Different ObjectId + validateImageRequest(req, res, next); + expect(res.status).toHaveBeenCalledWith(403); + expect(res.send).toHaveBeenCalledWith('Access Denied'); + }); + + test('should return 403 for invalid ObjectId format', () => { + const validToken = jwt.sign( + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, + process.env.JWT_REFRESH_SECRET, + ); + req.headers.cookie = `refreshToken=${validToken}`; + req.originalUrl = '/images/123/example.jpg'; // Invalid ObjectId validateImageRequest(req, res, next); expect(res.status).toHaveBeenCalledWith(403); expect(res.send).toHaveBeenCalledWith('Access Denied'); @@ -78,16 +91,16 @@ describe('validateImageRequest middleware', () => { // File traversal tests test('should prevent file traversal attempts', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; const traversalAttempts = [ - '/images/123/../../../etc/passwd', - '/images/123/..%2F..%2F..%2Fetc%2Fpasswd', - '/images/123/image.jpg/../../../etc/passwd', - '/images/123/%2e%2e%2f%2e%2e%2f%2e%2e%2fetc%2fpasswd', + `/images/${validObjectId}/../../../etc/passwd`, + `/images/${validObjectId}/..%2F..%2F..%2Fetc%2Fpasswd`, + `/images/${validObjectId}/image.jpg/../../../etc/passwd`, + `/images/${validObjectId}/%2e%2e%2f%2e%2e%2f%2e%2e%2fetc%2fpasswd`, ]; traversalAttempts.forEach((attempt) => { @@ -101,11 +114,11 @@ describe('validateImageRequest middleware', () => { test('should handle URL encoded characters in valid paths', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; - req.originalUrl = '/images/123/image%20with%20spaces.jpg'; + req.originalUrl = `/images/${validObjectId}/image%20with%20spaces.jpg`; validateImageRequest(req, res, next); expect(next).toHaveBeenCalled(); }); diff --git a/api/server/middleware/validateImageRequest.js b/api/server/middleware/validateImageRequest.js index e07e48cc716..eb37b9dbb56 100644 --- a/api/server/middleware/validateImageRequest.js +++ b/api/server/middleware/validateImageRequest.js @@ -2,6 +2,24 @@ const cookies = require('cookie'); const jwt = require('jsonwebtoken'); const { logger } = require('~/config'); +const OBJECT_ID_LENGTH = 24; +const OBJECT_ID_PATTERN = /^[0-9a-f]{24}$/i; + +/** + * Validates if a string is a valid MongoDB ObjectId + * @param {string} id - String to validate + * @returns {boolean} - Whether string is a valid ObjectId format + */ +function isValidObjectId(id) { + if (typeof id !== 'string') { + return false; + } + if (id.length !== OBJECT_ID_LENGTH) { + return false; + } + return OBJECT_ID_PATTERN.test(id); +} + /** * Middleware to validate image request. * Must be set by `secureImageLinks` via custom config file. @@ -25,6 +43,11 @@ function validateImageRequest(req, res, next) { return res.status(403).send('Access Denied'); } + if (!isValidObjectId(payload.id)) { + logger.warn('[validateImageRequest] Invalid User ID'); + return res.status(403).send('Access Denied'); + } + const currentTimeInSeconds = Math.floor(Date.now() / 1000); if (payload.exp < currentTimeInSeconds) { logger.warn('[validateImageRequest] Refresh token expired'); diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index 8fe5eb70d44..6d4ab7fefb7 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -66,7 +66,13 @@ router.delete('/', async (req, res) => { return; } - await processDeleteRequest({ req, files }); + const fileIds = files.map((file) => file.file_id); + const userFiles = await getFiles({ file_id: { $in: fileIds }, user: req.user.id }); + if (userFiles.length !== files.length) { + return res.status(403).json({ message: 'You can only delete your own files' }); + } + + await processDeleteRequest({ req, files: userFiles }); logger.debug( `[/files] Files deleted successfully: ${files diff --git a/api/server/routes/prompts.js b/api/server/routes/prompts.js index 54128d3b395..c78591265d4 100644 --- a/api/server/routes/prompts.js +++ b/api/server/routes/prompts.js @@ -134,7 +134,7 @@ const createPrompt = async (req, res) => { } }; -router.post('/', createPrompt); +router.post('/', checkPromptCreate, createPrompt); /** * Updates a prompt group diff --git a/api/server/services/Threads/manage.js b/api/server/services/Threads/manage.js index 5155d611092..8dbac189ab4 100644 --- a/api/server/services/Threads/manage.js +++ b/api/server/services/Threads/manage.js @@ -8,9 +8,9 @@ const { } = require('librechat-data-provider'); const { retrieveAndProcessFile } = require('~/server/services/Files/process'); const { recordMessage, getMessages } = require('~/models/Message'); +const { countTokens, escapeRegExp } = require('~/server/utils'); const { spendTokens } = require('~/models/spendTokens'); const { saveConvo } = require('~/models/Conversation'); -const { countTokens } = require('~/server/utils'); /** * Initializes a new thread or adds messages to an existing thread. @@ -518,14 +518,6 @@ const recordUsage = async ({ const uniqueCitationStart = '^====||==='; const uniqueCitationEnd = '==|||||^'; -/** Helper function to escape special characters in regex - * @param {string} string - The string to escape. - * @returns {string} The escaped string. - */ -function escapeRegExp(string) { - return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - /** * Sorts, processes, and flattens messages to a single string. * diff --git a/api/server/utils/citations.js b/api/server/utils/citations.js index 33136c18b8d..97cb69aca77 100644 --- a/api/server/utils/citations.js +++ b/api/server/utils/citations.js @@ -1,6 +1,14 @@ const citationRegex = /\[\^\d+?\^\]/g; const regex = / \[.*?]\(.*?\)/g; +/** Helper function to escape special characters in regex + * @param {string} string - The string to escape. + * @returns {string} The escaped string. + */ +function escapeRegExp(string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + const getCitations = (res) => { const adaptiveCards = res.details.adaptiveCards; const textBlocks = adaptiveCards && adaptiveCards[0].body; @@ -47,4 +55,4 @@ const citeText = (res, noLinks = false) => { return result; }; -module.exports = { getCitations, citeText }; +module.exports = { getCitations, citeText, escapeRegExp }; diff --git a/api/server/utils/import/__data__/chatgpt-citations.json b/api/server/utils/import/__data__/chatgpt-citations.json new file mode 100644 index 00000000000..5a08631a6f9 --- /dev/null +++ b/api/server/utils/import/__data__/chatgpt-citations.json @@ -0,0 +1,345 @@ +[ + { + "title": "Citation Convo", + "create_time": 1704629915.775304, + "update_time": 1704717442.442031, + "mapping": { + "9e874379-5008-4a2d-aa2e-628d1d705a04": { + "id": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "message": { + "id": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "author": { + "role": "tool", + "name": "browser", + "metadata": {} + }, + "create_time": 1708580164.172007, + "update_time": null, + "content": { + "content_type": "tether_browsing_display", + "result": "# 【0†Signal Sciences - Crunchbase Company Profile & Funding†www.crunchbase.com】\nFounded Date Mar 10, 2014. Founders Andrew Peterson, Nick Galbreath, Zane Lackey. Operating Status Active. Last Funding Type Series C. Legal Name Signal Sciences Corp. Company Type For Profit. Contact Email info@signalsciences.com. Phone Number +1 424-404-1139. Signal Sciences is the fastest growing web application security company in the world.\n# 【1†Demand More from Your WAF - Signal Sciences now part of Fastly†www.signalsciences.com】\nModernize Your Application/API Protection While Lowering Your TCO. Legacy WAFs weren't designed for today's web apps that are distributed across cloud, on-premise, edge or hybrid environments. Our next-gen web application firewall (NGWAF) and runtime application self protection (RASP) increase security and maintain reliability without ...\n# 【2†Security at Scale: Fastly Announces Intent to Acquire Signal Science†www.fastly.com】\nSignal Sciences' technology combined with Fastly's current solutions will form Fastly's upcoming new security offering, called Secure@Edge. Secure@Edge will be a modern, unified web application and API security solution that will integrate with our Compute platform, and power and protect companies looking to further or begin their digital ...\n# 【3†Web Application and API Protection - Signal Sciences now part of Fastly†www.signalsciences.com】\nOur customers have spoken: Signal Sciences is the highest rated Web Application Firewalls (WAF) vendor. We were named a 2021 Gartner Peer Insights Customers' Choice for WAF based on detailed end-user testimonials. In addition, we are the only vendor to be named a Customers' Choice for three consecutive times and have an overall rating of 5 ...\n# 【4†Fastly Completes Acquisition of Signal Sciences | Fastly†www.fastly.com】\nSAN FRANCISCO, October 1, 2020 – Fastly, Inc. (NYSE: FSLY), provider of an edge cloud platform, today announced it has successfully completed the acquisition of Signal Sciences (“Signal Sciences”). The transaction combines Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings to form a unified suite of ...\n# 【5†Signal Sciences: Enterprises still overlooking web app security†www.techtarget.com】\nRansomware threats are everywhere these days, but one startup is focusing on the most common cause behind data breaches. Signal Sciences is a Web Application Firewall (WAF) and Runtime Application Self-Protection (RASP) company that protects customers' web applications and APIs, regardless of the software architecture or deployment model used, such as on-premises, cloud, containers or hybrid ...\n# 【6†Signal Sciences Launches Cloud WAF for Easy Web Application Security†www.signalsciences.com】\nToday we are proud to announce the launch of the new Signal Sciences Cloud Web Application Firewall (WAF) deployment option for our award-winning WAF and RASP solution.As the fastest growing web application security company in the world, we know organizations must quickly detect and stop web application layer attacks wherever their apps, APIs, or microservices operate—whether that be in the ...\n# 【7†Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley†www.cooley.com】\nSan Francisco – August 27, 2020 – Cooley advised edge cloud platform provider Fastly on its agreement to acquire Signal Sciences, one of the fastest growing web application security companies in the world, for approximately $775 million in cash and stock.The acquisition, which is expected to close this year, will expand Fastly's robust security portfolio at a time when security at the ...\n# 【8†Signal Sciences†info.signalsciences.com】\nSignal Sciences next-gen WAF and RASP technology is designed to work quickly and effectively, enabling application developers and operations teams to deliver modern, business-critical web applications and APIs while ensuring they're well protected and running performantly.. There are many vendors claiming to provide effective and scalable offerings to protect applications and APIs, so we ...\n# 【9†Gartner names Signal Sciences a Visionary in the 2020 Magic ... - Fastly†www.fastly.com】\nWe believe Signal Sciences' innovation earned them recognition in the 2019 Gartner Magic Quadrant for WAF, ... Gartner research publications consist of the opinions of Gartner's research organization and should not be construed as statements of fact. Gartner disclaims all warranties, express or implied, with respect to this research ...\n# 【10†Fastly acquires Signal Sciences - 2020-08-27 - Crunchbase†www.crunchbase.com】\nSignal Sciences Signal Sciences is a software as a service platform providing security monitoring and defense for your web applications. Acquiring Organization: Fastly Fastly helps digital businesses keep pace with their customer expectations by delivering secure and online experiences. Announced Date Aug 27, 2020;\n# 【11†Signal Sciences - Funding, Financials, Valuation & Investors - Crunchbase†www.crunchbase.com】\nFunding. Signal Sciences has raised a total of. $61.7M. in funding over 4 rounds. Their latest funding was raised on Feb 5, 2019 from a Series C round. Signal Sciences is funded by 8 investors. CRV and Harrison Metal are the most recent investors. Signal Sciences has invested in Open Raven on Feb 11, 2020. This investment - Seed Round - Open ...\n# 【12†Signal Sciences WAF Review and Alternatives - Comparitech†www.comparitech.com】\nSignal Sciences WAF is completely agentless as it works in the cloud to apply both managed and custom rulesets to your application traffic. Protections extend beyond the OWASP Top 10 and include defense from botnet attacks, account takeovers, credential stuffing, API abuse, and DDoS mitigation. Automatic blocking and scanning are applied to ...\n# 【13†Integrations - Web Application Security - Signal Sciences now part of ...†www.signalsciences.com】\nThe Signal Sciences site extension protects any IIS web application hosted on AAS and provides production visibility necessary to detect and block malicious web requests before they can compromise your business-critical apps, APIs and microservices. ... and efficiently into an organization's existing hybrid, or cloud architecture ...\n# 【14†Fastly Next-Gen WAF professional services | Fastly Products†docs.fastly.com】\nFastly Next-Gen WAF (powered by Signal Sciences) (Next-Gen WAF) professional services provide your organization with training, implementation, and maintenance services for the Next-Gen WAF. Depending on the service offerings you select, our team will provide training and work with you to plan, test, deploy, and maintain a solution to protect ...\n# 【15†Andrew Peterson - Founder & CEO @ Signal Sciences - Crunchbase†www.crunchbase.com】\nAndrew Peterson is the founder and CEO of Signal Sciences, a Software-as-a-Service platform that provides security monitoring and defense solutions for web applications. He was previously the group product manager of international growth at Etsy. Prior to Etsy, Peterson was a health information management officer at the Clinton Foundation\n# 【16†Signal Sciences†info.signalsciences.com】\nThe webinar will cover: The most common attacks targeting distributed web apps. How a web application firewall (WAF) can inspect and decision on both north-south (client-to-app origin) and east-west (service-to-service) web requests. How to detect and prevent malicious client-side activity originating from website supply chain vendors. A live ...\n# 【17†Next-Generation Cloud Web Application Firewall (WAF) - Signal Sciences ...†www.signalsciences.com】\nWith a single change to a DNS record, web requests are routed to the Signal Sciences Cloud WAF to inspect and decision on those requests and block bad requests. Our unique solution provides robust detection and protection capabilities no matter where your organization's applications operate or how rapidly they scale. No complex software ...\n# 【18†How to Configure SAML 2.0 for Signal Sciences - UserDocs†saml-doc.okta.com】\nConfiguration Steps. Log in to Signal Sciences as an Organization Administrator. Navigate to organization > Settings.. Click the Switch to SAML button:. Click the I understand, configure SSO button:. On the Configure SSO with SAML page, enter the following:. Copy and paste the following IDP Login page into the SAML endpoint field:. Sign into the Okta Admin Dashboard to generate this variable.\n# 【19†Powering the best of the internet | Fastly†www.fastly.com】\nEverybody needs speed, reliability, security, savings, and scale – but different industries have different needs. Fastly's powerful network and smarter solutions can be tailored to your organization. We partner with you to guarantee a smooth migration, so you can deliver the best possible user experiences.\nVisible: 0% - 100%", + "summary": null, + "assets": [], + "tether_id": null + }, + "status": "finished_successfully", + "end_turn": null, + "weight": 0, + "metadata": { + "_cite_metadata": { + "citation_format": { + "name": "tether_og", + "regex": "【(\\d+)(?::(\\d+))?†([^†【】]*)[^【】]*?】" + }, + "metadata_list": [ + { + "type": "webpage", + "title": "Signal Sciences - Crunchbase Company Profile & Funding", + "url": "https://www.crunchbase.com/organization/signal-sciences", + "text": "\nFounded Date Mar 10, 2014. Founders Andrew Peterson, Nick Galbreath, Zane Lackey. Operating Status Active. Last Funding Type Series C. Legal Name Signal Sciences Corp. Company Type For Profit. Contact Email info@signalsciences.com. Phone Number +1 424-404-1139. Signal Sciences is the fastest growing web application security company in the world.\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Demand More from Your WAF - Signal Sciences now part of Fastly", + "url": "https://www.signalsciences.com/", + "text": "\nModernize Your Application/API Protection While Lowering Your TCO. Legacy WAFs weren't designed for today's web apps that are distributed across cloud, on-premise, edge or hybrid environments. Our next-gen web application firewall (NGWAF) and runtime application self protection (RASP) increase security and maintain reliability without ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Security at Scale: Fastly Announces Intent to Acquire Signal Science", + "url": "https://www.fastly.com/blog/fastly-intent-to-acquire-signal-sciences", + "text": "\nSignal Sciences' technology combined with Fastly's current solutions will form Fastly's upcoming new security offering, called Secure@Edge. Secure@Edge will be a modern, unified web application and API security solution that will integrate with our Compute platform, and power and protect companies looking to further or begin their digital ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Web Application and API Protection - Signal Sciences now part of Fastly", + "url": "https://www.signalsciences.com/products/", + "text": "\nOur customers have spoken: Signal Sciences is the highest rated Web Application Firewalls (WAF) vendor. We were named a 2021 Gartner Peer Insights Customers' Choice for WAF based on detailed end-user testimonials. In addition, we are the only vendor to be named a Customers' Choice for three consecutive times and have an overall rating of 5 ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly Completes Acquisition of Signal Sciences | Fastly", + "url": "https://www.fastly.com/press/press-releases/fastly-completes-acquisition-signal-sciences", + "text": "\nSAN FRANCISCO, October 1, 2020 – Fastly, Inc. (NYSE: FSLY), provider of an edge cloud platform, today announced it has successfully completed the acquisition of Signal Sciences (“Signal Sciences”). The transaction combines Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings to form a unified suite of ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences: Enterprises still overlooking web app security", + "url": "https://www.techtarget.com/searchsecurity/news/252476659/Signal-Sciences-Enterprises-still-overlooking-web-app-security", + "text": "\nRansomware threats are everywhere these days, but one startup is focusing on the most common cause behind data breaches. Signal Sciences is a Web Application Firewall (WAF) and Runtime Application Self-Protection (RASP) company that protects customers' web applications and APIs, regardless of the software architecture or deployment model used, such as on-premises, cloud, containers or hybrid ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences Launches Cloud WAF for Easy Web Application Security", + "url": "https://www.signalsciences.com/blog/cloud-waf-web-applications/", + "text": "\nToday we are proud to announce the launch of the new Signal Sciences Cloud Web Application Firewall (WAF) deployment option for our award-winning WAF and RASP solution.As the fastest growing web application security company in the world, we know organizations must quickly detect and stop web application layer attacks wherever their apps, APIs, or microservices operate—whether that be in the ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley", + "url": "https://www.cooley.com/news/coverage/2020/2020-08-27-fastly-agrees-to-acquire-signal-sciences-for-775-million", + "text": "\nSan Francisco – August 27, 2020 – Cooley advised edge cloud platform provider Fastly on its agreement to acquire Signal Sciences, one of the fastest growing web application security companies in the world, for approximately $775 million in cash and stock.The acquisition, which is expected to close this year, will expand Fastly's robust security portfolio at a time when security at the ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences", + "url": "https://info.signalsciences.com/10-key-capabilities-of-signal-sciences", + "text": "\nSignal Sciences next-gen WAF and RASP technology is designed to work quickly and effectively, enabling application developers and operations teams to deliver modern, business-critical web applications and APIs while ensuring they're well protected and running performantly.. There are many vendors claiming to provide effective and scalable offerings to protect applications and APIs, so we ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Gartner names Signal Sciences a Visionary in the 2020 Magic ... - Fastly", + "url": "https://www.fastly.com/blog/gartner-names-signal-sciences-a-visionary-in-the-2020-magic-quadrant-for-waf", + "text": "\nWe believe Signal Sciences' innovation earned them recognition in the 2019 Gartner Magic Quadrant for WAF, ... Gartner research publications consist of the opinions of Gartner's research organization and should not be construed as statements of fact. Gartner disclaims all warranties, express or implied, with respect to this research ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly acquires Signal Sciences - 2020-08-27 - Crunchbase", + "url": "https://www.crunchbase.com/acquisition/fastly-acquires-signal-sciences--2b836efb", + "text": "\nSignal Sciences Signal Sciences is a software as a service platform providing security monitoring and defense for your web applications. Acquiring Organization: Fastly Fastly helps digital businesses keep pace with their customer expectations by delivering secure and online experiences. Announced Date Aug 27, 2020;\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences - Funding, Financials, Valuation & Investors - Crunchbase", + "url": "https://www.crunchbase.com/organization/signal-sciences/company_financials", + "text": "\nFunding. Signal Sciences has raised a total of. $61.7M. in funding over 4 rounds. Their latest funding was raised on Feb 5, 2019 from a Series C round. Signal Sciences is funded by 8 investors. CRV and Harrison Metal are the most recent investors. Signal Sciences has invested in Open Raven on Feb 11, 2020. This investment - Seed Round - Open ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences WAF Review and Alternatives - Comparitech", + "url": "https://www.comparitech.com/net-admin/signal-sciences-waf-review/", + "text": "\nSignal Sciences WAF is completely agentless as it works in the cloud to apply both managed and custom rulesets to your application traffic. Protections extend beyond the OWASP Top 10 and include defense from botnet attacks, account takeovers, credential stuffing, API abuse, and DDoS mitigation. Automatic blocking and scanning are applied to ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Integrations - Web Application Security - Signal Sciences now part of ...", + "url": "https://www.signalsciences.com/integrations/", + "text": "\nThe Signal Sciences site extension protects any IIS web application hosted on AAS and provides production visibility necessary to detect and block malicious web requests before they can compromise your business-critical apps, APIs and microservices. ... and efficiently into an organization's existing hybrid, or cloud architecture ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly Next-Gen WAF professional services | Fastly Products", + "url": "https://docs.fastly.com/products/fastly-next-gen-waf-professional-services", + "text": "\nFastly Next-Gen WAF (powered by Signal Sciences) (Next-Gen WAF) professional services provide your organization with training, implementation, and maintenance services for the Next-Gen WAF. Depending on the service offerings you select, our team will provide training and work with you to plan, test, deploy, and maintain a solution to protect ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Andrew Peterson - Founder & CEO @ Signal Sciences - Crunchbase", + "url": "https://www.crunchbase.com/person/andrew-peterson", + "text": "\nAndrew Peterson is the founder and CEO of Signal Sciences, a Software-as-a-Service platform that provides security monitoring and defense solutions for web applications. He was previously the group product manager of international growth at Etsy. Prior to Etsy, Peterson was a health information management officer at the Clinton Foundation\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences", + "url": "https://info.signalsciences.com/preventing-modern-web-app-attacks-source-defense", + "text": "\nThe webinar will cover: The most common attacks targeting distributed web apps. How a web application firewall (WAF) can inspect and decision on both north-south (client-to-app origin) and east-west (service-to-service) web requests. How to detect and prevent malicious client-side activity originating from website supply chain vendors. A live ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Next-Generation Cloud Web Application Firewall (WAF) - Signal Sciences ...", + "url": "https://www.signalsciences.com/products/cloud-waf/", + "text": "\nWith a single change to a DNS record, web requests are routed to the Signal Sciences Cloud WAF to inspect and decision on those requests and block bad requests. Our unique solution provides robust detection and protection capabilities no matter where your organization's applications operate or how rapidly they scale. No complex software ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "How to Configure SAML 2.0 for Signal Sciences - UserDocs", + "url": "https://saml-doc.okta.com/SAML_Docs/How-to-Configure-SAML-2.0-for-Signal-Sciences.html", + "text": "\nConfiguration Steps. Log in to Signal Sciences as an Organization Administrator. Navigate to organization > Settings.. Click the Switch to SAML button:. Click the I understand, configure SSO button:. On the Configure SSO with SAML page, enter the following:. Copy and paste the following IDP Login page into the SAML endpoint field:. Sign into the Okta Admin Dashboard to generate this variable.\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Powering the best of the internet | Fastly", + "url": "https://www.fastly.com/", + "text": "\nEverybody needs speed, reliability, security, savings, and scale – but different industries have different needs. Fastly's powerful network and smarter solutions can be tailored to your organization. We partner with you to guarantee a smooth migration, so you can deliver the best possible user experiences.\nVisible: 0% - 100%", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Search results for query: 'Signal Sciences organization'", + "url": "", + "text": "# 【0†Signal Sciences - Crunchbase Company Profile & Funding†www.crunchbase.com】\nFounded Date Mar 10, 2014. Founders Andrew Peterson, Nick Galbreath, Zane Lackey. Operating Status Active. Last Funding Type Series C. Legal Name Signal Sciences Corp. Company Type For Profit. Contact Email info@signalsciences.com. Phone Number +1 424-404-1139. Signal Sciences is the fastest growing web application security company in the world.\n# 【1†Demand More from Your WAF - Signal Sciences now part of Fastly†www.signalsciences.com】\nModernize Your Application/API Protection While Lowering Your TCO. Legacy WAFs weren't designed for today's web apps that are distributed across cloud, on-premise, edge or hybrid environments. Our next-gen web application firewall (NGWAF) and runtime application self protection (RASP) increase security and maintain reliability without ...\n# 【2†Security at Scale: Fastly Announces Intent to Acquire Signal Science†www.fastly.com】\nSignal Sciences' technology combined with Fastly's current solutions will form Fastly's upcoming new security offering, called Secure@Edge. Secure@Edge will be a modern, unified web application and API security solution that will integrate with our Compute platform, and power and protect companies looking to further or begin their digital ...\n# 【3†Web Application and API Protection - Signal Sciences now part of Fastly†www.signalsciences.com】\nOur customers have spoken: Signal Sciences is the highest rated Web Application Firewalls (WAF) vendor. We were named a 2021 Gartner Peer Insights Customers' Choice for WAF based on detailed end-user testimonials. In addition, we are the only vendor to be named a Customers' Choice for three consecutive times and have an overall rating of 5 ...\n# 【4†Fastly Completes Acquisition of Signal Sciences | Fastly†www.fastly.com】\nSAN FRANCISCO, October 1, 2020 – Fastly, Inc. (NYSE: FSLY), provider of an edge cloud platform, today announced it has successfully completed the acquisition of Signal Sciences (“Signal Sciences”). The transaction combines Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings to form a unified suite of ...\n# 【5†Signal Sciences: Enterprises still overlooking web app security†www.techtarget.com】\nRansomware threats are everywhere these days, but one startup is focusing on the most common cause behind data breaches. Signal Sciences is a Web Application Firewall (WAF) and Runtime Application Self-Protection (RASP) company that protects customers' web applications and APIs, regardless of the software architecture or deployment model used, such as on-premises, cloud, containers or hybrid ...\n# 【6†Signal Sciences Launches Cloud WAF for Easy Web Application Security†www.signalsciences.com】\nToday we are proud to announce the launch of the new Signal Sciences Cloud Web Application Firewall (WAF) deployment option for our award-winning WAF and RASP solution.As the fastest growing web application security company in the world, we know organizations must quickly detect and stop web application layer attacks wherever their apps, APIs, or microservices operate—whether that be in the ...\n# 【7†Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley†www.cooley.com】\nSan Francisco – August 27, 2020 – Cooley advised edge cloud platform provider Fastly on its agreement to acquire Signal Sciences, one of the fastest growing web application security companies in the world, for approximately $775 million in cash and stock.The acquisition, which is expected to close this year, will expand Fastly's robust security portfolio at a time when security at the ...\n# 【8†Signal Sciences†info.signalsciences.com】\nSignal Sciences next-gen WAF and RASP technology is designed to work quickly and effectively, enabling application developers and operations teams to deliver modern, business-critical web applications and APIs while ensuring they're well protected and running performantly.. There are many vendors claiming to provide effective and scalable offerings to protect applications and APIs, so we ...\n# 【9†Gartner names Signal Sciences a Visionary in the 2020 Magic ... - Fastly†www.fastly.com】\nWe believe Signal Sciences' innovation earned them recognition in the 2019 Gartner Magic Quadrant for WAF, ... Gartner research publications consist of the opinions of Gartner's research organization and should not be construed as statements of fact. Gartner disclaims all warranties, express or implied, with respect to this research ...\n# 【10†Fastly acquires Signal Sciences - 2020-08-27 - Crunchbase†www.crunchbase.com】\nSignal Sciences Signal Sciences is a software as a service platform providing security monitoring and defense for your web applications. Acquiring Organization: Fastly Fastly helps digital businesses keep pace with their customer expectations by delivering secure and online experiences. Announced Date Aug 27, 2020;\n# 【11†Signal Sciences - Funding, Financials, Valuation & Investors - Crunchbase†www.crunchbase.com】\nFunding. Signal Sciences has raised a total of. $61.7M. in funding over 4 rounds. Their latest funding was raised on Feb 5, 2019 from a Series C round. Signal Sciences is funded by 8 investors. CRV and Harrison Metal are the most recent investors. Signal Sciences has invested in Open Raven on Feb 11, 2020. This investment - Seed Round - Open ...\n# 【12†Signal Sciences WAF Review and Alternatives - Comparitech†www.comparitech.com】\nSignal Sciences WAF is completely agentless as it works in the cloud to apply both managed and custom rulesets to your application traffic. Protections extend beyond the OWASP Top 10 and include defense from botnet attacks, account takeovers, credential stuffing, API abuse, and DDoS mitigation. Automatic blocking and scanning are applied to ...\n# 【13†Integrations - Web Application Security - Signal Sciences now part of ...†www.signalsciences.com】\nThe Signal Sciences site extension protects any IIS web application hosted on AAS and provides production visibility necessary to detect and block malicious web requests before they can compromise your business-critical apps, APIs and microservices. ... and efficiently into an organization's existing hybrid, or cloud architecture ...\n# 【14†Fastly Next-Gen WAF professional services | Fastly Products†docs.fastly.com】\nFastly Next-Gen WAF (powered by Signal Sciences) (Next-Gen WAF) professional services provide your organization with training, implementation, and maintenance services for the Next-Gen WAF. Depending on the service offerings you select, our team will provide training and work with you to plan, test, deploy, and maintain a solution to protect ...\n# 【15†Andrew Peterson - Founder & CEO @ Signal Sciences - Crunchbase†www.crunchbase.com】\nAndrew Peterson is the founder and CEO of Signal Sciences, a Software-as-a-Service platform that provides security monitoring and defense solutions for web applications. He was previously the group product manager of international growth at Etsy. Prior to Etsy, Peterson was a health information management officer at the Clinton Foundation\n# 【16†Signal Sciences†info.signalsciences.com】\nThe webinar will cover: The most common attacks targeting distributed web apps. How a web application firewall (WAF) can inspect and decision on both north-south (client-to-app origin) and east-west (service-to-service) web requests. How to detect and prevent malicious client-side activity originating from website supply chain vendors. A live ...\n# 【17†Next-Generation Cloud Web Application Firewall (WAF) - Signal Sciences ...†www.signalsciences.com】\nWith a single change to a DNS record, web requests are routed to the Signal Sciences Cloud WAF to inspect and decision on those requests and block bad requests. Our unique solution provides robust detection and protection capabilities no matter where your organization's applications operate or how rapidly they scale. No complex software ...\n# 【18†How to Configure SAML 2.0 for Signal Sciences - UserDocs†saml-doc.okta.com】\nConfiguration Steps. Log in to Signal Sciences as an Organization Administrator. Navigate to organization > Settings.. Click the Switch to SAML button:. Click the I understand, configure SSO button:. On the Configure SSO with SAML page, enter the following:. Copy and paste the following IDP Login page into the SAML endpoint field:. Sign into the Okta Admin Dashboard to generate this variable.\n# 【19†Powering the best of the internet | Fastly†www.fastly.com】\nEverybody needs speed, reliability, security, savings, and scale – but different industries have different needs. Fastly's powerful network and smarter solutions can be tailored to your organization. We partner with you to guarantee a smooth migration, so you can deliver the best possible user experiences.\nVisible: 0% - 100%", + "pub_date": null, + "extra": null + } + ], + "original_query": null + }, + "command": "search", + "args": ["Signal Sciences organization"], + "status": "finished", + "is_complete": true, + "message_type": null, + "model_slug": "gpt-4", + "parent_id": "5cc5e87b-2b74-485a-b3a7-23cb25f202db", + "request_id": "8594dde3b85705f4-IAD", + "timestamp_": "absolute" + }, + "recipient": "all" + }, + "parent": "5cc5e87b-2b74-485a-b3a7-23cb25f202db", + "children": ["4b3aec6b-5146-4bad-ae8e-204fdb6accda"] + }, + "4b3aec6b-5146-4bad-ae8e-204fdb6accda": { + "id": "4b3aec6b-5146-4bad-ae8e-204fdb6accda", + "message": { + "id": "4b3aec6b-5146-4bad-ae8e-204fdb6accda", + "author": { + "role": "assistant", + "name": null, + "metadata": {} + }, + "create_time": 1708580164.175371, + "update_time": null, + "content": { + "content_type": "text", + "parts": [ + "Signal Sciences is a web application security company that was founded on March 10, 2014, by Andrew Peterson, Nick Galbreath, and Zane Lackey. It operates as a for-profit company with its legal name being Signal Sciences Corp. The company has achieved significant growth and is recognized as the fastest-growing web application security company in the world. Signal Sciences developed a next-gen web application firewall (NGWAF) and runtime application self-protection (RASP) technologies designed to increase security and maintain reliability without compromising the performance of modern web applications distributed across cloud, on-premise, edge, or hybrid environments【3:0†source】【3:1†source】.\n\nIn a major development, Fastly, Inc., a provider of an edge cloud platform, announced the completion of its acquisition of Signal Sciences on October 1, 2020. This acquisition was valued at approximately $775 million in cash and stock. By integrating Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings, they aimed to form a unified suite of security solutions. The merger was aimed at expanding Fastly's security portfolio, particularly at a time when digital security has become paramount for businesses operating online【3:4†source】【3:7†source】." + ] + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 1, + "metadata": { + "finish_details": { + "type": "stop", + "stop_tokens": [100260] + }, + "citations": [ + { + "start_ix": 674, + "end_ix": 686, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Signal Sciences - Crunchbase Company Profile & Funding", + "url": "https://www.crunchbase.com/organization/signal-sciences", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 0 + } + } + }, + { + "start_ix": 686, + "end_ix": 698, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Demand More from Your WAF - Signal Sciences now part of Fastly", + "url": "https://www.signalsciences.com/", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 1 + } + } + }, + { + "start_ix": 1306, + "end_ix": 1318, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Fastly Completes Acquisition of Signal Sciences | Fastly", + "url": "https://www.fastly.com/press/press-releases/fastly-completes-acquisition-signal-sciences", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 4 + } + } + }, + { + "start_ix": 1318, + "end_ix": 1330, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley", + "url": "https://www.cooley.com/news/coverage/2020/2020-08-27-fastly-agrees-to-acquire-signal-sciences-for-775-million", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 7 + } + } + } + ], + "gizmo_id": null, + "is_complete": true, + "message_type": null, + "model_slug": "gpt-4", + "parent_id": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "request_id": "8594dde3b85705f4-IAD", + "timestamp_": "absolute" + }, + "recipient": "all" + }, + "parent": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "children": ["aaa27ed5-1e1c-4189-ac6c-737608404ec1"] + } + }, + "moderation_results": [], + "current_node": "88a0cf9f-e860-4b34-8e7e-65f8346f4862", + "plugin_ids": null, + "conversation_id": "d6523d1e-7ec3-474f-a363-0e9dffdb3d93", + "conversation_template_id": null, + "gizmo_id": null, + "is_archived": false, + "safe_urls": [ + "https://www.carsales.com.au/volkswagen/transporter/price/2014/tdi340-381631/", + "https://www.drive.com.au/reviews/volkswagen-transporter-review/", + "https://www.ultimatespecs.com/car-specs/volkswagen/118424/2021-volkswagen-transporter-t61-l2h1-20-tdi-110hp.html" + ], + "default_model_slug": null, + "id": "d6523d1e-7ec3-474f-a363-0e9dffdb3d93" + } +] diff --git a/api/server/utils/import/importers.js b/api/server/utils/import/importers.js index e262f1ae120..9c4c0dc5909 100644 --- a/api/server/utils/import/importers.js +++ b/api/server/utils/import/importers.js @@ -277,34 +277,39 @@ function processConversation(conv, importBatchBuilder, requestUserId) { /** * Processes text content of messages authored by an assistant, inserting citation links as required. - * Applies citation metadata to construct regex patterns and replacements for inserting links into the text. + * Uses citation start and end indices to place links at the correct positions. * * @param {ChatGPTMessage} messageData - The message data containing metadata about citations. * @param {string} messageText - The original text of the message which may be altered by inserting citation links. * @returns {string} - The updated message text after processing for citations. */ function processAssistantMessage(messageData, messageText) { - const citations = messageData.metadata.citations ?? []; + if (!messageText) { + return messageText; + } + + const citations = messageData.metadata?.citations ?? []; + + const sortedCitations = [...citations].sort((a, b) => b.start_ix - a.start_ix); - for (const citation of citations) { + let result = messageText; + for (const citation of sortedCitations) { if ( - !citation.metadata || - !citation.metadata.extra || - !citation.metadata.extra.cited_message_idx || - (citation.metadata.type && citation.metadata.type !== 'webpage') + !citation.metadata?.type || + citation.metadata.type !== 'webpage' || + typeof citation.start_ix !== 'number' || + typeof citation.end_ix !== 'number' || + citation.start_ix >= citation.end_ix ) { continue; } - const pattern = new RegExp( - `\\u3010${citation.metadata.extra.cited_message_idx}\\u2020.+?\\u3011`, - 'g', - ); const replacement = ` ([${citation.metadata.title}](${citation.metadata.url}))`; - messageText = messageText.replace(pattern, replacement); + + result = result.slice(0, citation.start_ix) + replacement + result.slice(citation.end_ix); } - return messageText; + return result; } /** @@ -342,4 +347,4 @@ function formatMessageText(messageData) { return messageText; } -module.exports = { getImporter }; +module.exports = { getImporter, processAssistantMessage }; diff --git a/api/server/utils/import/importers.spec.js b/api/server/utils/import/importers.spec.js index fa0b2e9f33b..a68bc3e7f3b 100644 --- a/api/server/utils/import/importers.spec.js +++ b/api/server/utils/import/importers.spec.js @@ -2,10 +2,10 @@ const fs = require('fs'); const path = require('path'); const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider'); const { bulkSaveConvos: _bulkSaveConvos } = require('~/models/Conversation'); +const { getImporter, processAssistantMessage } = require('./importers'); const { ImportBatchBuilder } = require('./importBatchBuilder'); const { bulkSaveMessages } = require('~/models/Message'); const getLogStores = require('~/cache/getLogStores'); -const { getImporter } = require('./importers'); jest.mock('~/cache/getLogStores'); const mockedCacheGet = jest.fn(); @@ -404,3 +404,234 @@ describe('getImporter', () => { expect(() => getImporter(jsonData)).toThrow('Unsupported import type'); }); }); + +describe('processAssistantMessage', () => { + const testMessage = 'This is a test citation 【3:0†source】【3:1†source】'; + + const messageData = { + metadata: { + citations: [ + { + start_ix: 23, // Position of first "【3:0†source】" + end_ix: 36, // End of first citation (including closing bracket) + citation_format_type: 'tether_og', + metadata: { + type: 'webpage', + title: 'Signal Sciences - Crunchbase Company Profile & Funding', + url: 'https://www.crunchbase.com/organization/signal-sciences', + text: '', + pub_date: null, + extra: { + evidence_text: 'source', + cited_message_idx: 3, + search_result_idx: 0, + }, + }, + }, + { + start_ix: 36, // Position of second "【3:1†source】" + end_ix: 49, // End of second citation (including closing bracket) + citation_format_type: 'tether_og', + metadata: { + type: 'webpage', + title: 'Demand More from Your WAF - Signal Sciences now part of Fastly', + url: 'https://www.signalsciences.com/', + text: '', + pub_date: null, + extra: { + evidence_text: 'source', + cited_message_idx: 3, + search_result_idx: 1, + }, + }, + }, + ], + }, + }; + + const messageText = testMessage; + const expectedOutput = + 'This is a test citation ([Signal Sciences - Crunchbase Company Profile & Funding](https://www.crunchbase.com/organization/signal-sciences)) ([Demand More from Your WAF - Signal Sciences now part of Fastly](https://www.signalsciences.com/))'; + + test('should correctly process citations and replace them with markdown links', () => { + const result = processAssistantMessage(messageData, messageText); + expect(result).toBe(expectedOutput); + }); + + test('should handle message with no citations', () => { + const messageWithNoCitations = { + metadata: {}, + }; + const result = processAssistantMessage(messageWithNoCitations, messageText); + expect(result).toBe(messageText); + }); + + test('should handle citations with missing metadata', () => { + const messageWithBadCitation = { + metadata: { + citations: [ + { + start_ix: 85, + end_ix: 97, + }, + ], + }, + }; + const result = processAssistantMessage(messageWithBadCitation, messageText); + expect(result).toBe(messageText); + }); + + test('should handle citations with non-webpage type', () => { + const messageWithNonWebpage = { + metadata: { + citations: [ + { + start_ix: 85, + end_ix: 97, + metadata: { + type: 'other', + title: 'Test', + url: 'http://test.com', + }, + }, + ], + }, + }; + const result = processAssistantMessage(messageWithNonWebpage, messageText); + expect(result).toBe(messageText); + }); + + test('should handle empty message text', () => { + const result = processAssistantMessage(messageData, ''); + expect(result).toBe(''); + }); + + test('should handle undefined message text', () => { + const result = processAssistantMessage(messageData, undefined); + expect(result).toBe(undefined); + }); + + test('should handle invalid citation indices', () => { + const messageWithBadIndices = { + metadata: { + citations: [ + { + start_ix: 100, + end_ix: 90, // end before start + metadata: { + type: 'webpage', + title: 'Test', + url: 'http://test.com', + }, + }, + ], + }, + }; + const result = processAssistantMessage(messageWithBadIndices, messageText); + expect(result).toBe(messageText); + }); + + test('should correctly process citations from real ChatGPT data', () => { + const jsonData = JSON.parse( + fs.readFileSync(path.join(__dirname, '__data__', 'chatgpt-citations.json'), 'utf8'), + ); + + // Get the message containing citations from the JSON data + const assistantMessage = jsonData[0].mapping['4b3aec6b-5146-4bad-ae8e-204fdb6accda'].message; + + const messageText = assistantMessage.content.parts[0]; + const citations = assistantMessage.metadata.citations; + + // Expected output should have all citations replaced with markdown links + const expectedOutput = + 'Signal Sciences is a web application security company that was founded on March 10, 2014, by Andrew Peterson, Nick Galbreath, and Zane Lackey. It operates as a for-profit company with its legal name being Signal Sciences Corp. The company has achieved significant growth and is recognized as the fastest-growing web application security company in the world. Signal Sciences developed a next-gen web application firewall (NGWAF) and runtime application self-protection (RASP) technologies designed to increase security and maintain reliability without compromising the performance of modern web applications distributed across cloud, on-premise, edge, or hybrid environments ([Signal Sciences - Crunchbase Company Profile & Funding](https://www.crunchbase.com/organization/signal-sciences)) ([Demand More from Your WAF - Signal Sciences now part of Fastly](https://www.signalsciences.com/)).\n\nIn a major development, Fastly, Inc., a provider of an edge cloud platform, announced the completion of its acquisition of Signal Sciences on October 1, 2020. This acquisition was valued at approximately $775 million in cash and stock. By integrating Signal Sciences\' powerful web application and API security solutions with Fastly\'s edge cloud platform and existing security offerings, they aimed to form a unified suite of security solutions. The merger was aimed at expanding Fastly\'s security portfolio, particularly at a time when digital security has become paramount for businesses operating online ([Fastly Completes Acquisition of Signal Sciences | Fastly](https://www.fastly.com/press/press-releases/fastly-completes-acquisition-signal-sciences)) ([Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley](https://www.cooley.com/news/coverage/2020/2020-08-27-fastly-agrees-to-acquire-signal-sciences-for-775-million)).'; + + const result = processAssistantMessage(assistantMessage, messageText); + expect(result).toBe(expectedOutput); + + // Additional checks to verify citation processing + citations.forEach((citation) => { + // Verify each citation was replaced + const markdownLink = `([${citation.metadata.title}](${citation.metadata.url}))`; + expect(result).toContain(markdownLink); + + // Verify original citation format is not present + const originalCitation = messageText.slice(citation.start_ix, citation.end_ix); + expect(result).not.toContain(originalCitation); + }); + }); + + test('should handle potential ReDoS attack payloads', () => { + // Test with increasing input sizes to check for exponential behavior + const sizes = [32, 33, 34]; // Adding more sizes would increase test time + const regExp = '(a+)+'; + const results = []; + + sizes.forEach((size) => { + const startTime = process.hrtime(); + + const maliciousMessageData = { + metadata: { + citations: [ + { + start_ix: 0, + end_ix: size, + citation_format_type: 'tether_og', + metadata: { + type: 'webpage', + title: 'Test', + url: 'http://test.com', + extra: { + cited_message_idx: regExp, + }, + }, + }, + ], + }, + }; + + const maliciousText = '【' + 'a'.repeat(size) + '】'; + + processAssistantMessage(maliciousMessageData, maliciousText); + + const endTime = process.hrtime(startTime); + const duration = endTime[0] * 1000 + endTime[1] / 1000000; // Convert to milliseconds + results.push(duration); + }); + + // Check if processing time increases exponentially + // In a ReDoS vulnerability, time would roughly double with each size increase + for (let i = 1; i < results.length; i++) { + const ratio = results[i] / results[i - 1]; + expect(ratio).toBeLessThan(2); // Processing time should not double + console.log(`Size ${sizes[i]} processing time ratio: ${ratio}`); + } + + // Also test with the exact payload from the security report + const maliciousPayload = { + metadata: { + citations: [ + { + metadata: { + extra: { + cited_message_idx: '(a+)+', + }, + type: 'webpage', + title: '1', + url: '2', + }, + }, + ], + }, + }; + + const text = '【' + 'a'.repeat(32); + const startTime = process.hrtime(); + processAssistantMessage(maliciousPayload, text); + const endTime = process.hrtime(startTime); + const duration = endTime[0] * 1000 + endTime[1] / 1000000; + + // The processing should complete quickly (under 100ms) + expect(duration).toBeLessThan(100); + }); +}); diff --git a/client/src/components/Plugins/Store/PluginPagination.tsx b/client/src/components/Plugins/Store/PluginPagination.tsx index 4a76e325600..7cd7a127636 100644 --- a/client/src/components/Plugins/Store/PluginPagination.tsx +++ b/client/src/components/Plugins/Store/PluginPagination.tsx @@ -1,4 +1,5 @@ import React from 'react'; +import { useLocalize } from '~/hooks'; type TPluginPaginationProps = { currentPage: number; @@ -11,6 +12,7 @@ const PluginPagination: React.FC = ({ maxPage, onChangePage, }) => { + const localize = useLocalize(); const pages = [...Array(maxPage).keys()].map((i) => i + 1); const handlePageChange = (page: number) => { @@ -24,8 +26,14 @@ const PluginPagination: React.FC = ({
handlePageChange(currentPage - 1)} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + onChangePage(currentPage - 1); + } + }} className={`flex cursor-default items-center text-sm ${ currentPage === 1 ? 'text-black/70 opacity-50 dark:text-white/70' @@ -45,14 +53,15 @@ const PluginPagination: React.FC = ({ width="1em" xmlns="http://www.w3.org/2000/svg" > - + - Prev + {localize('com_ui_prev')}
{pages.map((page) => (
= ({ }`} style={{ userSelect: 'none' }} onClick={() => onChangePage(page)} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + onChangePage(page); + } + }} > {page}
@@ -67,7 +81,13 @@ const PluginPagination: React.FC = ({
handlePageChange(currentPage + 1)} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + onChangePage(currentPage + 1); + } + }} className={`flex cursor-default items-center text-sm ${ currentPage === maxPage ? 'text-black/70 opacity-50 dark:text-white/70' @@ -75,7 +95,7 @@ const PluginPagination: React.FC = ({ }`} style={{ userSelect: 'none' }} > - Next + {localize('com_ui_next')} = ({ width="1em" xmlns="http://www.w3.org/2000/svg" > - +
diff --git a/client/src/components/Prompts/Groups/CreatePromptForm.tsx b/client/src/components/Prompts/Groups/CreatePromptForm.tsx index c409567bcdc..0c6c52de7c1 100644 --- a/client/src/components/Prompts/Groups/CreatePromptForm.tsx +++ b/client/src/components/Prompts/Groups/CreatePromptForm.tsx @@ -81,10 +81,10 @@ const CreatePromptForm = ({ CreateFormValues, 'name' | 'category' | 'oneliner' | 'command' >; - if ((oneliner?.length || 0) > 0) { + if ((oneliner?.length ?? 0) > 0) { groupData.oneliner = oneliner; } - if ((command?.length || 0) > 0) { + if ((command?.length ?? 0) > 0) { groupData.command = command; } createPromptMutation.mutate({ @@ -111,7 +111,7 @@ const CreatePromptForm = ({ diff --git a/client/src/components/Prompts/Groups/FilterPrompts.tsx b/client/src/components/Prompts/Groups/FilterPrompts.tsx index 1981faf5b12..88893868f2d 100644 --- a/client/src/components/Prompts/Groups/FilterPrompts.tsx +++ b/client/src/components/Prompts/Groups/FilterPrompts.tsx @@ -35,7 +35,7 @@ export function FilterItem({ > {icon} {label} - {isActive && ( + {isActive === true && ( @@ -122,7 +122,7 @@ export default function FilterPrompts({ return setCategory(''); } setCategory(category); - if (icon && React.isValidElement(icon)) { + if (icon != null && React.isValidElement(icon)) { setSelectedIcon(icon); } }, @@ -130,7 +130,7 @@ export default function FilterPrompts({ ); return ( -
+