forked from danny-avila/LibreChat
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
📂 feat: RAG Improvements (danny-avila#2169)
* feat: new vector file processing strategy * chore: remove unused client files * chore: remove more unused client files * chore: remove more unused client files and move used to new dir * chore(DataIcon): add className * WIP: Model Endpoint Settings Update, draft additional context settings * feat: improve parsing for augmented prompt, add full context option * chore: remove volume mounting from rag.yml as no longer necessary
- Loading branch information
1 parent
f427ad7
commit 45a95ac
Showing
40 changed files
with
716 additions
and
2,047 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| const fs = require('fs'); | ||
| const axios = require('axios'); | ||
| const FormData = require('form-data'); | ||
| const { FileSources } = require('librechat-data-provider'); | ||
| const { logger } = require('~/config'); | ||
|
|
||
| /** | ||
| * Deletes a file from the vector database. This function takes a file object, constructs the full path, and | ||
| * verifies the path's validity before deleting the file. If the path is invalid, an error is thrown. | ||
| * | ||
| * @param {Express.Request} req - The request object from Express. It should have an `app.locals.paths` object with | ||
| * a `publicPath` property. | ||
| * @param {MongoFile} file - The file object to be deleted. It should have a `filepath` property that is | ||
| * a string representing the path of the file relative to the publicPath. | ||
| * | ||
| * @returns {Promise<void>} | ||
| * A promise that resolves when the file has been successfully deleted, or throws an error if the | ||
| * file path is invalid or if there is an error in deletion. | ||
| */ | ||
| const deleteVectors = async (req, file) => { | ||
| if (file.embedded && process.env.RAG_API_URL) { | ||
| const jwtToken = req.headers.authorization.split(' ')[1]; | ||
| axios.delete(`${process.env.RAG_API_URL}/documents`, { | ||
| headers: { | ||
| Authorization: `Bearer ${jwtToken}`, | ||
| 'Content-Type': 'application/json', | ||
| accept: 'application/json', | ||
| }, | ||
| data: [file.file_id], | ||
| }); | ||
| } | ||
| }; | ||
|
|
||
| /** | ||
| * Uploads a file to the configured Vector database | ||
| * | ||
| * @param {Object} params - The params object. | ||
| * @param {Object} params.req - The request object from Express. It should have a `user` property with an `id` | ||
| * representing the user, and an `app.locals.paths` object with an `uploads` path. | ||
| * @param {Express.Multer.File} params.file - The file object, which is part of the request. The file object should | ||
| * have a `path` property that points to the location of the uploaded file. | ||
| * @param {string} params.file_id - The file ID. | ||
| * | ||
| * @returns {Promise<{ filepath: string, bytes: number }>} | ||
| * A promise that resolves to an object containing: | ||
| * - filepath: The path where the file is saved. | ||
| * - bytes: The size of the file in bytes. | ||
| */ | ||
| async function uploadVectors({ req, file, file_id }) { | ||
| if (!process.env.RAG_API_URL) { | ||
| throw new Error('RAG_API_URL not defined'); | ||
| } | ||
|
|
||
| try { | ||
| const jwtToken = req.headers.authorization.split(' ')[1]; | ||
| const formData = new FormData(); | ||
| formData.append('file_id', file_id); | ||
| formData.append('file', fs.createReadStream(file.path)); | ||
|
|
||
| const formHeaders = formData.getHeaders(); // Automatically sets the correct Content-Type | ||
|
|
||
| const response = await axios.post(`${process.env.RAG_API_URL}/embed`, formData, { | ||
| headers: { | ||
| Authorization: `Bearer ${jwtToken}`, | ||
| accept: 'application/json', | ||
| ...formHeaders, | ||
| }, | ||
| }); | ||
|
|
||
| const responseData = response.data; | ||
| logger.debug('Response from embedding file', responseData); | ||
|
|
||
| if (responseData.known_type === false) { | ||
| throw new Error(`File embedding failed. The filetype ${file.mimetype} is not supported`); | ||
| } | ||
|
|
||
| if (!responseData.status) { | ||
| throw new Error('File embedding failed.'); | ||
| } | ||
|
|
||
| return { | ||
| bytes: file.size, | ||
| filename: file.originalname, | ||
| filepath: FileSources.vectordb, | ||
| embedded: Boolean(responseData.known_type), | ||
| }; | ||
| } catch (error) { | ||
| logger.error('Error embedding file', error); | ||
| throw new Error(error.message || 'An error occurred during file upload.'); | ||
| } | ||
| } | ||
|
|
||
| module.exports = { | ||
| deleteVectors, | ||
| uploadVectors, | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| const crud = require('./crud'); | ||
|
|
||
| module.exports = { | ||
| ...crud, | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.