From 2a3a973831f1124db5a2d07c9573437a02263ff0 Mon Sep 17 00:00:00 2001 From: Mikhail Rodichenko Date: Tue, 21 Mar 2023 17:34:08 +0100 Subject: [PATCH] GUI Data sharing service: object storage access audit, backported to stage/0.16/uat Add storage id to storage audit logs (#3187), Data Sharing Service GUI, backported to stage/0.16/uat --- .../client/src/components/browser/Browser.js | 6 + .../client/src/models/s3Storage/s3Storage.js | 139 +-------- .../send-logs.js} | 14 +- .../client/src/models/user/WhoAmI.js | 28 ++ .../src/utils/audit-storage-access/index.js | 267 ++++++++++++++++++ 5 files changed, 315 insertions(+), 139 deletions(-) rename data-sharing-service/client/src/models/{dataStorage/DataStorageGenerateUploadUrl.js => system-logs/send-logs.js} (63%) create mode 100644 data-sharing-service/client/src/models/user/WhoAmI.js create mode 100644 data-sharing-service/client/src/utils/audit-storage-access/index.js diff --git a/data-sharing-service/client/src/components/browser/Browser.js b/data-sharing-service/client/src/components/browser/Browser.js index 9220ea6261..17019f6e95 100644 --- a/data-sharing-service/client/src/components/browser/Browser.js +++ b/data-sharing-service/client/src/components/browser/Browser.js @@ -41,6 +41,7 @@ import displayDate from '../../utils/displayDate'; import roleModel from '../../utils/roleModel'; import styles from './Browser.css'; import {NoStorage} from '../main/App'; +import auditStorageAccessManager from '../../utils/audit-storage-access'; const PAGE_SIZE = 40; @@ -184,6 +185,11 @@ export default class Browser extends React.Component { message.error(request.error); } else { hide(); + auditStorageAccessManager.reportReadAccess({ + storageId: this.props.storageId, + path: item.path, + reportStorageType: 'S3' + }); const a = document.createElement('a'); a.href = request.value.url; a.download = item.name; diff --git a/data-sharing-service/client/src/models/s3Storage/s3Storage.js b/data-sharing-service/client/src/models/s3Storage/s3Storage.js index 23a552f7c1..725601f2c6 100644 --- a/data-sharing-service/client/src/models/s3Storage/s3Storage.js +++ b/data-sharing-service/client/src/models/s3Storage/s3Storage.js @@ -17,12 +17,7 @@ import AWS from 'aws-sdk/index'; import DataStorageTempCredentials from '../dataStorage/DataStorageTempCredentials'; import Credentials from './Credentials'; - -const asyncForEach = async (array, callback) => { - for (let index = 0; index < array.length; index++) { - await callback(array[index], index, array); - } -}; +import auditStorageAccessManager from '../../utils/audit-storage-access'; class S3Storage { @@ -49,7 +44,7 @@ class S3Storage { } get prefix () { - return this._prefix; + return this._prefix || ''; } set prefix (value) { @@ -107,27 +102,6 @@ class S3Storage { return success; }; - listObjects = () => { - let params = { - Bucket: this._storage.path, - Prefix: this._prefix - }; - if (this._storage.delimiter) { - params.Delimiter = this._storage.delimiter; - } - - return this._s3.listObjects(params).promise(); - }; - - getStorageObject = async (key, startBytes, endBytes) => { - const params = { - Bucket: this._storage.path, - Key: key, - Range: `bytes=${startBytes}-${endBytes}` - }; - return this._s3.getObject(params).promise(); - }; - completeMultipartUploadStorageObject = async (name, parts, uploadId) => { const params = { Bucket: this._storage.path, @@ -168,115 +142,14 @@ class S3Storage { PartNumber: partNumber, UploadId: uploadId }; + auditStorageAccessManager.reportWriteAccess({ + fullPath: `s3://${this._storage.path}/${this.prefix + name}`, + storageId: this._storage.id + }); const upload = this._s3.uploadPart(params); upload.on('httpUploadProgress', uploadProgress); return upload; }; - - uploadStorageObject = async (name, body, uploadProgress) => { - const params = { - Body: body, - Bucket: this._storage.path, - Key: this.prefix + name - }; - const upload = this._s3.upload(params); - upload.on('httpUploadProgress', uploadProgress); - return upload.promise(); - }; - - putStorageObject = async (name, buffer) => { - const params = { - Body: buffer, - Bucket: this._storage.path, - Key: this._prefix + name - }; - - return this._s3.putObject(params).promise(); - }; - - renameStorageObject = async (key, newName) => { - let success = true; - - let params = { - Bucket: this._storage.path, - Prefix: key - }; - - if (key.endsWith(this._storage.delimiter)) { - try { - const data = await this._s3.listObjects(params).promise(); - await asyncForEach(data.Contents, async file => { - params = { - Bucket: this._storage.path, - CopySource: this._storage.path + this._storage.delimiter + file.Key, - Key: file.Key.replace(key, this._prefix + newName + this._storage.delimiter) - }; - - await this._s3.copyObject(params).promise(); - await this.deleteStorageObjects([file.Key]); - }); - success = true; - } catch (err) { - success = false; - return Promise.reject(new Error(err.message)); - } - } else { - params = { - Bucket: this._storage.path, - CopySource: this._storage.path + this._storage.delimiter + key, - Key: this._prefix + newName - }; - try { - await this._s3.copyObject(params).promise(); - await this.deleteStorageObjects([key]); - } catch (err) { - success = false; - return Promise.reject(new Error(err.message)); - } - } - return success; - }; - - deleteStorageObjects = async (keys) => { - let deleteObjects = []; - let success = true; - - try { - await asyncForEach(keys, async key => { - if (key.endsWith(this._storage.delimiter)) { - let params = { - Bucket: this._storage.path, - Prefix: key - }; - - const data = await this._s3.listObjects(params).promise(); - data.Contents.forEach(content => { - deleteObjects.push({ - Key: content.Key - }); - }); - } else { - deleteObjects.push({ - Key: key - }); - } - }); - - const params = { - Bucket: this._storage.path, - Delete: { - Objects: deleteObjects - } - }; - - await this._s3.deleteObjects(params).promise(); - } catch (err) { - success = false; - return Promise.reject(new Error(err.message)); - } - return success; - }; - } export default new S3Storage(); diff --git a/data-sharing-service/client/src/models/dataStorage/DataStorageGenerateUploadUrl.js b/data-sharing-service/client/src/models/system-logs/send-logs.js similarity index 63% rename from data-sharing-service/client/src/models/dataStorage/DataStorageGenerateUploadUrl.js rename to data-sharing-service/client/src/models/system-logs/send-logs.js index a576a8251a..12ad77c651 100644 --- a/data-sharing-service/client/src/models/dataStorage/DataStorageGenerateUploadUrl.js +++ b/data-sharing-service/client/src/models/system-logs/send-logs.js @@ -1,11 +1,11 @@ /* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,11 +14,13 @@ * limitations under the License. */ -import Remote from '../basic/Remote'; +import RemotePost from '../basic/RemotePost'; -export default class DataStorageGenerateUploadUrl extends Remote { - constructor (id, path) { +class SendSystemLogs extends RemotePost { + constructor () { super(); - this.url = `/datastorage/${id}/generateUploadUrl?path=${path}`; + this.url = '/log'; } } + +export default SendSystemLogs; diff --git a/data-sharing-service/client/src/models/user/WhoAmI.js b/data-sharing-service/client/src/models/user/WhoAmI.js new file mode 100644 index 0000000000..4771970f8f --- /dev/null +++ b/data-sharing-service/client/src/models/user/WhoAmI.js @@ -0,0 +1,28 @@ +/* + * + * * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +import Remote from '../basic/Remote'; + +class WhoAmI extends Remote { + constructor () { + super(); + this.url = '/whoami'; + } +} + +export default new WhoAmI(); diff --git a/data-sharing-service/client/src/utils/audit-storage-access/index.js b/data-sharing-service/client/src/utils/audit-storage-access/index.js new file mode 100644 index 0000000000..f3b10aadfa --- /dev/null +++ b/data-sharing-service/client/src/utils/audit-storage-access/index.js @@ -0,0 +1,267 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import moment from 'moment'; +import whoAmI from '../../models/user/WhoAmI'; +import dataStorages from '../../models/dataStorage/DataStorages'; +import SendSystemLogs from '../../models/system-logs/send-logs'; + +const AccessTypes = { + write: 'WRITE', + read: 'READ', + delete: 'DELETE' +}; + +/** + * @typedef {Object} StorageItemOptions + * @property {number|string} storageId + * @property {string} [path] + * @property {string} [fullPath] + * @property {string} [reportStorageType] + */ + +/** + * @typedef {Object} LogEntry + * @property {moment.Moment} timestamp + * @property {string} accessType + * @property {StorageItemOptions} object + */ + +/** + * @typedef {Object} PreparedLogEntry + * @property {moment.Moment} timestamp + * @property {string} message + * @property {number|string} storageId + */ + +const REPORT_DEBOUNCE_MS = 1000; +const DEBOUNCE_ENTRIES_THRESHOLD = 50; + +/** + * Converts number to a string with additional zeros from the left, i.e. + * 1 -> 000001 + * 123 -> 000123 + * 1234567 -> 1234567 + * @param {number} eventIndex + * @returns {string} + */ +function formatEventIndex (eventIndex) { + const shift = 3; + if (typeof String.prototype.padStart === 'function') { + return `${eventIndex}`.padStart(shift, '0'); + } + let str = `${eventIndex}`; + while (str.length < shift) { + str = '0'.concat(str); + } + return str; +} + +/** + * @param {number} [eventIndex=0] + * @returns {string} + */ +function getEventIdFromTimestamp (eventIndex = 0) { + return `${moment.utc().valueOf()}${formatEventIndex(eventIndex)}`; +} + +/** + * @param {LogEntry[]} items + * @returns {Promise} + */ +async function prepareAuditItems (items) { + const storagesIds = items + .filter((anItem) => anItem.object && + anItem.object.fullPath === undefined && + anItem.object.storageId !== undefined) + .map((anItem) => Number(anItem.object.storageId)); + const storages = storagesIds.map((id) => dataStorages.load(id)); + await Promise.all(storages.map((request) => request.fetchIfNeededOrWait())); + const loadedStorages = storages + .map((request) => request.loaded ? request.value : undefined) + .filter(Boolean); + return items + .filter((anItem) => !!anItem.object) + .map((anItem) => { + const { + object, + timestamp, + accessType + } = anItem; + const { + fullPath, + storageId, + path, + reportStorageType + } = object || {}; + if (!storageId) { + return undefined; + } + const storageIdValue = storageId && !Number.isNaN(Number(storageId)) + ? Number(storageId) + : storageId; + if (fullPath) { + return { + timestamp, + message: `${accessType} ${fullPath}`, + storageId: storageIdValue + }; + } + const storage = loadedStorages.find((aStorage) => aStorage.id === Number(storageId) && + (!reportStorageType || aStorage.type === reportStorageType)); + if (!storage) { + return undefined; + } + const {pathMask} = storage; + const full = [pathMask, path].filter(Boolean).join('/'); + return { + timestamp, + message: `${accessType} ${full}`, + storageId: storageIdValue + }; + }).filter(Boolean); +} + +class AuditStorageAccess { + /** + * @type {LogEntry[]} + */ + logs = []; + + async sendAuditLogs () { + if (this.sendInProgress) { + return; + } + this.sendInProgress = true; + if (this.debounceTimer) { + clearTimeout(this.debounceTimer); + this.debounceTimer = undefined; + } + try { + await whoAmI.fetchIfNeededOrWait(); + if (!whoAmI.loaded) { + throw new Error(whoAmI.error || 'Error fetching user info'); + } + const { + userName + } = whoAmI.value || {}; + const pendingEvents = this.logs.slice(); + if (pendingEvents.length === 0) { + return; + } + const prepared = await prepareAuditItems(pendingEvents); + const payload = prepared.map((log, entryIndex) => ({ + eventId: getEventIdFromTimestamp(entryIndex), + messageTimestamp: log.timestamp.format('YYYY-MM-DD HH:mm:ss.SSS'), + message: log.message, + type: 'audit', + severity: 'INFO', + user: userName, + serviceName: 'gui', + storageId: log.storageId + })); + if (payload.length > 0) { + const request = new SendSystemLogs(); + await request.send(payload); + if (request.error) { + throw new Error(request.error); + } + } + this.logs = this.logs.filter((log) => !pendingEvents.includes(log)); + this.sendInProgress = false; + if (this.logs.length > 0) { + await this.sendAuditLogs(); + } + } catch (error) { + console.warn(error.message); + this.sendInProgress = false; + } + } + + sendAuditLogsDebounced = () => { + if (this.debounceTimer) { + clearTimeout(this.debounceTimer); + this.debounceTimer = undefined; + } + if (this.logs.length > DEBOUNCE_ENTRIES_THRESHOLD) { + (this.sendAuditLogs)(); + } else { + this.debounceTimer = setTimeout(() => this.sendAuditLogs(), REPORT_DEBOUNCE_MS); + } + }; + + /** + * @param {string} accessType + * @param {boolean} debounced + * @param {StorageItemOptions} object + */ + reportAccess = (accessType, debounced, ...object) => { + const time = moment.utc(); + this.logs.push(...object.map((singleObject) => ({ + timestamp: time, + accessType, + object: singleObject + }))); + if (debounced) { + this.sendAuditLogsDebounced(); + } else { + (this.sendAuditLogs)(); + } + }; + + /** + * @param {StorageItemOptions} object + */ + reportReadAccess = (...object) => { + this.reportAccess(AccessTypes.read, false, ...object); + }; + + /** + * @param {StorageItemOptions} object + */ + reportWriteAccess = (...object) => { + this.reportAccess(AccessTypes.write, false, ...object); + }; + + /** + * @param {StorageItemOptions} object + */ + reportDelete = (...object) => { + this.reportAccess(AccessTypes.delete, false, ...object); + }; + + reportReadAccessDebounced = (...object) => { + this.reportAccess(AccessTypes.read, true, ...object); + }; + + /** + * @param {StorageItemOptions} object + */ + reportWriteAccessDebounced = (...object) => { + this.reportAccess(AccessTypes.write, true, ...object); + }; + + /** + * @param {StorageItemOptions} object + */ + reportDeleteDebounced = (...object) => { + this.reportAccess(AccessTypes.delete, true, ...object); + }; +} + +const auditStorageAccessManager = new AuditStorageAccess(); + +export default auditStorageAccessManager;