diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 19e1775f38..386014d80e 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -55,10 +55,41 @@ export async function cleanS3Bucket( await S3.deleteBucket(world.getCommandParameters()); } +/** + * @param {Zenko} this world object + * @param {string} objectName object name + * @returns {string} the object name based on the backend flakyness + */ +function getObjectNameWithBackendFlakiness(this: Zenko, objectName: string) { + let objectNameFinal; + const backendFlakinessRetryNumber = this.getSaved('backendFlakinessRetryNumber'); + const backendFlakiness = this.getSaved('backendFlakiness'); + + if (!backendFlakiness || !backendFlakinessRetryNumber || !objectName) { + return objectName; + } + + switch (backendFlakiness) { + case 'command': + objectNameFinal = `${objectName}.scal-retry-command-${backendFlakinessRetryNumber}`; + break; + case 'archive': + case 'restore': + objectNameFinal = `${objectName}.scal-retry-${backendFlakiness}-job-${backendFlakinessRetryNumber}`; + break; + default: + process.stdout.write(`Unknown backend flakyness ${backendFlakiness}\n`); + return objectName; + } + return objectNameFinal; +} + async function addMultipleObjects(this: Zenko, numberObjects: number, objectName: string, sizeBytes: number, userMD?: string) { for (let i = 1; i <= numberObjects; i++) { - this.addToSaved('objectName', `${objectName}-${i}` || Utils.randomString()); + const objectNameFinal = getObjectNameWithBackendFlakiness.call(this, `${objectName}-${i}`); + + this.addToSaved('objectName', `${objectNameFinal}` || Utils.randomString()); const objectPath = tmpNameSync({prefix: this.getSaved('objectName')}); fs.writeFileSync(objectPath, Buffer.alloc(sizeBytes, this.getSaved('objectName'))); this.resetCommand(); @@ -68,6 +99,7 @@ async function addMultipleObjects(this: Zenko, numberObjects: number, if (userMD) { this.addCommandParameter({ metadata: JSON.stringify(userMD) }); } + process.stdout.write(`Adding object ${objectNameFinal}\n`); this.addToSaved('versionId', extractPropertyFromResults( await S3.putObject(this.getCommandParameters()), 'VersionId') ); @@ -222,7 +254,7 @@ Given('a transition workflow to {string} location', async function (this: Zenko, }); When('i restore object {string} for {int} days', async function (this: Zenko, objectName: string, days: number) { - const objName = objectName || this.getSaved('objectName'); + const objName = getObjectNameWithBackendFlakiness.call(this, objectName) || this.getSaved('objectName'); this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); @@ -237,7 +269,8 @@ When('i restore object {string} for {int} days', async function (this: Zenko, ob // wait for object to transition to a location or get restored from it Then('object {string} should be {string} and have the storage class {string}', { timeout: 130000 }, async function (this: Zenko, objectName: string, objectTransitionStatus: string, storageClass: string) { - const objName = objectName || this.getSaved('objectName'); + const objName = + getObjectNameWithBackendFlakiness.call(this, objectName) || this.getSaved('objectName'); this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); @@ -279,7 +312,7 @@ Then('object {string} should be {string} and have the storage class {string}', { }); When('i delete object {string}', async function (this: Zenko, objectName: string) { - const objName = objectName || this.getSaved('objectName'); + const objName = getObjectNameWithBackendFlakiness.call(this, objectName) || this.getSaved('objectName'); this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); diff --git a/tests/ctst/features/dmf.feature b/tests/ctst/features/dmf.feature index 5b560480df..1e811b0583 100644 --- a/tests/ctst/features/dmf.feature +++ b/tests/ctst/features/dmf.feature @@ -21,6 +21,36 @@ Feature: DMF | Versioned | 2 | 100 | | Suspended | 2 | 100 | + @2.7.0 + @PreMerge + @Dmf + @ColdStorage + @Flaky + Scenario Outline: Retry DMF job/command upon failure + Given a "" bucket + And a flaky backend that will require retries for "" + And a transition workflow to "e2e-cold" location + And objects "obj" of size bytes + Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold" + And object "obj-2" should be "transitioned" and have the storage class "e2e-cold" + When i restore object "obj-1" for 5 days + Then object "obj-1" should be "restored" and have the storage class "e2e-cold" + When i delete object "obj-1" + And i delete object "obj-2" + Then dmf volume should contain 0 objects + + Examples: + | versioningConfiguration | objectCount | objectSize | retryNumber | operation | + | Non versioned | 2 | 100 | 1 | archive | + | Versioned | 2 | 100 | 1 | archive | + | Suspended | 2 | 100 | 1 | archive | + | Non versioned | 2 | 100 | 1 | restore | + | Versioned | 2 | 100 | 1 | restore | + | Suspended | 2 | 100 | 1 | restore | + | Non versioned | 2 | 100 | 1 | command | + | Versioned | 2 | 100 | 1 | command | + | Suspended | 2 | 100 | 1 | command | + @2.7.0 @PreMerge @Dmf diff --git a/tests/ctst/steps/dmf.ts b/tests/ctst/steps/dmf.ts index 34e0386df1..2a15c17271 100644 --- a/tests/ctst/steps/dmf.ts +++ b/tests/ctst/steps/dmf.ts @@ -1,10 +1,15 @@ -import { Then, setDefaultTimeout } from '@cucumber/cucumber'; +import { Then, Given, setDefaultTimeout, After } from '@cucumber/cucumber'; import assert from 'assert'; import { Constants } from 'cli-testing'; import { execShellCommand } from 'common/utils'; +import Zenko from 'world/Zenko'; setDefaultTimeout(Constants.DEFAULT_TIMEOUT); +async function cleanDmfVolume(Zenko: Zenko) { + await execShellCommand('rm -rf /cold-data/*'); +} + Then('dmf volume should contain {int} objects', async (objectCount: number) => { let conditionOk = false; while (!conditionOk) { @@ -16,3 +21,16 @@ Then('dmf volume should contain {int} objects', async (objectCount: number) => { } assert(conditionOk); }); + +Given('a flaky backend that will require {int} retries for {string}', + function (this: Zenko, retryNumber: number, op: string) { + assert(['restore', 'archive', 'command'].includes(op), `Invalid operation ${op}`); + assert(retryNumber > 0, `Invalid retry number ${retryNumber}`); + + this.addToSaved('backendFlakinessRetryNumber', retryNumber); + this.addToSaved('backendFlakiness', op); + }); + +After({ tags: '@Dmf' }, async function (this: Zenko) { + await cleanDmfVolume(this); +});