Merge remote-tracking branch 'origin/improvement/ZENKO-4737-add-e2e-t…

…est-for-dmf-retries' into w/2.8/improvement/ZENKO-4737-add-e2e-test-for-dmf-retries
scality · Apr 3, 2024 · 9087a21 · 9087a21
2 parents c44d063 + 7449080
commit 9087a21
Show file tree

Hide file tree

Showing 3 changed files with 86 additions and 5 deletions.
diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts
@@ -55,10 +55,41 @@ export async function cleanS3Bucket(
     await S3.deleteBucket(world.getCommandParameters());
 }
 
+/**
+ * @param {Zenko} this world object
+ * @param {string} objectName object name
+ * @returns {string} the object name based on the backend flakyness
+ */
+function getObjectNameWithBackendFlakiness(this: Zenko, objectName: string) {
+    let objectNameFinal;
+    const backendFlakinessRetryNumber = this.getSaved<string>('backendFlakinessRetryNumber');
+    const backendFlakiness = this.getSaved<string>('backendFlakiness');
+
+    if (!backendFlakiness || !backendFlakinessRetryNumber || !objectName) {
+        return objectName;
+    }
+
+    switch (backendFlakiness) {
+    case 'command':
+        objectNameFinal = `${objectName}.scal-retry-command-${backendFlakinessRetryNumber}`;
+        break;
+    case 'archive':
+    case 'restore':
+        objectNameFinal = `${objectName}.scal-retry-${backendFlakiness}-job-${backendFlakinessRetryNumber}`;
+        break;
+    default:
+        process.stdout.write(`Unknown backend flakyness ${backendFlakiness}\n`);
+        return objectName;
+    }
+    return objectNameFinal;
+}
+
 async function addMultipleObjects(this: Zenko, numberObjects: number,
     objectName: string, sizeBytes: number, userMD?: string) {
     for (let i = 1; i <= numberObjects; i++) {
-        this.addToSaved('objectName', `${objectName}-${i}` || Utils.randomString());
+        const objectNameFinal = getObjectNameWithBackendFlakiness.call(this, `${objectName}-${i}`);
+
+        this.addToSaved('objectName', `${objectNameFinal}` || Utils.randomString());
         const objectPath = tmpNameSync({prefix: this.getSaved<string>('objectName')});
         fs.writeFileSync(objectPath, Buffer.alloc(sizeBytes, this.getSaved<string>('objectName')));
         this.resetCommand();
@@ -68,6 +99,7 @@ async function addMultipleObjects(this: Zenko, numberObjects: number,
         if (userMD) {
             this.addCommandParameter({ metadata: JSON.stringify(userMD) });
         }
+        process.stdout.write(`Adding object ${objectNameFinal}\n`);
         this.addToSaved('versionId', extractPropertyFromResults(
             await S3.putObject(this.getCommandParameters()), 'VersionId')
         );
@@ -222,7 +254,7 @@ Given('a transition workflow to {string} location', async function (this: Zenko,
 });
 
 When('i restore object {string} for {int} days', async function (this: Zenko, objectName: string, days: number) {
-    const objName = objectName ||  this.getSaved<string>('objectName');
+    const objName = getObjectNameWithBackendFlakiness.call(this, objectName) ||  this.getSaved<string>('objectName');
     this.resetCommand();
     this.addCommandParameter({ bucket: this.getSaved<string>('bucketName') });
     this.addCommandParameter({ key: objName });
@@ -237,7 +269,8 @@ When('i restore object {string} for {int} days', async function (this: Zenko, ob
 // wait for object to transition to a location or get restored from it
 Then('object {string} should be {string} and have the storage class {string}', { timeout: 130000 },
     async function (this: Zenko, objectName: string, objectTransitionStatus: string, storageClass: string) {
-        const objName = objectName ||  this.getSaved<string>('objectName');
+        const objName =
+            getObjectNameWithBackendFlakiness.call(this, objectName) || this.getSaved<string>('objectName');
         this.resetCommand();
         this.addCommandParameter({ bucket: this.getSaved<string>('bucketName') });
         this.addCommandParameter({ key: objName });
@@ -279,7 +312,7 @@ Then('object {string} should be {string} and have the storage class {string}', {
     });
 
 When('i delete object {string}', async function (this: Zenko, objectName: string) {
-    const objName = objectName ||  this.getSaved<string>('objectName');
+    const objName = getObjectNameWithBackendFlakiness.call(this, objectName) ||  this.getSaved<string>('objectName');
     this.resetCommand();
     this.addCommandParameter({ bucket: this.getSaved<string>('bucketName') });
     this.addCommandParameter({ key: objName });

diff --git a/tests/ctst/features/dmf.feature b/tests/ctst/features/dmf.feature
@@ -21,6 +21,36 @@ Feature: DMF
     |               Versioned |           2 |        100 |
     |               Suspended |           2 |        100 |
 
+    @2.7.0
+    @PreMerge
+    @Dmf
+    @ColdStorage
+    @Flaky
+    Scenario Outline: Retry DMF job/command upon failure
+    Given a "<versioningConfiguration>" bucket
+    And a flaky backend that will require <retryNumber> retries for "<operation>"
+    And a transition workflow to "e2e-cold" location
+    And <objectCount> objects "obj" of size <objectSize> bytes
+    Then object "obj-1" should be "transitioned" and have the storage class "e2e-cold"
+    And object "obj-2" should be "transitioned" and have the storage class "e2e-cold"
+    When i restore object "obj-1" for 5 days
+    Then object "obj-1" should be "restored" and have the storage class "e2e-cold"
+    When i delete object "obj-1"
+    And i delete object "obj-2"
+    Then dmf volume should contain 0 objects
+
+    Examples:
+    | versioningConfiguration | objectCount | objectSize | retryNumber | operation |
+    |           Non versioned |           2 |        100 |           1 |   archive |
+    |               Versioned |           2 |        100 |           1 |   archive |
+    |               Suspended |           2 |        100 |           1 |   archive |
+    |           Non versioned |           2 |        100 |           1 |   restore |
+    |               Versioned |           2 |        100 |           1 |   restore |
+    |               Suspended |           2 |        100 |           1 |   restore |
+    |           Non versioned |           2 |        100 |           1 |   command |
+    |               Versioned |           2 |        100 |           1 |   command |
+    |               Suspended |           2 |        100 |           1 |   command |
+
     @2.7.0
     @PreMerge
     @Dmf

diff --git a/tests/ctst/steps/dmf.ts b/tests/ctst/steps/dmf.ts
@@ -1,10 +1,15 @@
-import { Then, setDefaultTimeout } from '@cucumber/cucumber';
+import { Then, Given, setDefaultTimeout, After } from '@cucumber/cucumber';
 import assert from 'assert';
 import { Constants } from 'cli-testing';
 import { execShellCommand } from 'common/utils';
+import Zenko from 'world/Zenko';
 
 setDefaultTimeout(Constants.DEFAULT_TIMEOUT);
 
+async function cleanDmfVolume(Zenko: Zenko) {
+    await execShellCommand('rm -rf /cold-data/*');
+}
+
 Then('dmf volume should contain {int} objects', async (objectCount: number) => {
     let conditionOk = false;
     while (!conditionOk) {
@@ -16,3 +21,16 @@ Then('dmf volume should contain {int} objects', async (objectCount: number) => {
     }
     assert(conditionOk);
 });
+
+Given('a flaky backend that will require {int} retries for {string}',
+    function (this: Zenko, retryNumber: number, op: string) {
+        assert(['restore', 'archive', 'command'].includes(op), `Invalid operation ${op}`);
+        assert(retryNumber > 0, `Invalid retry number ${retryNumber}`);
+
+        this.addToSaved('backendFlakinessRetryNumber', retryNumber);
+        this.addToSaved('backendFlakiness', op);
+    });
+
+After({ tags: '@Dmf' }, async function (this: Zenko) {
+    await cleanDmfVolume(this);
+});