Add end2end retry command dmf tests

Issue: ZENKO-4737
scality · Mar 29, 2024 · 07b4fb7 · 07b4fb7
1 parent 497bf13
commit 07b4fb7
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 2 deletions.
diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts
@@ -10,10 +10,72 @@ import { createBucketWithConfiguration, putObject } from 'steps/utils/utils';
 
 setDefaultTimeout(Constants.DEFAULT_TIMEOUT);
 
+type retryDmf = {
+    commandRetryNumber?: number,
+    jobRetryNumber?: number,
+    jobRetryOP?: string,
+}
+
+type listingObject = {
+    Key: string,
+    VersionId: string,
+}
+
+type listingResult = {
+    Versions: listingObject[],
+    DeleteMarkers: listingObject[],
+}
+
+/**
+ * Cleans the created test bucket
+ * @param {Zenko} world world object
+ * @param {string} bucketName bucket name
+ * @returns {void}
+ */
+export async function cleanS3Bucket(
+    world: Zenko,
+    bucketName: string,
+): Promise<void> {
+    if (!bucketName) {
+        return;
+    }
+    world.resetCommand();
+    world.addCommandParameter({ bucket: bucketName });
+    const createdObjects = world.getSaved<Map<string, string>>('createdObjects');
+    if (createdObjects !== undefined) {
+        const results = await S3.listObjectVersions(world.getCommandParameters());
+        const res = safeJsonParse(results.stdout);
+        assert(res.ok);
+        const parsedResults = res.result as listingResult;
+        const versions = parsedResults.Versions || [];
+        const deleteMarkers = parsedResults.DeleteMarkers || [];
+        await Promise.all(versions.concat(deleteMarkers).map(obj => {
+            world.addCommandParameter({ key: obj.Key });
+            world.addCommandParameter({ versionId: obj.VersionId });
+            return S3.deleteObject(world.getCommandParameters());
+        }));
+        world.deleteKeyFromCommand('key');
+        world.deleteKeyFromCommand('versionId');
+    }
+    await S3.deleteBucketLifecycle(world.getCommandParameters());
+    await S3.deleteBucket(world.getCommandParameters());
+}
+
 async function addMultipleObjects(this: Zenko, numberObjects: number,
-    objectName: string, sizeBytes: number, userMD?: string) {
+    objectName: string, sizeBytes: number, userMD?: string,
+    retryDMF?: retryDmf) {
+    assert(!retryDMF?.commandRetryNumber || !retryDMF?.jobRetryNumber,
+        'Cannot have both retryCommandNumber and retryJobNumber');
     for (let i = 1; i <= numberObjects; i++) {
-        this.addToSaved('objectName', `${objectName}-${i}` || Utils.randomString());
+        let objectNameFinal = `${objectName}-${i}`;
+
+        if (retryDMF?.commandRetryNumber) {
+            objectNameFinal = `${objectNameFinal}.scal-retry-command-${retryDMF.commandRetryNumber}`;
+        } else if (retryDMF?.jobRetryNumber && retryDMF.jobRetryOP) {
+            objectNameFinal = `${objectNameFinal}.scal-retry-${retryDMF.jobRetryOP}-job-${retryDMF.jobRetryNumber}`;
+        }
+
+        this.addToSaved('objectName', `${objectNameFinal}` || Utils.randomString());
         const objectPath = tmpNameSync({prefix: this.getSaved<string>('objectName')});
         fs.writeFileSync(objectPath, Buffer.alloc(sizeBytes, this.getSaved<string>('objectName')));
         this.resetCommand();
@@ -23,6 +85,7 @@ async function addMultipleObjects(this: Zenko, numberObjects: number,
         if (userMD) {
             this.addCommandParameter({ metadata: JSON.stringify(userMD) });
         }
+        process.stdout.write(`\nAdding object ${objectNameFinal}\n`);
         this.addToSaved('versionId', extractPropertyFromResults(
             await S3.putObject(this.getCommandParameters()), 'VersionId')
         );
@@ -78,6 +141,20 @@ Given('{int} objects {string} of size {int} bytes with user metadata {string}',
         await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, userMD);
     });
 
+Given('{int} objects {string} of size {int} bytes that will need {int} job retries on {string} operation',
+    async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number,
+        numberOfRetries: string, retryOP: string) {
+        await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, undefined,
+            { jobRetryNumber: parseInt(numberOfRetries), jobRetryOP: retryOP });
+    });
+
+Given('{int} objects {string} of size {int} bytes that will need {int} command retries',
+    async function (this: Zenko, numberObjects: number, objectName: string, sizeBytes: number,
+        numberOfRetries: string) {
+        await addMultipleObjects.call(this, numberObjects, objectName, sizeBytes, undefined,
+            { commandRetryNumber: parseInt(numberOfRetries)});
+    });
+
 Given('a tag on object {string} with key {string} and value {string}',
     async function (this: Zenko, objectName: string, tagKey: string, tagValue: string) {
         this.resetCommand();

diff --git a/tests/ctst/features/dmf.feature b/tests/ctst/features/dmf.feature
@@ -23,6 +23,28 @@ Feature: DMF
     @2.7.0
     @PreMerge
     @Dmf
+    @ColdStorage
+    Scenario Outline: Retry after failure
+    Given a "<versioningConfiguration>" bucket
+    And a transition workflow to "e2e-cold" location
+    And <objectCount> objects "obj" of size <objectSize> bytes that will need <retryNumber> command retries
+    Then object "obj-1.scal-retry-command-2" should be "transitioned" and have the storage class "e2e-cold"
+    And object "obj-2.scal-retry-command-2" should be "transitioned" and have the storage class "e2e-cold"
+    And dmf volume should contain <objectCount> objects
+    When i delete object "obj-1.scal-retry-command-2"
+    And i delete object "obj-2.scal-retry-command-2"
+    Then dmf volume should contain 0 objects
+
+    Examples:
+    | versioningConfiguration | objectCount | objectSize | retryNumber |
+    |           Non versioned |           2 |        100 |           2 |
+    |               Versioned |           2 |        100 |           2 |
+    |               Suspended |           2 |        100 |           2 |
+
+    @2.7.0
+    @PreMerge
+    @Dmf
+    @ColdStorage
     Scenario Outline: Deletion of a restored object
     Given a "<versioningConfiguration>" bucket
     And a transition workflow to "e2e-cold" location