Issue #2574: pipe storage cp shall start data upload before traversin…

…g full source hierarchy - cleanups
epam · Apr 11, 2022 · 54a9ede · 54a9ede
1 parent 2255ef4
commit 54a9ede
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 7 deletions.
diff --git a/pipe-cli/src/utilities/datastorage_operations.py b/pipe-cli/src/utilities/datastorage_operations.py
@@ -163,10 +163,6 @@ def _transfer(cls, source_wrapper, destination_wrapper, items_part, manager, per
                 cls._flush_transfer_results(source_wrapper, destination_wrapper,
                                             transfer_results, clean=clean, flush_size=1)
 
-    @classmethod
-    def _is_cloud_source(cls, source_wrapper_type):
-        return source_wrapper_type in [WrapperType.S3, WrapperType.AZURE, WrapperType.GS]
-
     @classmethod
     def _filter_items(cls, items, manager, source_wrapper, destination_wrapper, permission_to_check,
                       include, exclude, force, quiet, skip_existing, verify_destination):

diff --git a/pipe-cli/src/utilities/storage/azure.py b/pipe-cli/src/utilities/storage/azure.py
@@ -149,7 +149,8 @@ def __init__(self, blob_service, bucket):
         self.delimiter = StorageOperations.PATH_SEPARATOR
         self.listing_manager = AzureListingManager(self.service, self.bucket)
 
-    def delete_items(self, relative_path, recursive=False, exclude=[], include=[], version=None, hard_delete=False):
+    def delete_items(self, relative_path, recursive=False, exclude=[], include=[], version=None, hard_delete=False,
+                     page_size=None):
         if version or hard_delete:
             raise RuntimeError('Versioning is not supported by AZURE cloud provider')
         prefix = StorageOperations.get_prefix(relative_path)

diff --git a/pipe-cli/src/utilities/storage/common.py b/pipe-cli/src/utilities/storage/common.py
@@ -356,7 +356,8 @@ class AbstractDeleteManager:
     __metaclass__ = ABCMeta
 
     @abstractmethod
-    def delete_items(self, relative_path, recursive=False, exclude=[], include=[], version=None, hard_delete=False):
+    def delete_items(self, relative_path, recursive=False, exclude=[], include=[], version=None, hard_delete=False,
+                     page_size=None):
         """
         Deletes all items under the given path.
 
@@ -366,6 +367,7 @@ def delete_items(self, relative_path, recursive=False, exclude=[], include=[], v
         :param include: Include item pattern.
         :param version: Version to be deleted.
         :param hard_delete: Specifies if all item versions have to be deleted.
+        :param page_size: Specifies page size for listing objects.
         """
         pass
 

diff --git a/pipe-cli/src/utilities/storage/gs.py b/pipe-cli/src/utilities/storage/gs.py
@@ -520,7 +520,8 @@ def __init__(self, client, bucket):
         self.bucket = bucket
         self.delimiter = StorageOperations.PATH_SEPARATOR
 
-    def delete_items(self, relative_path, recursive=False, exclude=[], include=[], version=None, hard_delete=False):
+    def delete_items(self, relative_path, recursive=False, exclude=[], include=[], version=None, hard_delete=False,
+                     page_size=None):
         if recursive and version:
             raise RuntimeError('Recursive folder deletion with specified version is not available '
                                'for GCP cloud provider.')