refactor: move out recursive file paths loop

TyShkan · Mar 24, 2023 · 4761772 · 4761772
1 parent fe28566
commit 4761772
Showing 1 changed file with 12 additions and 5 deletions.
diff --git a/tap_csv/client.py b/tap_csv/client.py
@@ -50,6 +50,17 @@ def get_records(self, context: Optional[dict]) -> Iterable[dict]:
 
                 yield dict(zip(self.header, row))
 
+    def _get_recursive_file_paths(self, file_path: str) -> list:
+        file_paths = []
+
+        for dirpath, _, filenames in os.walk(file_path):
+            for filename in filenames:
+                file_path = os.path.join(dirpath, filename)
+                if self.is_valid_filename(file_path):
+                    file_paths.append(file_path)
+
+        return file_paths
+
     def get_file_paths(self) -> list:
         """Return a list of file paths to read.
 
@@ -67,11 +78,7 @@ def get_file_paths(self) -> list:
         file_paths = []
         if os.path.isdir(file_path):
             clean_file_path = os.path.normpath(file_path) + os.sep
-            for dirpath, _, filenames in os.walk(clean_file_path):
-                for filename in filenames:
-                    file_path = os.path.join(dirpath, filename)
-                    if self.is_valid_filename(file_path):
-                        file_paths.append(file_path)
+            file_paths = self._get_recursive_file_paths(clean_file_path)
         else:
             if self.is_valid_filename(file_path):
                 file_paths.append(file_path)