Merge pull request #1125 from activeloopai/feature/gcs_support

GCS support
activeloopai · Sep 13, 2021 · ca05d8e · ca05d8e
2 parents 2e0fa1d + a2f65a8
commit ca05d8e
Show file tree

Hide file tree

Showing 20 changed files with 454 additions and 18 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -106,8 +106,9 @@ commands:
       - run:
           name: "Install dependencies"
           command: |
-              brew install zlib libjpeg webp       	
+              brew install zlib libjpeg webp
               sudo ln -s /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/* /usr/local/include/
+              sudo launchctl limit maxfiles 65536 200000
   info:
     steps:
       - run:
@@ -176,7 +177,8 @@ commands:
                 command: |
                   $Env:GOOGLE_APPLICATION_CREDENTIALS = $Env:CI_GCS_PATH
                   setx /m GOOGLE_APPLICATION_CREDENTIALS "$Env:GOOGLE_APPLICATION_CREDENTIALS"
-                  python3 -m pytest --cov-report=xml --cov=./ --local --s3 --hub-cloud --kaggle --ignore-glob=buH/*
+                  python3 -m pytest --cov-report=xml --cov=./ --local --s3 --gcs --hub-cloud --kaggle --ignore-glob=buH/*
+                no_output_timeout: 30m
       - when:
           condition: << parameters.unix-like >>
           steps:
@@ -186,9 +188,10 @@ commands:
                   BUGGER_OFF: "true"
                 command: |
                   export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.secrets/gcs.json
-                  python3 -m pytest --cov-report=xml --cov=./ --local --s3 --hub-cloud --kaggle --ignore-glob=buH/*
-                  
+                  python3 -m pytest --cov-report=xml --cov=./ --local --s3 --gcs --hub-cloud --kaggle --ignore-glob=buH/*
+                no_output_timeout: 30m
           parallelism: 10
+
   run-backwards-compatibility-tests:
     steps:
       - run:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -31,6 +31,7 @@ pip3 install -r hub/requirements/tests.txt
 - `pytest .`: Run all tests with memory only.
 - `pytest . --local`: Run all tests with memory and local.
 - `pytest . --s3`: Run all tests with memory and s3.
+- `pytest . --gcs`: Run all tests with memory and GCS 
 - `pytest . --kaggle`: Run all tests that use the kaggle API.
 - `pytest . --memory-skip --hub-cloud`: Run all tests with hub cloud only.
 #### Backwards Compatibility Tests
@@ -41,6 +42,7 @@ Combine any of the following options to suit your test cases.
 
 - `--local`: Enable local tests.
 - `--s3`: Enable S3 tests.
+- `--gcs`: Enable GCS tests.
 - `--hub-cloud`: Enable hub cloud tests.
 - `--memory-skip`: Disable memory tests.
 - `--s3-path`: Specify an s3 path if you don't have access to our internal testing bucket.

diff --git a/conftest.py b/conftest.py
@@ -28,6 +28,9 @@ def pytest_addoption(parser):
         LOCAL_OPT, action="store_true", help="Local tests will run if enabled."
     )
     parser.addoption(S3_OPT, action="store_true", help="S3 tests will run if enabled.")
+    parser.addoption(
+        GCS_OPT, action="store_true", help="GCS tests will run if enabled."
+    )
     parser.addoption(
         HUB_CLOUD_OPT, action="store_true", help="Hub cloud tests will run if enabled."
     )

diff --git a/hub/api/tests/test_api.py b/hub/api/tests/test_api.py
@@ -20,6 +20,7 @@
 from hub.tests.dataset_fixtures import (
     enabled_datasets,
     enabled_persistent_dataset_generators,
+    enabled_non_gcs_datasets,
 )
 
 
@@ -145,15 +146,15 @@ def test_stringify_with_path(local_ds):
     assert str(ds) == f"Dataset(path='{local_ds.path}', tensors=[])"
 
 
-@enabled_datasets
+@enabled_non_gcs_datasets
 def test_compute_fixed_tensor(ds):
     ds.create_tensor("image")
     ds.image.extend(np.ones((32, 28, 28)))
     assert len(ds) == 32
     np.testing.assert_array_equal(ds.image.numpy(), np.ones((32, 28, 28)))
 
 
-@enabled_datasets
+@enabled_non_gcs_datasets
 def test_compute_dynamic_tensor(ds):
     ds.create_tensor("image")
 
@@ -216,7 +217,7 @@ def test_empty_samples(ds: Dataset):
         np.testing.assert_array_equal(actual, expected)
 
 
-@enabled_datasets
+@enabled_non_gcs_datasets
 def test_safe_downcasting(ds: Dataset):
     int_tensor = ds.create_tensor("int", dtype="uint8")
     int_tensor.append(0)

diff --git a/hub/constants.py b/hub/constants.py
@@ -62,17 +62,20 @@
 PYTEST_MEMORY_PROVIDER_BASE_ROOT = "mem://hub_pytest"
 PYTEST_LOCAL_PROVIDER_BASE_ROOT = "/tmp/hub_pytest/"  # TODO: may fail for windows
 PYTEST_S3_PROVIDER_BASE_ROOT = "s3://hub-2.0-tests/"
+PYTEST_GCS_PROVIDER_BASE_ROOT = "gcs://snark-test/"
 PYTEST_HUB_CLOUD_PROVIDER_BASE_ROOT = f"hub://{HUB_CLOUD_DEV_USERNAME}/"
 
 # environment variables
 ENV_HUB_DEV_PASSWORD = "ACTIVELOOP_HUB_PASSWORD"
 ENV_KAGGLE_USERNAME = "KAGGLE_USERNAME"
 ENV_KAGGLE_KEY = "KAGGLE_KEY"
+ENV_GOOGLE_APPLICATION_CREDENTIALS = "GOOGLE_APPLICATION_CREDENTIALS"
 
 # pytest options
 MEMORY_OPT = "--memory-skip"
 LOCAL_OPT = "--local"
 S3_OPT = "--s3"
+GCS_OPT = "--gcs"
 HUB_CLOUD_OPT = "--hub-cloud"
 S3_PATH_OPT = "--s3-path"
 KEEP_STORAGE_OPT = "--keep-storage"

diff --git a/hub/core/compute/process.py b/hub/core/compute/process.py
@@ -9,3 +9,8 @@ def __init__(self, workers):
 
     def map(self, func, iterable):
         return self.pool.map(func, iterable)
+
+    def close(self):
+        self.pool.close()
+        self.pool.join()
+        self.pool.clear()
diff --git a/hub/core/compute/provider.py b/hub/core/compute/provider.py
@@ -13,3 +13,7 @@ def map(self, func, iterable):
         """Apply 'func' to each element in 'iterable', collecting the results
         in a list that is returned.
         """
+
+    @abstractmethod
+    def close(self):
+        """Closes the provider."""
diff --git a/hub/core/compute/serial.py b/hub/core/compute/serial.py
@@ -7,3 +7,6 @@ def __init__(self):
 
     def map(self, func, iterable):
         return map(func, iterable)
+
+    def close(self):
+        return
diff --git a/hub/core/compute/thread.py b/hub/core/compute/thread.py
@@ -9,3 +9,8 @@ def __init__(self, workers):
 
     def map(self, func, iterable):
         return self.pool.map(func, iterable)
+
+    def close(self):
+        self.pool.close()
+        self.pool.join()
+        self.pool.clear()