test fixes

lzlz99 · Oct 13, 2020 · 53e646c · 53e646c
1 parent 6e36d42
commit 53e646c
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 50 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -9,13 +9,13 @@ workflows:
       - test:
           context: 
             # - test
-            - test-aws
+            # - test-aws
   test_and_deploy:
     jobs:
       - test:
           context: 
             # - test
-            - test-aws
+            # - test-aws
           filters:
             tags:
               only: /[0-9]+(\.[0-9]+)*/

diff --git a/hub/cli/auth.py b/hub/cli/auth.py
@@ -58,17 +58,14 @@ def register(username, email, password):
         logger.debug("Prompting for username.")
         username = click.prompt("Username", type=str)
     username = username.strip()
-
     if not email:
         logger.debug("Prompting for email.")
         email = click.prompt("Email", type=str)
     email = email.strip()
-
     if not password:
         logger.debug("Prompting for password.")
         password = click.prompt("Password", type=str, hide_input=True)
     password = password.strip()
-
     AuthClient().register(username, email, password)
     token = AuthClient().get_access_token(username, password)
-    TokenManager.set_token(token)
+    TokenManager.set_token(token)
diff --git a/hub/client/base.py b/hub/client/base.py
@@ -61,6 +61,7 @@ def request(
 
         try:
             logger.debug("Sending: Headers {}, Json: {}".format(headers, json))
+            print(method, request_url, params, data)
             response = requests.request(
                 method,
                 request_url,

diff --git a/hub/collections/dataset/core.py b/hub/collections/dataset/core.py
@@ -17,11 +17,6 @@
 from hub.log import logger
 from hub.exceptions import PermissionException
 
-try:
-    import torch
-except ImportError:
-    pass
-
 
 def _flatten(l):
     """
@@ -751,6 +746,8 @@ def _is_tensor_dynamic(tensor):
 
 
 class TorchDataset:
+    import torch
+
     def __init__(self, ds, transform=None, max_text_len=30):
         self._ds = ds
         self._transform = transform

diff --git a/hub/collections/tests/test_collections.py b/hub/collections/tests/test_collections.py
@@ -1,6 +1,7 @@
 import numpy as np
 import pytest
 from hub.collections import dataset, tensor
+from hub.utils import gcp_creds_exist, s3_creds_exist, tensorflow_loaded, pytorch_loaded
 
 
 def test_tensor_getitem0():
@@ -187,6 +188,7 @@ def test_unknown_size_input():
     assert (ds["arr"][5].compute() == np.array([2, 3, 4, 5, 6], dtype="int32")).all()
 
 
+@pytest.mark.skipif(s3_creds_exist(), reason="requires s3 credentials")
 def test_s3_dataset():
     ds = dataset.generate(UnknownCountGenerator(), range(1, 3))
     assert ds["arr"].shape == (-1, 5)
@@ -198,6 +200,7 @@ def test_s3_dataset():
     assert (ds["arr"][1].compute() == np.array([0, 1, 2, 3, 4], dtype="int32")).all()
 
 
+@pytest.mark.skipif(gcp_creds_exist(), reason="requires gcs credentials")
 def test_gcs_dataset():
     ds = dataset.generate(UnknownCountGenerator(), range(1, 3))
     assert ds["arr"].shape == (-1, 5)
@@ -209,13 +212,9 @@ def test_gcs_dataset():
     assert (ds["arr"][1].compute() == np.array([0, 1, 2, 3, 4], dtype="int32")).all()
 
 
+@pytest.mark.skipif(pytorch_loaded(), reason="requires pytorch to be loaded")
 def test_to_pytorch():
-
-    try:
-        import torch
-    except ImportError:
-        print("Pytorch hasn't been imported and tested")
-        return
+    import torch
 
     t1 = tensor.from_array(np.array([[1, 2], [3, 4]], dtype="int32"))
     np_arr = np.empty(2, object)
@@ -239,13 +238,13 @@ def test_to_pytorch():
     assert data[1]["t2"][0] == [7, 8, 9]
 
 
+@pytest.mark.skipif(
+    tensorflow_loaded() or pytorch_loaded(),
+    reason="requires both pytorch and tensorflow to be loaded",
+)
 def test_to_backend_with_tf_and_pytorch():
-    try:
-        import torch
-        import tensorflow as tf
-    except ImportError:
-        print("Pytorch hasn't been imported and tested")
-        return
+    import tensorflow as tf
+    import torch
 
     tf.compat.v1.enable_eager_execution()
     ds = dataset.load("mnist/mnist")
@@ -267,13 +266,13 @@ def test_to_backend_with_tf_and_pytorch():
             break
 
 
+@pytest.mark.skipif(
+    tensorflow_loaded() or pytorch_loaded(),
+    reason="requires both pytorch and tensorflow to be loaded",
+)
 def test_to_backend_with_tf_and_pytorch_multiworker():
-    try:
-        import torch
-        import tensorflow as tf
-    except ImportError:
-        print("Pytorch hasn't been imported and tested")
-        return
+    import tensorflow as tf
+    import torch
 
     tf.compat.v1.enable_eager_execution()
     ds = dataset.load("mnist/mnist")

diff --git a/hub/store/storage_tensor.py b/hub/store/storage_tensor.py
@@ -1,20 +1,15 @@
-from hub.exceptions import (
-    StorageTensorNotFoundException,
-)
-
+from hub.exceptions import StorageTensorNotFoundException
 import typing
 import math
 import json
-
 import zarr
 import numpy as np
+from hub.utils import get_fs_and_path, get_storage_map
+from hub.log import logger
+from hub.store.tensor import Tensor
 
-import hub.store.tensor
-import hub.store.store
-import hub.utils as utils
 
-
-class StorageTensor(hub.store.tensor.Tensor):
+class StorageTensor(Tensor):
     @classmethod
     def _determine_chunksizes(cls, shape, dtype):
         sz = np.dtype(dtype).itemsize
@@ -41,11 +36,16 @@ def __init__(
         fs=None,
         fs_map=None,
     ):
-        fs, path = (fs, url) if fs else utils.get_fs_and_path(url, token=token)
+        fs, path = (fs, url) if fs else get_fs_and_path(url, token=token)
         if ("w" in mode or "a" in mode) and not fs.exists(path):
             fs.makedirs(path)
-        fs_map = fs_map or utils.get_storage_map(fs, path, memcache)
-        exist_ = bool(fs_map.get(".hub.storage_tensor"))
+        fs_map = fs_map or get_storage_map(fs, path, memcache)
+
+        try:
+            exist_ = bool(fs_map.get(".hub.storage_tensor"))
+        except Exception as e:
+            logger.error(e)
+            exist_ = False
         # if not exist_ and len(fs_map) > 0 and "w" in mode:
         #     raise OverwriteIsNotSafeException()
         exist = False if "w" in mode else exist_

diff --git a/hub/store/store.py b/hub/store/store.py
diff --git a/hub/store/tests/test_storage_tensor.py b/hub/store/tests/test_storage_tensor.py
@@ -1,5 +1,7 @@
 import numpy as np
+import pytest
 from hub.store.storage_tensor import StorageTensor
+from hub.utils import gcp_creds_exist, s3_creds_exist
 
 
 def test_open():
@@ -16,6 +18,7 @@ def test_open():
     assert tensor.dtype == "int32"
 
 
+@pytest.mark.skipif(s3_creds_exist(), reason="requires s3 credentials")
 def test_s3_open():
     StorageTensor(
         "s3://snark-test/test_storage_tensor/test_s3_open",
@@ -30,6 +33,7 @@ def test_s3_open():
     assert tensor.dtype == "int32"
 
 
+@pytest.mark.skipif(gcp_creds_exist(), reason="requires gcp credentials")
 def test_gcs_open():
     StorageTensor(
         "gcs://snark-test/test_storage_tensor/test_gcs_open",
@@ -61,8 +65,12 @@ def test_memcache():
 
 
 def test_hubbackend():
+    from hub import config
+
+    config.HUB_REST_ENDPOINT = config.HUB_DEV_REST_ENDPOINT
+
     tensor = StorageTensor(
-        "davit5/mnist",
+        "testtest/mnist",
         mode="r",
         shape=[200, 100, 100],
         dtype="float32",
@@ -73,11 +81,6 @@ def test_hubbackend():
     assert tensor.dtype == "float32"
 
 
-def main():
-    # test_overwrite_safety()
-    test_memcache()
-
-
 if __name__ == "__main__":
     # test_s3_open()
     test_hubbackend()
diff --git a/hub/utils.py b/hub/utils.py
@@ -174,5 +174,39 @@ def commit(self):
         self._fs_map.commit()
 
 
-if __name__ == "__main__":
-    test_meta_storage()
+def gcp_creds_exist():
+    """Checks if credentials exists"""
+    from google.cloud import storage
+
+    try:
+        storage.Client()
+    except Exception:
+        return False
+    return True
+
+
+def s3_creds_exist():
+    import boto3
+
+    sts = boto3.client("sts")
+    try:
+        sts.get_caller_identity()
+    except boto3.exceptions.ClientError:
+        return False
+    return True
+
+
+def pytorch_loaded():
+    try:
+        import torch
+    except ImportError:
+        return False
+    return True
+
+
+def tensorflow_loaded():
+    try:
+        import tensorflow
+    except ImportError:
+        return False
+    return True