Format with Black

activeloopai · Oct 25, 2020 · 501a927 · 501a927
1 parent 61562f0
commit 501a927
Show file tree

Hide file tree

Showing 27 changed files with 1,060 additions and 659 deletions.
diff --git a/benchmarks/areal_caching_benchmark.py b/benchmarks/areal_caching_benchmark.py
@@ -24,4 +24,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/examples/fashion-mnist/train_pytorch.py b/examples/fashion-mnist/train_pytorch.py
@@ -49,13 +49,19 @@ def test(model, test_loader):
             labels = batch["labels"]
             labels = labels.type(torch.LongTensor)
             output = model(data)
-            test_loss += F.nll_loss(output, labels, reduction='sum').item()
+            test_loss += F.nll_loss(output, labels, reduction="sum").item()
             pred = output.data.max(1, keepdim=True)[1]
             correct += pred.eq(labels.data.view_as(pred)).sum()
 
     test_loss /= len(test_loader.dataset)
-    print('Test set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
-        test_loss, correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset)))
+    print(
+        "Test set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format(
+            test_loss,
+            correct,
+            len(test_loader.dataset),
+            100.0 * correct / len(test_loader.dataset),
+        )
+    )
 
 
 def main():
@@ -78,8 +84,12 @@ def main():
     train_dataset = torch.utils.data.Subset(ds, range(60000))
     test_dataset = torch.utils.data.Subset(ds, range(60000, 70000))
 
-    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn)
-    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn)
+    train_loader = torch.utils.data.DataLoader(
+        train_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn
+    )
+    test_loader = torch.utils.data.DataLoader(
+        test_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn
+    )
 
     model = CNN()
     optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM)
@@ -92,15 +102,15 @@ def main():
 
     # sanity check to see outputs of model
     for batch in test_loader:
-        print("\nNamed Labels:",dataset.get_text(batch["named_labels"]))
-        print("\nLabels:",batch["labels"])
+        print("\nNamed Labels:", dataset.get_text(batch["named_labels"]))
+        print("\nLabels:", batch["labels"])
 
         data = batch["data"]
         data = torch.unsqueeze(data, 1)
 
         output = model(data)
         pred = output.data.max(1)[1]
-        print("\nPredictions:",pred)
+        print("\nPredictions:", pred)
         break
 
 

diff --git a/examples/fashion-mnist/train_tf_fit.py b/examples/fashion-mnist/train_tf_fit.py
@@ -4,16 +4,28 @@
 
 def create_CNN():
     model = tf.keras.Sequential()
-    model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28, 28, 1)))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=64,
+            kernel_size=2,
+            padding="same",
+            activation="relu",
+            input_shape=(28, 28, 1),
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
-    model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=32, kernel_size=2, padding="same", activation="relu"
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
     model.add(tf.keras.layers.Flatten())
-    model.add(tf.keras.layers.Dense(256, activation='relu'))
+    model.add(tf.keras.layers.Dense(256, activation="relu"))
     model.add(tf.keras.layers.Dropout(0.5))
-    model.add(tf.keras.layers.Dense(10, activation='softmax'))
+    model.add(tf.keras.layers.Dense(10, activation="softmax"))
     return model
 
 
@@ -33,7 +45,7 @@ def main():
 
     # transform into Tensorflow dataset
     # max_text_len is an optional argument that fixes the maximum length of text labels
-    ds = ds.to_tensorflow(max_text_len = 15)
+    ds = ds.to_tensorflow(max_text_len=15)
 
     # converting ds so that it can be directly used in model.fit
     ds = ds.map(lambda x: to_model_fit(x))
@@ -47,8 +59,12 @@ def main():
 
     model = create_CNN()
     # model.summary()
-    model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
-    model.fit(train_dataset, epochs=EPOCHS, validation_data=test_dataset, validation_steps=1)
+    model.compile(
+        loss="sparse_categorical_crossentropy", optimizer="adam", metrics=["accuracy"]
+    )
+    model.fit(
+        train_dataset, epochs=EPOCHS, validation_data=test_dataset, validation_steps=1
+    )
 
 
 if __name__ == "__main__":

diff --git a/examples/fashion-mnist/train_tf_gradient_tape.py b/examples/fashion-mnist/train_tf_gradient_tape.py
@@ -7,16 +7,28 @@
 
 def create_CNN():
     model = tf.keras.Sequential()
-    model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28, 28, 1)))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=64,
+            kernel_size=2,
+            padding="same",
+            activation="relu",
+            input_shape=(28, 28, 1),
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
-    model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=32, kernel_size=2, padding="same", activation="relu"
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
     model.add(tf.keras.layers.Flatten())
-    model.add(tf.keras.layers.Dense(256, activation='relu'))
+    model.add(tf.keras.layers.Dense(256, activation="relu"))
     model.add(tf.keras.layers.Dropout(0.5))
-    model.add(tf.keras.layers.Dense(10, activation='softmax'))
+    model.add(tf.keras.layers.Dense(10, activation="softmax"))
     return model
 
 
@@ -61,7 +73,7 @@ def main():
 
     # transform into Tensorflow dataset
     # max_text_len is an optional argument that sets the maximum length of text labels, default is 30
-    ds = ds.to_tensorflow(max_text_len = 15)
+    ds = ds.to_tensorflow(max_text_len=15)
 
     # Splitting back into the original train and test sets
     train_dataset = ds.take(60000)
@@ -81,14 +93,15 @@ def main():
 
     # sanity check to see outputs of model
     for batch in test_dataset:
-        print("\nNamed Labels:",dataset.get_text(batch["named_labels"]))
-        print("\nLabels:",batch["labels"])
+        print("\nNamed Labels:", dataset.get_text(batch["named_labels"]))
+        print("\nLabels:", batch["labels"])
 
         output = model(tf.expand_dims(batch["data"], axis=3), training=False)
         print(type(output))
         pred = np.argmax(output, axis=-1)
-        print("\nPredictions:",pred)
+        print("\nPredictions:", pred)
         break
 
+
 if __name__ == "__main__":
     main()
diff --git a/examples/fashion-mnist/upload.py b/examples/fashion-mnist/upload.py
@@ -46,17 +46,18 @@ def main():
     dicts = []
 
     # required to generate named labels
-    mapping = {0: "T-shirt/top",
-               1: "Trouser",
-               2: "Pullover",
-               3: "Dress",
-               4: "Coat",
-               5: "Sandal",
-               6: "Shirt",
-               7: "Sneaker",
-               8: "Bag",
-               9: "Ankle boot"
-               }
+    mapping = {
+        0: "T-shirt/top",
+        1: "Trouser",
+        2: "Pullover",
+        3: "Dress",
+        4: "Coat",
+        5: "Sandal",
+        6: "Shirt",
+        7: "Sneaker",
+        8: "Bag",
+        9: "Ankle boot",
+    }
 
     for f in files:
         images, labels = load_fashion_mnist(f, path="./data/fashion-mnist")
@@ -71,7 +72,9 @@ def main():
     labels_t = tensor.from_array(labels, dtag="text")
     named_labels_t = tensor.from_array(named_labels, dtag="text")
 
-    ds = dataset.from_tensors({"data": images_t, "labels": labels_t, "named_labels": named_labels_t})
+    ds = dataset.from_tensors(
+        {"data": images_t, "labels": labels_t, "named_labels": named_labels_t}
+    )
     ds.store("mnist/fashion-mnist")
 
 

diff --git a/examples/mnist/upload.py b/examples/mnist/upload.py
@@ -52,7 +52,11 @@ def main():
         default="./data/mnist",
     )
     parser.add_argument(
-        "-o", "--output_name", type=str, help="Dataset output name", default="mnist",
+        "-o",
+        "--output_name",
+        type=str,
+        help="Dataset output name",
+        default="mnist",
     )
     args = parser.parse_args()
     files = ["training", "testing"]

diff --git a/hub/areal/tests/test_storage_tensor.py b/hub/areal/tests/test_storage_tensor.py
@@ -96,4 +96,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/hub/cli/auth.py b/hub/cli/auth.py
@@ -71,4 +71,4 @@ def register(username, email, password):
 
     AuthClient().register(username, email, password)
     token = AuthClient().get_access_token(username, password)
-    TokenManager.set_token(token)
+    TokenManager.set_token(token)
diff --git a/hub/cli/utils.py b/hub/cli/utils.py
@@ -42,8 +42,10 @@ def get_proxy_command(proxy):
     ssh_proxy = ""
     if proxy and proxy != " " and proxy != "None" and proxy != "":
         if check_program_exists("ncat"):
-            ssh_proxy = '-o "ProxyCommand=ncat --proxy-type socks5 --proxy {} %h %p"'.format(
-                proxy
+            ssh_proxy = (
+                '-o "ProxyCommand=ncat --proxy-type socks5 --proxy {} %h %p"'.format(
+                    proxy
+                )
             )
         else:
             raise HubException(

diff --git a/hub/client/base.py b/hub/client/base.py
@@ -99,9 +99,7 @@ def check_response_status(self, response):
             except Exception:
                 message = " "
 
-            logger.debug(
-                f'Error received: status code: {code}, message: "{message}"'
-            )
+            logger.debug(f'Error received: status code: {code}, message: "{message}"')
             if code == 400:
                 raise BadRequestException(response)
             elif response.status_code == 401:

diff --git a/hub/client/hub_control.py b/hub/client/hub_control.py
@@ -38,7 +38,9 @@ def get_credentials(self):
             self.auth_header = f"Bearer {token}"
 
         r = self.request(
-            "GET", config.GET_CREDENTIALS_SUFFIX, endpoint=config.HUB_REST_ENDPOINT,
+            "GET",
+            config.GET_CREDENTIALS_SUFFIX,
+            endpoint=config.HUB_REST_ENDPOINT,
         ).json()
 
         details = {

diff --git a/hub/client/token_manager.py b/hub/client/token_manager.py
@@ -13,9 +13,7 @@ def is_authenticated(cls):
 
     @classmethod
     def set_token(cls, token):
-        logger.debug(
-            f"Putting the key {token} into {config.TOKEN_FILE_PATH}."
-        )
+        logger.debug(f"Putting the key {token} into {config.TOKEN_FILE_PATH}.")
         path = Path(config.TOKEN_FILE_PATH)
         os.makedirs(path.parent, exist_ok=True)
         with open(config.TOKEN_FILE_PATH, "w") as f:

diff --git a/hub/codec/image.py b/hub/codec/image.py
@@ -53,4 +53,3 @@ def decode(self, content: bytes) -> np.ndarray:
                 arr = np.asarray(img)
                 array[index] = arr
             return array
-
diff --git a/hub/collections/_store_version.py b/hub/collections/_store_version.py
@@ -1,2 +1 @@
 CURRENT_STORE_VERSION = 1
-
diff --git a/hub/collections/client_manager.py b/hub/collections/client_manager.py
@@ -36,18 +36,18 @@ def init(
 ):
     """Initializes cluster either local or on the cloud
 
-        Parameters
-        ----------
-        token: str
-            token provided by snark
-        cache: float
-            Amount on local memory to cache locally, default 2e9 (2GB)
-        cloud: bool
-            Should be run locally or on the cloud
-        n_workers: int
-            number of concurrent workers, default to1
-        threads_per_worker: int
-            Number of threads per each worker
+    Parameters
+    ----------
+    token: str
+        token provided by snark
+    cache: float
+        Amount on local memory to cache locally, default 2e9 (2GB)
+    cloud: bool
+        Should be run locally or on the cloud
+    n_workers: int
+        number of concurrent workers, default to1
+    threads_per_worker: int
+        Number of threads per each worker
     """
     print("initialized")
     global _client
@@ -69,9 +69,9 @@ def init(
         )
 
         local_directory = os.path.join(
-            os.path.expanduser('~'),
-            '.activeloop',
-            'tmp',
+            os.path.expanduser("~"),
+            ".activeloop",
+            "tmp",
         )
         if not os.path.exists(local_directory):
             os.makedirs(local_directory)

diff --git a/hub/collections/dataset/__init__.py b/hub/collections/dataset/__init__.py
@@ -34,7 +34,7 @@ def _meta_preprocess(meta: dict):
 
 
 def generate(generator: DatasetGenerator, input) -> Dataset:
-    """ Generates dataset based on DatabaseGenerator class instance and iterable input
+    """Generates dataset based on DatabaseGenerator class instance and iterable input
     For every element in input runs generators __call__ function.
     That function should return dict of numpy arrays containing single or multiple outputs for axis 0 of generating dataset
     """
@@ -80,8 +80,7 @@ def from_tensors(
     citation: str = None,
     howtoload: str = None,
 ) -> Dataset:
-    """ Creates a dataset from dict of tensors
-    """
+    """Creates a dataset from dict of tensors"""
     return Dataset(
         tensors,
         metainfo={
@@ -109,7 +108,7 @@ def _meta_concat(metas: Tuple[Dict[str, object]]):
 
 
 def concat(datasets: Iterable[Dataset]) -> Dataset:
-    """ Concats multiple datasets into one along axis 0
+    """Concats multiple datasets into one along axis 0
     This is equivalent to concat every tensor with the same key
     """
     keys = [sorted(dataset._tensors.keys()) for dataset in datasets]
@@ -138,8 +137,7 @@ def concat(datasets: Iterable[Dataset]) -> Dataset:
 
 
 def merge(datasets: Iterable[Dataset]) -> Dataset:
-    """ Merges multiple datasets that have distinct keys into one big datasets containing all keys
-    """
+    """Merges multiple datasets that have distinct keys into one big datasets containing all keys"""
     tensors = {}
     for dataset in datasets:
         for tname, tvalue in dataset._tensors.items():
Original file line number	Diff line number	Diff line change
Expand Up		@@ -24,4 +24,4 @@ def main():


		if __name__ == "__main__":
		main()
		main()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -96,4 +96,4 @@ def main():


		if __name__ == "__main__":
		main()
		main()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -53,4 +53,3 @@ def decode(self, content: bytes) -> np.ndarray:
		arr = np.asarray(img)
		array[index] = arr
		return array