Merge pull request activeloopai#136 from hugovk/lint-ci

Lint with Black on GitHub Actions and format with Black
Gyanachand1 · Oct 25, 2020 · 77f54b4 · 77f54b4
2 parents 7a400c3 + 501a927
commit 77f54b4
Show file tree

Hide file tree

Showing 35 changed files with 1,157 additions and 725 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -0,0 +1,12 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: pre-commit/action@v2.0.0
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,6 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+        args: ["--target-version", "py36"]
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -15,14 +15,14 @@ To contribute a feature/fix:
 ## How you can help
 
 * Adding new datasets following [this example](https://docs.activeloop.ai/en/latest/concepts/dataset.html#how-to-upload-a-dataset)
-* Fix an issue from Github Issues
+* Fix an issue from GitHub Issues
 * Add a feature. For an extended feature please create an issue to discuss.
 
 
 ## Formatting and Linting
 Hub uses Black and Flake8 to ensure a consistent code format throughout the project.
 if you are using vscode then Replace `.vscode/settings.json` content with the following:
-```
+```json
 {
     "[py]": {
         "editor.formatOnSave": true

diff --git a/benchmarks/areal_caching_benchmark.py b/benchmarks/areal_caching_benchmark.py
@@ -24,4 +24,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/examples/coco/upload_coco2017.py b/examples/coco/upload_coco2017.py
@@ -77,7 +77,6 @@ def get_image_name(args, tag, id):
 def load_dataset(args, tag):
     with open(
         os.path.join(args.dataset_path, f"annotations/instances_{tag}{args.year}.json"),
-        "r",
     ) as f:
         instances = json.load(f)
     # print(instances.keys())

diff --git a/examples/fashion-mnist/train_pytorch.py b/examples/fashion-mnist/train_pytorch.py
@@ -7,7 +7,7 @@
 
 class CNN(nn.Module):
     def __init__(self):
-        super(CNN, self).__init__()
+        super().__init__()
         self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
         self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
         self.conv2_drop = nn.Dropout2d()
@@ -49,13 +49,19 @@ def test(model, test_loader):
             labels = batch["labels"]
             labels = labels.type(torch.LongTensor)
             output = model(data)
-            test_loss += F.nll_loss(output, labels, reduction='sum').item()
+            test_loss += F.nll_loss(output, labels, reduction="sum").item()
             pred = output.data.max(1, keepdim=True)[1]
             correct += pred.eq(labels.data.view_as(pred)).sum()
 
     test_loss /= len(test_loader.dataset)
-    print('Test set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
-        test_loss, correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset)))
+    print(
+        "Test set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format(
+            test_loss,
+            correct,
+            len(test_loader.dataset),
+            100.0 * correct / len(test_loader.dataset),
+        )
+    )
 
 
 def main():
@@ -78,29 +84,33 @@ def main():
     train_dataset = torch.utils.data.Subset(ds, range(60000))
     test_dataset = torch.utils.data.Subset(ds, range(60000, 70000))
 
-    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn)
-    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn)
+    train_loader = torch.utils.data.DataLoader(
+        train_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn
+    )
+    test_loader = torch.utils.data.DataLoader(
+        test_dataset, batch_size=BATCH_SIZE, collate_fn=ds.collate_fn
+    )
 
     model = CNN()
     optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM)
 
     for epoch in range(EPOCHS):
-        print("Starting Training Epoch {}".format(epoch))
+        print(f"Starting Training Epoch {epoch}")
         train(model, train_loader, optimizer)
-        print("Training Epoch {} finished\n".format(epoch))
+        print(f"Training Epoch {epoch} finished\n")
         test(model, test_loader)
 
     # sanity check to see outputs of model
     for batch in test_loader:
-        print("\nNamed Labels:",dataset.get_text(batch["named_labels"]))
-        print("\nLabels:",batch["labels"])
+        print("\nNamed Labels:", dataset.get_text(batch["named_labels"]))
+        print("\nLabels:", batch["labels"])
 
         data = batch["data"]
         data = torch.unsqueeze(data, 1)
 
         output = model(data)
         pred = output.data.max(1)[1]
-        print("\nPredictions:",pred)
+        print("\nPredictions:", pred)
         break
 
 

diff --git a/examples/fashion-mnist/train_tf_fit.py b/examples/fashion-mnist/train_tf_fit.py
@@ -4,16 +4,28 @@
 
 def create_CNN():
     model = tf.keras.Sequential()
-    model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28, 28, 1)))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=64,
+            kernel_size=2,
+            padding="same",
+            activation="relu",
+            input_shape=(28, 28, 1),
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
-    model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=32, kernel_size=2, padding="same", activation="relu"
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
     model.add(tf.keras.layers.Flatten())
-    model.add(tf.keras.layers.Dense(256, activation='relu'))
+    model.add(tf.keras.layers.Dense(256, activation="relu"))
     model.add(tf.keras.layers.Dropout(0.5))
-    model.add(tf.keras.layers.Dense(10, activation='softmax'))
+    model.add(tf.keras.layers.Dense(10, activation="softmax"))
     return model
 
 
@@ -33,7 +45,7 @@ def main():
 
     # transform into Tensorflow dataset
     # max_text_len is an optional argument that fixes the maximum length of text labels
-    ds = ds.to_tensorflow(max_text_len = 15)
+    ds = ds.to_tensorflow(max_text_len=15)
 
     # converting ds so that it can be directly used in model.fit
     ds = ds.map(lambda x: to_model_fit(x))
@@ -47,8 +59,12 @@ def main():
 
     model = create_CNN()
     # model.summary()
-    model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
-    model.fit(train_dataset, epochs=EPOCHS, validation_data=test_dataset, validation_steps=1)
+    model.compile(
+        loss="sparse_categorical_crossentropy", optimizer="adam", metrics=["accuracy"]
+    )
+    model.fit(
+        train_dataset, epochs=EPOCHS, validation_data=test_dataset, validation_steps=1
+    )
 
 
 if __name__ == "__main__":

diff --git a/examples/fashion-mnist/train_tf_gradient_tape.py b/examples/fashion-mnist/train_tf_gradient_tape.py
@@ -7,16 +7,28 @@
 
 def create_CNN():
     model = tf.keras.Sequential()
-    model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28, 28, 1)))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=64,
+            kernel_size=2,
+            padding="same",
+            activation="relu",
+            input_shape=(28, 28, 1),
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
-    model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))
+    model.add(
+        tf.keras.layers.Conv2D(
+            filters=32, kernel_size=2, padding="same", activation="relu"
+        )
+    )
     model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
     model.add(tf.keras.layers.Dropout(0.3))
     model.add(tf.keras.layers.Flatten())
-    model.add(tf.keras.layers.Dense(256, activation='relu'))
+    model.add(tf.keras.layers.Dense(256, activation="relu"))
     model.add(tf.keras.layers.Dropout(0.5))
-    model.add(tf.keras.layers.Dense(10, activation='softmax'))
+    model.add(tf.keras.layers.Dense(10, activation="softmax"))
     return model
 
 
@@ -32,7 +44,7 @@ def train(model, train_dataset, optimizer, loss_fn, train_acc_metric):
         train_acc_metric.update_state(batch["labels"], pred)
 
     train_acc = train_acc_metric.result()
-    print("Training acc: %.4f" % (float(train_acc),))
+    print("Training acc: {:.4f}".format(float(train_acc)))
     train_acc_metric.reset_states()
 
 
@@ -43,7 +55,7 @@ def test(model, test_dataset, test_acc_metric):
         test_acc_metric.update_state(batch["labels"], pred)
 
     test_acc = test_acc_metric.result()
-    print("Test acc: %.4f" % (float(test_acc),))
+    print("Test acc: {:.4f}".format(float(test_acc)))
     test_acc_metric.reset_states()
 
 
@@ -61,7 +73,7 @@ def main():
 
     # transform into Tensorflow dataset
     # max_text_len is an optional argument that sets the maximum length of text labels, default is 30
-    ds = ds.to_tensorflow(max_text_len = 15)
+    ds = ds.to_tensorflow(max_text_len=15)
 
     # Splitting back into the original train and test sets
     train_dataset = ds.take(60000)
@@ -74,21 +86,22 @@ def main():
     # model.summary()
 
     for epoch in range(EPOCHS):
-        print("\nStarting Training Epoch {}".format(epoch))
+        print(f"\nStarting Training Epoch {epoch}")
         train(model, train_dataset, optimizer, loss_fn, train_acc_metric)
-        print("Training Epoch {} finished\n".format(epoch))
+        print(f"Training Epoch {epoch} finished\n")
         test(model, test_dataset, test_acc_metric)
 
     # sanity check to see outputs of model
     for batch in test_dataset:
-        print("\nNamed Labels:",dataset.get_text(batch["named_labels"]))
-        print("\nLabels:",batch["labels"])
+        print("\nNamed Labels:", dataset.get_text(batch["named_labels"]))
+        print("\nLabels:", batch["labels"])
 
         output = model(tf.expand_dims(batch["data"], axis=3), training=False)
         print(type(output))
         pred = np.argmax(output, axis=-1)
-        print("\nPredictions:",pred)
+        print("\nPredictions:", pred)
         break
 
+
 if __name__ == "__main__":
     main()
diff --git a/examples/fashion-mnist/upload.py b/examples/fashion-mnist/upload.py
@@ -46,17 +46,18 @@ def main():
     dicts = []
 
     # required to generate named labels
-    mapping = {0: "T-shirt/top",
-               1: "Trouser",
-               2: "Pullover",
-               3: "Dress",
-               4: "Coat",
-               5: "Sandal",
-               6: "Shirt",
-               7: "Sneaker",
-               8: "Bag",
-               9: "Ankle boot"
-               }
+    mapping = {
+        0: "T-shirt/top",
+        1: "Trouser",
+        2: "Pullover",
+        3: "Dress",
+        4: "Coat",
+        5: "Sandal",
+        6: "Shirt",
+        7: "Sneaker",
+        8: "Bag",
+        9: "Ankle boot",
+    }
 
     for f in files:
         images, labels = load_fashion_mnist(f, path="./data/fashion-mnist")
@@ -71,7 +72,9 @@ def main():
     labels_t = tensor.from_array(labels, dtag="text")
     named_labels_t = tensor.from_array(named_labels, dtag="text")
 
-    ds = dataset.from_tensors({"data": images_t, "labels": labels_t, "named_labels": named_labels_t})
+    ds = dataset.from_tensors(
+        {"data": images_t, "labels": labels_t, "named_labels": named_labels_t}
+    )
     ds.store("mnist/fashion-mnist")
 
 

diff --git a/examples/mnist/upload.py b/examples/mnist/upload.py
@@ -52,7 +52,11 @@ def main():
         default="./data/mnist",
     )
     parser.add_argument(
-        "-o", "--output_name", type=str, help="Dataset output name", default="mnist",
+        "-o",
+        "--output_name",
+        type=str,
+        help="Dataset output name",
+        default="mnist",
     )
     args = parser.parse_args()
     files = ["training", "testing"]

diff --git a/hub/areal/tests/test_storage_tensor.py b/hub/areal/tests/test_storage_tensor.py
@@ -96,4 +96,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/hub/cli/auth.py b/hub/cli/auth.py
@@ -71,4 +71,4 @@ def register(username, email, password):
 
     AuthClient().register(username, email, password)
     token = AuthClient().get_access_token(username, password)
-    TokenManager.set_token(token)
+    TokenManager.set_token(token)
diff --git a/hub/cli/command.py b/hub/cli/command.py
@@ -9,7 +9,7 @@
 @click.option(
     "-h",
     "--host",
-    default="{}".format(config.HUB_REST_ENDPOINT),
+    default=f"{config.HUB_REST_ENDPOINT}",
     help="Hub rest endpoint",
 )
 @click.option("-v", "--verbose", count=True, help="Devel debugging")

diff --git a/hub/cli/utils.py b/hub/cli/utils.py
@@ -42,8 +42,10 @@ def get_proxy_command(proxy):
     ssh_proxy = ""
     if proxy and proxy != " " and proxy != "None" and proxy != "":
         if check_program_exists("ncat"):
-            ssh_proxy = '-o "ProxyCommand=ncat --proxy-type socks5 --proxy {} %h %p"'.format(
-                proxy
+            ssh_proxy = (
+                '-o "ProxyCommand=ncat --proxy-type socks5 --proxy {} %h %p"'.format(
+                    proxy
+                )
             )
         else:
             raise HubException(

diff --git a/hub/client/auth.py b/hub/client/auth.py
@@ -10,10 +10,10 @@ class AuthClient(HubHttpClient):
     """
 
     def __init__(self):
-        super(AuthClient, self).__init__()
+        super().__init__()
 
     def check_token(self, access_token):
-        auth = "Bearer {}".format(access_token)
+        auth = f"Bearer {access_token}"
         response = self.request(
             "GET", config.CHECK_TOKEN_REST_SUFFIX, headers={"Authorization": auth}
         )
@@ -22,7 +22,7 @@ def check_token(self, access_token):
             response_dict = response.json()
             is_valid = response_dict["is_valid"]
         except Exception as e:
-            logger.error("Exception occured while validating token: {}.".format(e))
+            logger.error(f"Exception occured while validating token: {e}.")
             raise HubException(
                 "Error while validating the token. \
                                   Please try logging in using username ans password."
@@ -41,7 +41,7 @@ def get_access_token(self, username, password):
             token_dict = response.json()
             token = token_dict["token"]
         except Exception as e:
-            logger.error("Exception occured while getting token: {}.".format(e))
+            logger.error(f"Exception occured while getting token: {e}.")
             raise HubException(
                 "Error while loggin in. \
                                   Please try logging in using access token."
Original file line number	Diff line number	Diff line change
Expand Up		@@ -24,4 +24,4 @@ def main():


		if __name__ == "__main__":
		main()
		main()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -96,4 +96,4 @@ def main():


		if __name__ == "__main__":
		main()
		main()