merge from master

Gyanachand1 · Nov 30, 2020 · 83599ac · 83599ac
2 parents 49fdd3b + 933c244
commit 83599ac
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 28 deletions.
diff --git a/README.md b/README.md
@@ -3,33 +3,38 @@
     <img src="https://raw.githubusercontent.com/snarkai/Hub/master/docs/logo/hub_logo.png" width="50%"/>
     </br>
 </p>
+
 <p align="center">
     <a href="http://docs.activeloop.ai/">
         <img alt="Docs" src="https://readthedocs.org/projects/hubdb/badge/?version=latest">
     </a>
     <a href="https://pypi.org/project/hub/"><img src="https://badge.fury.io/py/hub.svg" alt="PyPI version" height="18"></a>
     <a href="https://pypi.org/project/hub/"><img src="https://img.shields.io/pypi/dm/hub.svg" alt="PyPI version" height="18"></a>
     <a href="https://app.circleci.com/pipelines/github/activeloopai/Hub">
-    <img alt="CircleCI" src="https://img.shields.io/circleci/build/github/activeloopai/Hub?logo=circleci">
-    </a>
+    <img alt="CircleCI" src="https://img.shields.io/circleci/build/github/activeloopai/Hub?logo=circleci"> </a>
+     <a href="https://github.com/activeloopai/Hub/issues">
+    <img alt="GitHub issues" src="https://img.shields.io/github/issues/activeloopai/Hub"> </a>
     <a href="https://codecov.io/gh/activeloopai/Hub/branch/master"><img src="https://codecov.io/gh/activeloopai/Hub/branch/master/graph/badge.svg" alt="codecov" height="18"></a>
-    <a href="https://twitter.com/intent/tweet?text=The%20fastest%20way%20to%20access%20and%20manage%20PyTorch%20and%20Tensorflow%20datasets%20is%20open-source&url=https://activeloop.ai/&via=activeloopai&hashtags=opensource,pytorch,tensorflow,data,datascience,datapipelines,sqlforimages,activeloop"> 
-        <img alt="tweet" src="https://img.shields.io/twitter/url/http/shields.io.svg?style=social">
-    </a>  
-    <br />
-<a href="https://join.slack.com/t/hubdb/shared_invite/zt-ivhsj8sz-GWv9c5FLBDVw8vn~sxRKqQ">
+    <a href="https://twitter.com/intent/tweet?text=The%20fastest%20way%20to%20access%20and%20manage%20PyTorch%20and%20Tensorflow%20datasets%20is%20open-source&url=https://activeloop.ai/&via=activeloopai&hashtags=opensource,pytorch,tensorflow,data,datascience,datapipelines,activeloop,dockerhubfordatasets"> 
+        <img alt="tweet" src="https://img.shields.io/twitter/url/http/shields.io.svg?style=social"> </a>  
+   </br> 
+    <a href="https://join.slack.com/t/hubdb/shared_invite/zt-ivhsj8sz-GWv9c5FLBDVw8vn~sxRKqQ">
   <img src="https://user-images.githubusercontent.com/13848158/97266254-9532b000-1841-11eb-8b06-ed73e99c2e5f.png" height="35" />
-</a>
+    </a>
 
-  </a>
+---
+
+</a>
 </p>
 
-<h3 align="center">
-The fastest way to access and manage datasets for PyTorch and TensorFlow
-</h3>
 
-Hub provides the fastest access to the state-of-the-art datasets for Deep Learning, enabling data scientists to manage them, build scalable data pipelines and connect to Pytorch and Tensorflow.
+Activeloop's Hacktoberfest is extended till Nov 27! Check out our [Hacktoberfest Project Dashboard](https://github.com/activeloopai/Hub/projects/2) to contribute! 
+
+
+<h3 align="center"> The Docker Hub for datasets. </h3>
+<h4 align="center"> Hub is the fastest way to access & manage datasets for PyTorch and TensorFlow, and build scalable data pipelines.</h4>
 
+---
 
 ### Contributors
 

diff --git a/examples/old/MPII Human Pose Dataset/mpii_data_upload.py b/examples/old/MPII Human Pose Dataset/mpii_data_upload.py
@@ -1,10 +1,18 @@
+"""
+Folder Structure:
+mpii_human_pose_v1
+  -images
+  -mpii_annotations.json
+"""
+
+
+import argparse
 import os
 import json
 import time
 import numpy as np
 from PIL import Image
 
-import hub
 from hub.collections import dataset
 from hub.log import logger
 
@@ -48,8 +56,6 @@ def __call__(self, input):
 
         try:
             ds = {}
-            img_path = "/home/sanchit/images/"
-
             n = 1  # for 1 row
             ds["image"] = np.empty(n, object)
             ds["dataset"] = np.empty(n, object)
@@ -67,7 +73,11 @@ def __call__(self, input):
             ds["people_index"] = np.empty(n, object)
             ds["numOtherPeople"] = np.empty(n, object)
 
-            ds["image"][0] = np.array(Image.open(img_path + input["img_paths"]))
+            ds["image"][0] = np.array(
+                Image.open(
+                    os.path.join(self._args.dataset_path, "images", input["img_paths"])
+                )
+            )
             ds["dataset"][0] = input["dataset"]
             ds["isValidation"][0] = input["isValidation"]
             ds["img_paths"][0] = input["img_paths"]
@@ -93,7 +103,7 @@ def __call__(self, input):
             logger.error(e, exc_info=e, stack_info=True)
 
 
-def load_dataset():
+def load_dataset(args):
     """
     This function is used to load json annotations in the form of dictionary and then
     appending all the dictionaries(25205 examples) in the list named annotations.
@@ -102,7 +112,7 @@ def load_dataset():
     in the list(annotations). Finally it returns the complete dataset with all examples.
     """
 
-    with open("/home/sanchit/mpii_annotations.json", "r") as f:
+    with open(os.path.join(args.dataset_path, "mpii_annotations.json"), "r") as f:
 
         instances = json.load(f)
 
@@ -120,10 +130,30 @@ def load_dataset():
     return ds
 
 
-t1 = time.time()
-# Call the load_dataset function to generate the complete dataset
-ds = load_dataset()
-# ds.store stores the dataset in username/MPII_human_pose_data
-ds.store("MPII_human_pose_data")
-t2 = time.time()
-print(f"Pipeline took {(t2 - t1) / 60} minutes")
+def main():
+    t1 = time.time()
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "dataset_path",
+        metavar="P",
+        type=str,
+        help="Path to MPII Human Pose Dataset",
+        default="./mpii_human_pose_v1",
+    )
+    parser.add_argument(
+        "output_path",
+        metavar="N",
+        type=str,
+        help="Dataset output path",
+        default="MPII_human_pose_data",
+    )
+
+    args = parser.parse_args()
+    ds = load_dataset(args)
+    ds.store(f"{args.output_path}")
+    t2 = time.time()
+    logger.info(f"Pipeline took {(t2 - t1) / 60} minutes")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -2,4 +2,4 @@ pytest==6.1.2
 pytest-cov==2.10.1
 flake8==3.8.4
 black==20.8b1
-ray==1.0.0
+ray==1.0.0
diff --git a/requirements.txt b/requirements.txt
@@ -2,7 +2,7 @@ click>=6.7,<8
 numpy>=1.13.0,<2 
 requests>=2,<3
 cachey==0.2.1
-fsspec==0.7.4
+fsspec==0.8.4
 s3fs==0.4.2
 gcsfs==0.6.2
 outdated==0.2.0