move around shape detector

Gyanachand1 · Dec 1, 2020 · 1d4c420 · 1d4c420
1 parent bdf5de8
commit 1d4c420
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 2 deletions.
diff --git a/examples/upload_tfds.py b/examples/upload_tfds.py
@@ -5,6 +5,14 @@
     path = "./data/test/tfds_new/coco"
     with Timer("Eurosat TFDS"):
         out_ds = hub.Dataset.from_tfds("coco", num=100)
+
+        ds = hub.Dataset(
+            "./data/test/tfds_new/coco2", schema=out_ds.schema, shape=(10000,), mode="w"
+        )
+        print(out_ds.schema)
+        for key in ds.keys:
+            print(ds[key].chunksize)
+        exit()
         res_ds = out_ds.store(path)
         ds = hub.load(path)
         print(ds)

diff --git a/hub/api/dataset.py b/hub/api/dataset.py
@@ -677,7 +677,6 @@ def from_tfds(dataset, split=None, num=-1, sampling_amount=1):
         max_dict = defaultdict(lambda: None)
 
         def sampling(ds):
-            # FIXME len(ds), does not work for all iteratables
             try:
                 subset_len = len(ds) if hasattr(ds, "__len__") else num
             except Exception:

diff --git a/hub/store/dynamic_tensor.py b/hub/store/dynamic_tensor.py
@@ -9,7 +9,7 @@
 import numcodecs
 
 from hub.store.nested_store import NestedStore
-from hub.shape_detector import ShapeDetector
+from hub.store.shape_detector import ShapeDetector
 
 from hub.exceptions import (
     DynamicTensorNotFoundException,

diff --git a/hub/shape_detector.py → hub/store/shape_detector.py b/hub/shape_detector.py → hub/store/shape_detector.py