remove chunk sizes also remove hub.core.typing

gaurav879 · Jul 9, 2021 · bd7250f · bd7250f
1 parent 0c8b222
commit bd7250f
Show file tree

Hide file tree

Showing 8 changed files with 5 additions and 26 deletions.
diff --git a/conftest.py b/conftest.py
@@ -1,3 +1,4 @@
+from hub.core.storage.provider import StorageProvider
 import os
 import posixpath
 import logging
@@ -20,10 +21,10 @@
 )
 from hub.core.storage import LocalProvider, MemoryProvider, S3Provider
 from hub.core.tests.common import LOCAL, MEMORY, S3
-from hub.core.typing import StorageProvider
 from hub.tests.common import SESSION_ID, current_test_name, get_dummy_data_path
 from hub.util.cache_chain import get_cache_chain
 
+
 MEMORY_OPT = "--memory-skip"
 LOCAL_OPT = "--local"
 S3_OPT = "--s3"

diff --git a/hub/api/dataset.py b/hub/api/dataset.py
@@ -1,3 +1,4 @@
+from hub.core.storage.provider import StorageProvider
 from hub.core.tensor import create_tensor
 from hub.constants import DEFAULT_HTYPE
 from typing import Callable, Dict, Optional, Union, Tuple, List
@@ -8,7 +9,6 @@
 
 from hub.core.meta.dataset_meta import DatasetMeta
 
-from hub.core.typing import StorageProvider
 from hub.core.index import Index
 from hub.integrations import dataset_to_tensorflow
 from hub.util.keys import dataset_exists, get_dataset_meta_key, tensor_exists
@@ -143,7 +143,6 @@ def create_tensor(
         self,
         name: str,
         htype: str = DEFAULT_HTYPE,
-        chunk_size: int = None,
         dtype: Union[str, np.dtype, type] = None,
         sample_compression: str = None,
         chunk_compression: str = None,
@@ -158,9 +157,6 @@ def create_tensor(
                 For example, `htype="image"` would have `dtype` default to `uint8`.
                 These defaults can be overridden by explicitly passing any of the other parameters to this function.
                 May also modify the defaults for other parameters.
-            chunk_size (int): Optionally override this tensor's `chunk_size`. In short, `chunk_size` determines the
-                size of files (chunks) being created to represent this tensor's samples.
-                For more on chunking, check out `hub.core.chunk_engine.chunker`.
             dtype (str): Optionally override this tensor's `dtype`. All subsequent samples are required to have this `dtype`.
             sample_compression (str): Optionally override this tensor's `sample_compression`. Only used when the incoming data is uncompressed.
             chunk_compression (str): Optionally override this tensor's `chunk_compression`. Currently not implemented.
@@ -186,7 +182,6 @@ def create_tensor(
             name,
             self.storage,
             htype=htype,
-            chunk_size=chunk_size,
             dtype=dtype,
             sample_compression=sample_compression,
             chunk_compression=chunk_compression,

diff --git a/hub/core/meta/tensor_meta.py b/hub/core/meta/tensor_meta.py
@@ -32,7 +32,6 @@ class TensorMeta(Meta):
     dtype: str
     min_shape: List[int]
     max_shape: List[int]
-    chunk_size: int
     length: int
     sample_compression: str
     chunk_compression: str
@@ -149,7 +148,6 @@ def _required_meta_from_htype(htype: str) -> dict:
     required_meta = {
         "htype": htype,
         "dtype": defaults.get("dtype", None),
-        "chunk_size": defaults["chunk_size"],
         "min_shape": [],
         "max_shape": [],
         "length": 0,
@@ -185,14 +183,6 @@ def _validate_htype_overwrites(htype: str, htype_overwrite: dict):
         if key not in defaults:
             raise TensorMetaInvalidHtypeOverwriteKey(htype, key, list(defaults.keys()))
 
-    if "chunk_size" in htype_overwrite:
-        _raise_if_condition(
-            "chunk_size",
-            htype_overwrite,
-            lambda chunk_size: chunk_size <= 0,
-            "Chunk size must be greater than 0.",
-        )
-
     if "dtype" in htype_overwrite:
         _raise_if_condition(
             "dtype",

diff --git a/hub/core/tensor.py b/hub/core/tensor.py
@@ -1,8 +1,8 @@
 from hub.constants import DEFAULT_HTYPE
 from hub.core.meta.tensor_meta import TensorMeta
+from hub.core.storage import StorageProvider
 
 from hub.util.keys import get_tensor_meta_key, tensor_exists
-from hub.core.typing import StorageProvider
 from hub.util.exceptions import (
     TensorAlreadyExistsError,
 )

diff --git a/hub/core/tests/common.py b/hub/core/tests/common.py
@@ -1,11 +1,7 @@
 import pytest
 
-from typing import Dict, List
-
 import pytest
 
-from hub.core.typing import StorageProvider
-
 STORAGE_FIXTURE_NAME = "storage"
 DATASET_FIXTURE_NAME = "ds"
 

diff --git a/hub/core/typing.py b/hub/core/typing.py
diff --git a/hub/htypes.py b/hub/htypes.py
@@ -15,7 +15,6 @@
 from typing import Dict
 from hub.constants import (
     DEFAULT_CHUNK_COMPRESSION,
-    DEFAULT_MAX_CHUNK_SIZE,
     DEFAULT_HTYPE,
     DEFAULT_SAMPLE_COMPRESSION,
     UNCOMPRESSED,
@@ -42,7 +41,6 @@
 
 # these configs are added to every `htype`
 COMMON_CONFIGS = {
-    "chunk_size": DEFAULT_MAX_CHUNK_SIZE,
     "chunk_compression": DEFAULT_CHUNK_COMPRESSION,
     "sample_compression": DEFAULT_SAMPLE_COMPRESSION,
 }

diff --git a/hub/util/remove_cache.py b/hub/util/remove_cache.py
@@ -1,6 +1,6 @@
+from hub.core.storage.provider import StorageProvider
 from hub.core.storage.lru_cache import LRUCache
 from hub.core.storage import MemoryProvider
-from hub.core.typing import StorageProvider
 
 
 def remove_memory_cache(storage: StorageProvider):