diff --git a/examples/basic_v1.py b/examples/basic_v1.py new file mode 100644 index 0000000000..5eac5f58b0 --- /dev/null +++ b/examples/basic_v1.py @@ -0,0 +1,49 @@ +from hub import Dataset, features +import numpy as np +from pprint import pprint +import time + +if False: + t1 = time.time() + ds = Dataset( + "davit/basic6", + shape=(4,), + schema={ + "image": features.Tensor((512, 512), dtype="float"), + "label": features.Tensor((512, 512), dtype="float"), + }, + ) + t2 = time.time() + ds["image"][:] = np.zeros((4, 512, 512)) + ds["label"][:] = np.zeros((4, 512, 512)) + t3 = time.time() + ds.commit() + +t4 = time.time() +ds = Dataset( + "./data/test/davit/basic10", + shape=(4,), + mode="a", + schema={ + "image": features.Tensor((512, 512), dtype="float"), + "label": features.Tensor((512, 512), dtype="float"), + }, +) + +ds["image"][0] = 1 + +ds = Dataset("./data/test/davit/basic10", mode="a") +t5 = time.time() +print(ds["image"][0].compute()) +t6 = time.time() +exit() + +pprint( + { + "dataset creating": t2 - t1, + "assigning": t3 - t2, + "commting": t4 - t3, + "loading dataset": t5 - t4, + "accessing": t6 - t5, + } +) diff --git a/hub/api/dataset.py b/hub/api/dataset.py index 7833e1cecf..35250ebf04 100644 --- a/hub/api/dataset.py +++ b/hub/api/dataset.py @@ -49,7 +49,7 @@ class Dataset: def __init__( self, url: str, - mode: str = "w", + mode: str = "a", safe_mode: bool = False, shape=None, schema=None, diff --git a/requirements-dev.txt b/requirements-dev.txt index 601e69531a..8078e9340b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,8 @@ tensorflow>=1.14,<2 tensorflow-datasets>=3.0,<4 torch>=1,<2 -waymo-open-dataset-tf-1-15-0 \ No newline at end of file +waymo-open-dataset-tf-1-15-0 +pytest +pytest-cov +flake8 +black \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index df597ccbdc..ea47af7a1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,6 @@ outdated==0.2.0 lz4>=3,<4 zarr==2.5 lmdb==1.0.0 -boto3==1.14.29 +boto3==1.16.10 ray==1.0.0 -#dev -pytest -pytest-cov -flake8 -black +