fix for rewrite error

m-star18 · Nov 5, 2020 · eaefc84 · eaefc84
1 parent 93695db
commit eaefc84
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 8 deletions.
diff --git a/examples/basic_v1.py b/examples/basic_v1.py
@@ -0,0 +1,49 @@
+from hub import Dataset, features
+import numpy as np
+from pprint import pprint
+import time
+
+if False:
+    t1 = time.time()
+    ds = Dataset(
+        "davit/basic6",
+        shape=(4,),
+        schema={
+            "image": features.Tensor((512, 512), dtype="float"),
+            "label": features.Tensor((512, 512), dtype="float"),
+        },
+    )
+    t2 = time.time()
+    ds["image"][:] = np.zeros((4, 512, 512))
+    ds["label"][:] = np.zeros((4, 512, 512))
+    t3 = time.time()
+    ds.commit()
+
+t4 = time.time()
+ds = Dataset(
+    "./data/test/davit/basic10",
+    shape=(4,),
+    mode="a",
+    schema={
+        "image": features.Tensor((512, 512), dtype="float"),
+        "label": features.Tensor((512, 512), dtype="float"),
+    },
+)
+
+ds["image"][0] = 1
+
+ds = Dataset("./data/test/davit/basic10", mode="a")
+t5 = time.time()
+print(ds["image"][0].compute())
+t6 = time.time()
+exit()
+
+pprint(
+    {
+        "dataset creating": t2 - t1,
+        "assigning": t3 - t2,
+        "commting": t4 - t3,
+        "loading dataset": t5 - t4,
+        "accessing": t6 - t5,
+    }
+)
diff --git a/hub/api/dataset.py b/hub/api/dataset.py
@@ -49,7 +49,7 @@ class Dataset:
     def __init__(
         self,
         url: str,
-        mode: str = "w",
+        mode: str = "a",
         safe_mode: bool = False,
         shape=None,
         schema=None,

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,4 +1,8 @@
 tensorflow>=1.14,<2
 tensorflow-datasets>=3.0,<4
 torch>=1,<2
-waymo-open-dataset-tf-1-15-0
+waymo-open-dataset-tf-1-15-0
+pytest
+pytest-cov
+flake8
+black
diff --git a/requirements.txt b/requirements.txt
@@ -10,10 +10,6 @@ outdated==0.2.0
 lz4>=3,<4
 zarr==2.5
 lmdb==1.0.0
-boto3==1.14.29
+boto3==1.16.10
 ray==1.0.0
-#dev
-pytest
-pytest-cov
-flake8
-black
+