commit before transform start

activeloopai · May 12, 2023 · 254658b · 254658b
1 parent 19cbd3f
commit 254658b
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/deeplake/core/transform/test_transform.py b/deeplake/core/transform/test_transform.py
@@ -1334,13 +1334,14 @@ def double(data_in, ds):
     assert ds.abc.numpy(aslist=True) == data_in
 
 
-def test_transform_checkpoint_store_data(local_ds_generator):
+@pytest.mark.parametrize("bad_sample_index", [10, 50])
+def test_transform_checkpoint_store_data(local_ds_generator, bad_sample_index):
     @deeplake.compute
     def upload(i, ds):
         ds.abc.append(i)
 
     samples = list(range(100))
-    samples.insert(50, "bad sample")
+    samples.insert(bad_sample_index, "bad sample")
 
     with pytest.raises(TransformError):
         with local_ds_generator() as ds:
@@ -1354,10 +1355,11 @@ def upload(i, ds):
 
     ds = local_ds_generator()
 
-    assert len(ds.abc) == 40
+    nsamples = 0 if bad_sample_index == 10 else 40
+    assert len(ds.abc) == nsamples
     last_checkpoint = ds.version_state["commit_node"].parent
     assert last_checkpoint.is_checkpoint == True
-    assert last_checkpoint.total_samples_processed == 40
+    assert last_checkpoint.total_samples_processed == nsamples
 
 
 def create_test_ds(path):

diff --git a/deeplake/core/transform/transform.py b/deeplake/core/transform/transform.py
@@ -255,7 +255,7 @@ def my_fn(sample_in: Any, samples_out, my_arg0, my_arg1=0):
         completed = False
         progress = 0.0
         for data_in in datas_in:
-            if checkpointing_enabled and progress > 0:
+            if checkpointing_enabled:
                 target_ds._commit(
                     f"Auto-commit during deeplake.compute of {desc} after {progress}% progress",
                     None,