Skip to content

Commit

Permalink
black
Browse files Browse the repository at this point in the history
  • Loading branch information
FayazRahman committed May 12, 2023
1 parent de8b7f0 commit ebe75b4
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 5 deletions.
9 changes: 8 additions & 1 deletion deeplake/core/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -1539,7 +1539,14 @@ def _commit(
self.storage.autoflush = False
try:
self._unlock()
commit(self, message, hash, flush_version_control_info, is_checkpoint=is_checkpoint, total_samples_processed=total_samples_processed)
commit(
self,
message,
hash,
flush_version_control_info,
is_checkpoint=is_checkpoint,
total_samples_processed=total_samples_processed,
)
if not flush_version_control_info:
self.__dict__["_vc_info_updated"] = True
self._lock()
Expand Down
8 changes: 7 additions & 1 deletion deeplake/core/transform/test_transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -1332,6 +1332,7 @@ def double(data_in, ds):
)
assert ds.abc.numpy(aslist=True) == data_in


def test_transform_checkpoint_store_data(local_ds_generator):
@deeplake.compute
def upload(i, ds):
Expand All @@ -1343,7 +1344,12 @@ def upload(i, ds):
with pytest.raises(TransformError):
with local_ds_generator() as ds:
ds.create_tensor("abc")
upload().eval(samples, ds, num_workers=TRANSFORM_TEST_NUM_WORKERS, checkpoint_interval=20)
upload().eval(
samples,
ds,
num_workers=TRANSFORM_TEST_NUM_WORKERS,
checkpoint_interval=20,
)

ds = local_ds_generator()

Expand Down
6 changes: 5 additions & 1 deletion deeplake/core/transform/transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,11 @@ def my_fn(sample_in: Any, samples_out, my_arg0, my_arg1=0):
for data_in in datas_in:
if checkpointing_enabled and progress > 0:
target_ds._commit(
f"Auto-commit during deeplake.compute of {desc} after {progress}% progress", None, False, is_checkpoint=True, total_samples_processed=samples_processed
f"Auto-commit during deeplake.compute of {desc} after {progress}% progress",
None,
False,
is_checkpoint=True,
total_samples_processed=samples_processed,
)
progress = round(
(samples_processed + len(data_in)) / total_samples * 100, 2
Expand Down
4 changes: 3 additions & 1 deletion deeplake/core/version_control/commit_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ def is_merge_node(self):
def __repr__(self) -> str:
return (
f"Commit : {self.commit_id} ({self.branch}) \nAuthor : {self.commit_user_name}\nTime : {str(self.commit_time)[:-7]}\nMessage: {self.commit_message}"
+ f"\nTotal samples processed in transform: {self.total_samples_processed}" if self.is_checkpoint else ""
+ f"\nTotal samples processed in transform: {self.total_samples_processed}"
if self.is_checkpoint
else ""
)

@property
Expand Down
8 changes: 7 additions & 1 deletion deeplake/util/version_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,13 @@ def _merge_node(commit_id):
node2 = map2[commit_id]
merged_node = CommitNode(node1.branch, node2.commit_id)

for attr in ("commit_message", "commit_user_name", "commit_time", "is_checkpoint", "total_samples_processed"):
for attr in (
"commit_message",
"commit_user_name",
"commit_time",
"is_checkpoint",
"total_samples_processed",
):
setattr(merged_node, attr, getattr(node1, attr) or getattr(node2, attr))
for child in set(
[node.commit_id for node in node1.children]
Expand Down

0 comments on commit ebe75b4

Please sign in to comment.