connect test + docs update

puyuanOT · Jan 22, 2023 · 6c4352d · 6c4352d
1 parent b43c03c
commit 6c4352d
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 4 deletions.
diff --git a/deeplake/api/dataset.py b/deeplake/api/dataset.py
@@ -1087,7 +1087,7 @@ def ingest_coco(
             progressbar (bool): Enables or disables ingestion progress bar. Set to ``True`` by default.
             num_workers (int): The number of workers to use for ingestion. Set to ``0`` by default.
             token (Optional[str]): The token to use for accessing the dataset and/or connecting it to Deep Lake.
-            connect_kwargs (Optional[Dict]): If specified, the dataset will be connected to Deep Lake, and connect_kwargs will be passed to :func:`ds.connect`.
+            connect_kwargs (Optional[Dict]): If specified, the dataset will be connected to Deep Lake, and connect_kwargs will be passed to :meth:`Dataset.connect`.
             **dataset_kwargs: Any arguments passed here will be forwarded to the dataset creator function. See :func:`deeplake.empty`.
 
         Returns:
@@ -1129,7 +1129,8 @@ def ingest_coco(
             dest, creds=dest_creds, verbose=False, token=token, **dataset_kwargs
         )
         if connect_kwargs is not None:
-            ds.connect(**connect_kwargs, token=token)
+            connect_kwargs["token"] = token or connect_kwargs.get(token)
+            ds.connect(**connect_kwargs)
 
         structure.create_missing(ds)
 
@@ -1203,7 +1204,7 @@ def ingest_yolo(
             progressbar (bool): Enables or disables ingestion progress bar. Set to ``True`` by default.
             num_workers (int): The number of workers to use for ingestion. Set to ``0`` by default.
             token (Optional[str]): The token to use for accessing the dataset and/or connecting it to Deep Lake.
-            connect_kwargs (Optional[Dict]): If specified, the dataset will be connected to Deep Lake, and connect_kwargs will be passed to :func:`ds.connect`.
+            connect_kwargs (Optional[Dict]): If specified, the dataset will be connected to Deep Lake, and connect_kwargs will be passed to :meth:`Dataset.connect`.
             **dataset_kwargs: Any arguments passed here will be forwarded to the dataset creator function. See :func:`deeplake.empty`.
 
         Returns:
@@ -1254,7 +1255,8 @@ def ingest_yolo(
             dest, creds=dest_creds, verbose=False, token=token, **dataset_kwargs
         )
         if connect_kwargs is not None:
-            ds.connect(**connect_kwargs, token=token)
+            connect_kwargs["token"] = token or connect_kwargs.get("token")
+            ds.connect(**connect_kwargs)
 
         structure.create_missing(ds)
 

diff --git a/deeplake/auto/tests/test_yolo_template.py b/deeplake/auto/tests/test_yolo_template.py
@@ -141,3 +141,33 @@ def test_minimal_yolo_ingestion_with_linked_images(local_path, yolo_ingestion_da
     assert "labels" in ds.tensors
     assert len(ds.labels.info["class_names"]) > 0
     assert ds.linked_images.htype == "link[image]"
+
+
+def test_minimal_yolo_with_connect(
+    s3_path,
+    yolo_ingestion_data,
+    hub_cloud_path,
+    hub_cloud_dev_token,
+    hub_cloud_dev_managed_creds_key,
+):
+    params = {
+        "data_directory": yolo_ingestion_data["data_directory"],
+        "class_names_file": yolo_ingestion_data["class_names_file"],
+    }
+
+    ds = deeplake.ingest_yolo(
+        **params,
+        dest=s3_path,
+        connect_kwargs={
+            "dest_path": hub_cloud_path,
+            "creds_key": hub_cloud_dev_managed_creds_key,
+            "token": hub_cloud_dev_token,
+        },
+    )
+
+    assert ds.path == hub_cloud_path
+    assert "images" in ds.tensors
+    assert "boxes" in ds.tensors
+    assert "labels" in ds.tensors
+    assert len(ds.labels.info["class_names"]) > 0
+    assert ds.boxes.htype == "bbox"
diff --git a/docs/source/Datasets.rst b/docs/source/Datasets.rst
@@ -15,6 +15,7 @@ Creating Datasets
     deeplake.like
     deeplake.ingest
     deeplake.ingest_coco
+    deeplake.ingest_yolo
     deeplake.ingest_kaggle
     deeplake.ingest_dataframe
     deeplake.ingest_huggingface
@@ -60,6 +61,7 @@ Dataset Operations
     Dataset.copy
     Dataset.delete
     Dataset.rename
+    Dataset.connect
     Dataset.visualize
     Dataset.pop
     Dataset.rechunk

diff --git a/docs/source/deeplake.api.dataset.rst b/docs/source/deeplake.api.dataset.rst
@@ -47,6 +47,10 @@ deeplake.api.dataset
 
         See :func:`deeplake.ingest_coco`.
 
+    .. staticmethod:: def ingest_yolo(data_directory: Union[str, pathlib.Path], dest: Union[str, pathlib.Path], class_names_file: Optional[Union[str, pathlib.Path]] = None, annotations_directory: Optional[Union[str, pathlib.Path]] = None, allow_no_annotation: bool = False, image_params: Optional[Dict] = None, label_params: Optional[Dict] = None, coordinates_params: Optional[Dict] = None, src_creds: Optional[Dict] = None, dest_creds: Optional[Dict] = None, image_creds_key: Optional[str] = None, inspect_limit: int = 1000, progressbar: bool = True, num_workers: int = 0, token: Optional[str] = None, connect_kwargs: Optional[Dict] = None, **dataset_kwargs) -> Dataset:
+
+        See :func:`deeplake.ingest_yolo`
+
     .. staticmethod:: ingest_kaggle(tag: str, src: Union[str, pathlib.Path], dest: Union[str, pathlib.Path], exist_ok: bool = False, images_compression: str = "auto", dest_creds: dict = None, kaggle_credentials: dict = None, progressbar: bool = True, summary: bool = True, **dataset_kwargs) -> Dataset
 
         See :func:`deeplake.ingest_kaggle`.

diff --git a/docs/source/deeplake.rst b/docs/source/deeplake.rst
@@ -9,6 +9,7 @@ deeplake
 .. autofunction:: like
 .. autofunction:: ingest
 .. autofunction:: ingest_coco
+.. autofunction:: ingest_yolo
 .. autofunction:: ingest_kaggle
 .. autofunction:: ingest_dataframe
 .. autofunction:: ingest_huggingface