Revert "Added support for project with 3d tasks"

This reverts commit 0687808.
cvat-ai · nmanovic · Aug 6, 2021 · Jun 22, 2021 · Jun 23, 2021 · Jun 24, 2021
commit 0a06baffc4e9d24e0ecf37f1d22675ffd715c447
@@ -513,7 +513,7 @@ class ProjectData(InstanceLabelData):
     Track = NamedTuple('Track', [('label', str), ('group', int), ('source', str), ('shapes', List[TrackedShape]), ('task_id', int)])
     Tag = NamedTuple('Tag', [('frame', int), ('label', str), ('attributes', List[InstanceLabelData.Attribute]), ('source', str), ('group', int), ('task_id', int)])
     Tag.__new__.__defaults__ = (0, )
-    Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('id', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])])
+    Frame = NamedTuple('Frame', [('task_id', int), ('subset', str), ('idx', int), ('frame', int), ('name', str), ('width', int), ('height', int), ('labeled_shapes', List[Union[LabeledShape, TrackedShape]]), ('tags', List[Tag])])
 
     def __init__(self, annotation_irs: Mapping[str, AnnotationIR], db_project: Project, host: str, create_callback: Callable = None):
         self._annotation_irs = annotation_irs
@@ -579,7 +579,6 @@ def _init_frame_info(self):
                 } for frame in range(task.data.size)})
             else:
                 self._frame_info.update({(task.id, self.rel_frame_id(task.id, db_image.frame)): {
-                    "id": db_image.id,
                     "path": mangle_image_name(db_image.path, task.subset, original_names),
                     "width": db_image.width,
                     "height": db_image.height,
@@ -683,7 +682,6 @@ def get_frame(task_id: int, idx: int) -> ProjectData.Frame:
                     task_id=task_id,
                     subset=frame_info["subset"],
                     idx=idx,
-                    id=frame_info.get('id',0),
                     frame=abs_frame,
                     name=frame_info["path"],
                     height=frame_info["height"],
@@ -797,25 +795,20 @@ def __init__(self):
         self._categories: Dict[datumaro.AnnotationType, datumaro.LabelCategories] = {}
 
     @staticmethod
-    def _load_categories(meta: dict, dimension):
+    def _load_categories(labels: list):
         categories: Dict[datumaro.AnnotationType, datumaro.Categories] = {}
 
         label_categories = datumaro.LabelCategories(attributes=['occluded'])
 
-        user_info = {}
-        if dimension == DimensionType.DIM_3D:
-            user_info = {"name": meta['owner']['username'],
-                         "createdAt": meta['created'],
-                         "updatedAt": meta['updated']}
-        for _, label in meta['labels']:
+        for _, label in labels:
             label_categories.add(label['name'])
             for _, attr in label['attributes']:
                 label_categories.attributes.add(attr['name'])
 
-
         categories[datumaro.AnnotationType.label] = label_categories
 
-        return categories, user_info
+        return categories
+
 
     def _read_cvat_anno(self, cvat_frame_anno: Union[ProjectData.Frame, TaskData.Frame], labels: list):
         categories = self.categories()
@@ -830,9 +823,9 @@ def map_label(name): return label_cat.find(name)[0]
 
 
 class CvatTaskDataExtractor(CVATDataExtractor):
-    def __init__(self, task_data, include_images = False, format_type = None, dimension = DimensionType.DIM_2D):
+    def __init__(self, task_data, include_images=False, format_type=None, dimension=DimensionType.DIM_2D):
         super().__init__()
-        self._categories, self._user = self._load_categories(task_data.meta['task'], dimension=dimension)
+        self._categories, self._user = self._load_categories(task_data, dimension=dimension)
         self._dimension = dimension
         self._format_type = format_type
         dm_items = []
@@ -898,9 +891,11 @@ def _make_image(i, **kwargs):
                     attributes["createdAt"] = self._user["createdAt"]
                     attributes["updatedAt"] = self._user["updatedAt"]
                     attributes["labels"] = []
-                    for (idx, (_, label)) in enumerate(task_data.meta['task']['labels']):
-                        attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]})
+                    index = 0
+                    for _, label in task_data.meta['task']['labels']:
+                        attributes["labels"].append({"label_id": index, "name": label["name"], "color": label["color"]})
                         attributes["track_id"] = -1
+                        index += 1
 
                 dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0],
                                                annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1],
@@ -910,6 +905,27 @@ def _make_image(i, **kwargs):
 
         self._items = dm_items
 
+    @staticmethod
+    def _load_categories(cvat_anno, dimension): # pylint: disable=arguments-differ
+        categories = {}
+
+        label_categories = datumaro.LabelCategories(attributes=['occluded'])
+
+        user_info = {}
+        if dimension == DimensionType.DIM_3D:
+            user_info = {"name": cvat_anno.meta['task']['owner']['username'],
+                         "createdAt": cvat_anno.meta['task']['created'],
+                         "updatedAt": cvat_anno.meta['task']['updated']}
+        for _, label in cvat_anno.meta['task']['labels']:
+            label_categories.add(label['name'])
+            for _, attr in label['attributes']:
+                label_categories.attributes.add(attr['name'])
+
+
+        categories[datumaro.AnnotationType.label] = label_categories
+
+        return categories, user_info
+
     def _read_cvat_anno(self, cvat_frame_anno: TaskData.Frame, labels: list):
         categories = self.categories()
         label_cat = categories[datumaro.AnnotationType.label]
@@ -922,11 +938,9 @@ def map_label(name): return label_cat.find(name)[0]
         return convert_cvat_anno_to_dm(cvat_frame_anno, label_attrs, map_label, self._format_type, self._dimension)
 
 class CVATProjectDataExtractor(CVATDataExtractor):
-    def __init__(self, project_data: ProjectData, include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D):
+    def __init__(self, project_data: ProjectData, include_images: bool = False):
         super().__init__()
-        self._categories, self._user = self._load_categories(project_data.meta['project'], dimension)
-        self._dimension = dimension
-        self._format_type = format_type
+        self._categories = self._load_categories(project_data.meta['project']['labels'])
 
         dm_items: List[datumaro.DatasetItem] = []
 
@@ -936,27 +950,12 @@ def __init__(self, project_data: ProjectData, include_images: bool = False, form
         for task in project_data.tasks:
             is_video = task.mode == 'interpolation'
             ext_per_task[task.id] = FrameProvider.VIDEO_FRAME_EXT if is_video else ''
-            if self._dimension == DimensionType.DIM_3D:
-                def image_maker_factory(task):
-                    def _make_image(i, **kwargs):
-                        loader = osp.join(
-                            task.data.get_upload_dirname(), kwargs['path'],
-                        )
-                        related_images = []
-                        image = Img.objects.get(id=i)
-                        for i in image.related_files.all():
-                            path = osp.realpath(str(i.path))
-                            if osp.isfile(path):
-                                related_images.append(path)
-                        return loader, related_images
-                    return _make_image
-                image_maker_per_task[task.id] = image_maker_factory(task)
-            elif include_images:
+            if include_images:
+                frame_provider = FrameProvider(task.data)
                 if is_video:
                     # optimization for videos: use numpy arrays instead of bytes
                     # some formats or transforms can require image data
-                    def image_maker_factory(task):
-                        frame_provider = FrameProvider(task.data)
+                    def image_maker_factory(frame_provider):
                         def _make_image(i, **kwargs):
                             loader = lambda _: frame_provider.get_frame(i,
                                 quality=frame_provider.Quality.ORIGINAL,
@@ -965,58 +964,40 @@ def _make_image(i, **kwargs):
                         return _make_image
                 else:
                     # for images use encoded data to avoid recoding
-                    def image_maker_factory(task):
-                        frame_provider = FrameProvider(task.data)
+                    def image_maker_factory(frame_provider):
                         def _make_image(i, **kwargs):
                             loader = lambda _: frame_provider.get_frame(i,
                                 quality=frame_provider.Quality.ORIGINAL,
                                 out_type=frame_provider.Type.BUFFER)[0].getvalue()
                             return ByteImage(data=loader, **kwargs)
                         return _make_image
-                image_maker_per_task[task.id] = image_maker_factory(task)
+                image_maker_per_task[task.id] = image_maker_factory(frame_provider)
 
         for frame_data in project_data.group_by_frame(include_empty=True):
             image_args = {
                 'path': frame_data.name + ext_per_task[frame_data.task_id],
                 'size': (frame_data.height, frame_data.width),
             }
-            if self._dimension == DimensionType.DIM_3D:
-                dm_image = image_maker_per_task[frame_data.task_id](frame_data.id, **image_args)
-            elif include_images:
+            if include_images:
                 dm_image = image_maker_per_task[frame_data.task_id](frame_data.idx, **image_args)
             else:
                 dm_image = Image(**image_args)
             dm_anno = self._read_cvat_anno(frame_data, project_data.meta['project']['labels'])
-            if self._dimension == DimensionType.DIM_2D:
-                dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0],
-                    annotations=dm_anno, image=dm_image,
-                    subset=frame_data.subset,
-                    attributes={'frame': frame_data.frame}
-                )
-            else:
-                attributes = {'frame': frame_data.frame}
-                if format_type == "sly_pointcloud":
-                    attributes["name"] = self._user["name"]
-                    attributes["createdAt"] = self._user["createdAt"]
-                    attributes["updatedAt"] = self._user["updatedAt"]
-                    attributes["labels"] = []
-                    for (idx, (_, label)) in enumerate(project_data.meta['project']['labels']):
-                        attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]})
-                        attributes["track_id"] = -1
-
-                dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0],
-                                               annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1],
-                                               attributes=attributes)
+            dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0],
+                annotations=dm_anno, image=dm_image,
+                subset=frame_data.subset,
+                attributes={'frame': frame_data.frame}
+            )
             dm_items.append(dm_item)
 
         self._items = dm_items
 
 
-def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool = False, format_type: str = None, dimension: DimensionType = DimensionType.DIM_2D):
+def GetCVATDataExtractor(instance_data: Union[ProjectData, TaskData], include_images: bool=False):
     if isinstance(instance_data, ProjectData):
-        return CVATProjectDataExtractor(instance_data, include_images, format_type, dimension)
+        return CVATProjectDataExtractor(instance_data, include_images)
     else:
-        return CvatTaskDataExtractor(instance_data, include_images, format_type, dimension)
+        return CvatTaskDataExtractor(instance_data, include_images)
 
 class CvatImportError(Exception):
     pass

@@ -7,7 +7,7 @@
 
 from datumaro.components.dataset import Dataset
 
-from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor,
+from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, TaskData,
                                                 import_dm_annotations)
 from cvat.apps.dataset_manager.util import make_zip_archive
 from cvat.apps.engine.models import DimensionType
@@ -18,7 +18,10 @@
 @exporter(name='Sly Point Cloud Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D)
 def _export_images(dst_file, task_data, save_images=False):
 
-    dataset = Dataset.from_extractors(GetCVATDataExtractor(
+    if not isinstance(task_data, TaskData):
+        raise Exception("Export to \"Sly Point Cloud\" format is working only with tasks temporarily")
+
+    dataset = Dataset.from_extractors(CvatTaskDataExtractor(
         task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env)
 
     with TemporaryDirectory() as temp_dir:

@@ -7,7 +7,7 @@
 
 from datumaro.components.dataset import Dataset
 
-from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \
+from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor, TaskData, \
     import_dm_annotations
 from .registry import dm_env
 
@@ -20,7 +20,10 @@
 @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D)
 def _export_images(dst_file, task_data, save_images=False):
 
-    dataset = Dataset.from_extractors(GetCVATDataExtractor(
+    if not isinstance(task_data, TaskData):
+        raise Exception("Export to \"Kitti raw\" format is working only with tasks temporarily")
+
+    dataset = Dataset.from_extractors(CvatTaskDataExtractor(
         task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env)
 
     with TemporaryDirectory() as temp_dir: