cvat-ai · zhiltsov-max · Dec 29, 2024 · Dec 29, 2024 · Dec 29, 2024 · Dec 30, 2024
@@ -38,7 +38,7 @@ export default class QualitySettings {
     #objectVisibilityThreshold: number;
     #panopticComparison: boolean;
     #compareAttributes: boolean;
-    #matchEmptyFrames: boolean;
+    #emptyIsAnnotated: boolean;
     #descriptions: Record<string, string>;
 
     constructor(initialData: SerializedQualitySettingsData) {
@@ -60,7 +60,7 @@ export default class QualitySettings {
         this.#objectVisibilityThreshold = initialData.object_visibility_threshold;
         this.#panopticComparison = initialData.panoptic_comparison;
         this.#compareAttributes = initialData.compare_attributes;
-        this.#matchEmptyFrames = initialData.match_empty_frames;
+        this.#emptyIsAnnotated = initialData.empty_is_annotated;
         this.#descriptions = initialData.descriptions;
     }
 
@@ -200,12 +200,12 @@ export default class QualitySettings {
         this.#maxValidationsPerJob = newVal;
     }
 
-    get matchEmptyFrames(): boolean {
-        return this.#matchEmptyFrames;
+    get emptyIsAnnotated(): boolean {
+        return this.#emptyIsAnnotated;
     }
 
-    set matchEmptyFrames(newVal: boolean) {
-        this.#matchEmptyFrames = newVal;
+    set emptyIsAnnotated(newVal: boolean) {
+        this.#emptyIsAnnotated = newVal;
     }
 
     get descriptions(): Record<string, string> {
@@ -236,7 +236,7 @@ export default class QualitySettings {
             target_metric: this.#targetMetric,
             target_metric_threshold: this.#targetMetricThreshold,
             max_validations_per_job: this.#maxValidationsPerJob,
-            match_empty_frames: this.#matchEmptyFrames,
+            empty_is_annotated: this.#emptyIsAnnotated,
         };
 
         return result;

@@ -258,7 +258,7 @@ export interface SerializedQualitySettingsData {
     object_visibility_threshold?: number;
     panoptic_comparison?: boolean;
     compare_attributes?: boolean;
-    match_empty_frames?: boolean;
+    empty_is_annotated?: boolean;
     descriptions?: Record<string, string>;
 }
 

@@ -223,7 +223,7 @@ function QualityControlPage(): JSX.Element {
                 settings.lowOverlapThreshold = values.lowOverlapThreshold / 100;
                 settings.iouThreshold = values.iouThreshold / 100;
                 settings.compareAttributes = values.compareAttributes;
-                settings.matchEmptyFrames = values.matchEmptyFrames;
+                settings.emptyIsAnnotated = values.emptyIsAnnotated;
 
                 settings.oksSigma = values.oksSigma / 100;
                 settings.pointSizeBase = values.pointSizeBase;

@@ -34,7 +34,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
         lowOverlapThreshold: settings.lowOverlapThreshold * 100,
         iouThreshold: settings.iouThreshold * 100,
         compareAttributes: settings.compareAttributes,
-        matchEmptyFrames: settings.matchEmptyFrames,
+        emptyIsAnnotated: settings.emptyIsAnnotated,
 
         oksSigma: settings.oksSigma * 100,
         pointSizeBase: settings.pointSizeBase,
@@ -81,7 +81,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
             {makeTooltipFragment('Target metric', targetMetricDescription)}
             {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)}
             {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)}
-            {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)}
+            {makeTooltipFragment('Empty frames are annotated', settings.descriptions.emptyIsAnnotated)}
         </>,
     );
 
@@ -198,12 +198,12 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
                 </Col>
                 <Col span={12}>
                     <Form.Item
-                        name='matchEmptyFrames'
+                        name='emptyIsAnnotated'
                         valuePropName='checked'
                         rules={[{ required: true }]}
                     >
                         <Checkbox>
-                            <Text className='cvat-text-color'>Match empty frames</Text>
+                            <Text className='cvat-text-color'>Empty frames are annotated</Text>
                         </Checkbox>
                     </Form.Item>
                 </Col>

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-12-29 19:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("quality_control", "0005_qualitysettings_match_empty"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="qualitysettings",
+            old_name="match_empty_frames",
+            new_name="empty_is_annotated",
+        ),
+    ]
@@ -235,7 +235,7 @@ class QualitySettings(models.Model):
 
     compare_attributes = models.BooleanField()
 
-    match_empty_frames = models.BooleanField(default=False)
+    empty_is_annotated = models.BooleanField(default=False)
 
     target_metric = models.CharField(
         max_length=32,

@@ -215,10 +215,11 @@ class ComparisonParameters(_Serializable):
     panoptic_comparison: bool = True
     "Use only the visible part of the masks and polygons in comparisons"
 
-    match_empty_frames: bool = False
+    empty_is_annotated: bool = False
     """
-    Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy,
-    will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead.
+    Consider unannotated (empty) frames virtually annotated as "nothing".
+    If disabled, quality metrics, such as accuracy, will be 0 if both GT and DS frames
+    have no annotations. When enabled, they will be 1 instead.
     This will also add virtual annotations to empty frames in the comparison results.
     """
 
@@ -1977,15 +1978,22 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
             gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX
             confusion_matrix[ds_label_idx, gt_label_idx] += 1
 
-        if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations:
+        if self.settings.empty_is_annotated and (
+            not gt_item.annotations or not ds_item.annotations
+        ):
             # Add virtual annotations for empty frames
-            valid_labels_count = 1
-            total_labels_count = 1
+            if not gt_item.annotations and not ds_item.annotations:
+                valid_labels_count = 1
+                total_labels_count = 1
+
+                valid_shapes_count = 1
+                total_shapes_count = 1
 
-            valid_shapes_count = 1
-            total_shapes_count = 1
-            ds_shapes_count = 1
-            gt_shapes_count = 1
+            if not ds_item.annotations:
+                ds_shapes_count = 1
+
+            if not gt_item.annotations:
+                gt_shapes_count = 1
 
         self._frame_results[frame_id] = ComparisonReportFrameSummary(
             annotations=self._generate_frame_annotations_summary(
@@ -2078,12 +2086,17 @@ def _generate_frame_annotations_summary(
     ) -> ComparisonReportAnnotationsSummary:
         summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels)
 
-        if self.settings.match_empty_frames and summary.total_count == 0:
+        if self.settings.empty_is_annotated and (not summary.ds_count or not summary.gt_count):
             # Add virtual annotations for empty frames
-            summary.valid_count = 1
-            summary.total_count = 1
-            summary.ds_count = 1
-            summary.gt_count = 1
+            if not summary.total_count:
+                summary.valid_count = 1
+                summary.total_count = 1
+
+            if not summary.ds_count:
+                summary.ds_count = 1
+
+            if not summary.gt_count:
+                summary.gt_count = 1
 
         return summary
 
@@ -2108,14 +2121,26 @@ def _generate_dataset_annotations_summary(
             ),
         )
         mean_ious = []
-        empty_frame_count = 0
+        empty_gt_frames = set()
+        empty_ds_frames = set()
         confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix()
 
-        for frame_result in frame_summaries.values():
+        for frame_id, frame_result in frame_summaries.items():
             confusion_matrix += frame_result.annotations.confusion_matrix.rows
 
-            if not np.any(frame_result.annotations.confusion_matrix.rows):
-                empty_frame_count += 1
+            if self.settings.empty_is_annotated and not np.any(
+                frame_result.annotations.confusion_matrix.rows[
+                    np.triu_indices_from(frame_result.annotations.confusion_matrix.rows)
+                ]
+            ):
+                empty_ds_frames.add(frame_id)
+
+            if self.settings.empty_is_annotated and not np.any(
+                frame_result.annotations.confusion_matrix.rows[
+                    np.tril_indices_from(frame_result.annotations.confusion_matrix.rows)
+                ]
+            ):
+                empty_gt_frames.add(frame_id)
 
             if annotation_components is None:
                 annotation_components = deepcopy(frame_result.annotation_components)
@@ -2128,13 +2153,13 @@ def _generate_dataset_annotations_summary(
             confusion_matrix, confusion_matrix_labels
         )
 
-        if self.settings.match_empty_frames and empty_frame_count:
+        if self.settings.empty_is_annotated:
             # Add virtual annotations for empty frames,
             # they are not included in the confusion matrix
-            annotation_summary.valid_count += empty_frame_count
-            annotation_summary.total_count += empty_frame_count
-            annotation_summary.ds_count += empty_frame_count
-            annotation_summary.gt_count += empty_frame_count
+            annotation_summary.valid_count += len(empty_ds_frames & empty_gt_frames)
+            annotation_summary.total_count += len(empty_ds_frames | empty_gt_frames)
+            annotation_summary.ds_count += len(empty_ds_frames)
+            annotation_summary.gt_count += len(empty_gt_frames)
 
         # Cannot be computed in accumulate()
         annotation_components.shape.mean_iou = np.mean(mean_ious)

@@ -92,15 +92,15 @@ class Meta:
             "object_visibility_threshold",
             "panoptic_comparison",
             "compare_attributes",
-            "match_empty_frames",
+            "empty_is_annotated",
         )
         read_only_fields = (
             "id",
             "task_id",
         )
 
         extra_kwargs = {k: {"required": False} for k in fields}
-        extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False)
+        extra_kwargs.setdefault("empty_is_annotated", {}).setdefault("default", False)
 
         for field_name, help_text in {
             "target_metric": "The primary metric used for quality estimation",
@@ -166,8 +166,8 @@ class Meta:
                 Use only the visible part of the masks and polygons in comparisons
             """,
             "compare_attributes": "Enables or disables annotation attribute comparison",
-            "match_empty_frames": """
-                Count empty frames as matching. This affects target metrics like accuracy in cases
+            "empty_is_annotated": """
+                Count empty frames as annotated. This affects target metrics like accuracy in cases
                 there are no annotations. If disabled, frames without annotations
                 are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
                 This will also add virtual annotations to empty frames in the comparison results.

@@ -9775,11 +9775,11 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
-        match_empty_frames:
+        empty_is_annotated:
           type: boolean
           default: false
           description: |
-            Count empty frames as matching. This affects target metrics like accuracy in cases
+            Count empty frames as annotated. This affects target metrics like accuracy in cases
             there are no annotations. If disabled, frames without annotations
             are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
             This will also add virtual annotations to empty frames in the comparison results.
@@ -10282,11 +10282,11 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
-        match_empty_frames:
+        empty_is_annotated:
           type: boolean
           default: false
           description: |
-            Count empty frames as matching. This affects target metrics like accuracy in cases
+            Count empty frames as annotated. This affects target metrics like accuracy in cases
             there are no annotations. If disabled, frames without annotations
             are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
             This will also add virtual annotations to empty frames in the comparison results.

@@ -1213,7 +1213,7 @@ def test_modified_task_produces_different_metrics(
             "compare_line_orientation",
             "panoptic_comparison",
             "point_size_base",
-            "match_empty_frames",
+            "empty_is_annotated",
         ],
     )
     def test_settings_affect_metrics(
@@ -1246,8 +1246,11 @@ def test_settings_affect_metrics(
             )
 
         new_report = self.create_quality_report(admin_user, task_id)
-        if parameter == "match_empty_frames":
+        if parameter == "empty_is_annotated":
             assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"]
+            assert new_report["summary"]["total_count`"] != old_report["summary"]["total_count`"]
+            assert new_report["summary"]["ds_count"] != old_report["summary"]["ds_count"]
+            assert new_report["summary"]["gt_count"] != old_report["summary"]["gt_count"]
         else:
             assert (
                 new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"]