Validation management core (#8471)

cvat-ai · Oct 7, 2024 · 0572fa1 · 0572fa1
1 parent 0a45bc7
commit 0572fa1
Show file tree

Hide file tree

Showing 28 changed files with 1,857 additions and 123 deletions.
diff --git a/changelog.d/20240930_171153_mzhiltso_validation_management_core.md b/changelog.d/20240930_171153_mzhiltso_validation_management_core.md
@@ -0,0 +1,14 @@
+### Added
+
+- \[Server API\] An option to change real frames for honeypot frames in tasks with honeypots
+  (<https://github.com/cvat-ai/cvat/pull/8471>)
+- \[Server API\] New endpoints for validation configuration management in tasks and jobs
+  `/api/tasks/{id}/validation_layout`, `/api/jobs/{id}/validation_layout`
+  (<https://github.com/cvat-ai/cvat/pull/8471>)
+
+### Changed
+- \[Server API\] Now chunks in tasks can be changed.
+  There are new API elements to check chunk relevancy, if they are cached:
+  `/api/tasks/{id}/data/meta` got a new field `chunks_updated_date`,
+  `/api/tasks/{id}/data/?type=chunk` got 2 new headers: `X-Updated-Date`, `X-Checksum`
+  (<https://github.com/cvat-ai/cvat/pull/8471>)
diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts
@@ -26,6 +26,7 @@ import QualityReport from './quality-report';
 import QualityConflict from './quality-conflict';
 import QualitySettings from './quality-settings';
 import AnalyticsReport from './analytics-report';
+import ValidationLayout from './validation-layout';
 import { Request } from './request';
 
 import * as enums from './enums';
@@ -426,6 +427,7 @@ function build(): CVATCore {
             QualityReport,
             Request,
             FramesMetaData,
+            ValidationLayout,
         },
         utils: {
             mask2Rle,

diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts
@@ -16,6 +16,7 @@ import { FieldUpdateTrigger } from './common';
 // frame storage by job id
 const frameDataCache: Record<string, {
     meta: FramesMetaData;
+    metaFetchedTimestamp: number;
     chunkSize: number;
     mode: 'annotation' | 'interpolation';
     startFrame: number;
@@ -57,6 +58,7 @@ export class FramesMetaData {
     public stopFrame: number;
     public frameStep: number;
     public chunkCount: number;
+    public chunksUpdatedDate: string;
 
     #updateTrigger: FieldUpdateTrigger;
 
@@ -71,6 +73,7 @@ export class FramesMetaData {
             size: undefined,
             start_frame: undefined,
             stop_frame: undefined,
+            chunks_updated_date: undefined,
         };
 
         this.#updateTrigger = new FieldUpdateTrigger();
@@ -149,6 +152,9 @@ export class FramesMetaData {
                 frameStep: {
                     get: () => frameStep,
                 },
+                chunksUpdatedDate: {
+                    get: () => data.chunks_updated_date,
+                },
             }),
         );
 
@@ -592,13 +598,45 @@ function getFrameMeta(jobID, frame): SerializedFramesMetaData['frames'][0] {
     return frameMeta;
 }
 
+async function refreshJobCacheIfOutdated(jobID: number): Promise<void> {
+    const cached = frameDataCache[jobID];
+    if (!cached) {
+        throw new Error('Frame data cache is abscent');
+    }
+
+    const META_DATA_RELOAD_PERIOD = 1 * 60 * 60 * 1000; // 1 hour
+    const isOutdated = (Date.now() - cached.metaFetchedTimestamp) > META_DATA_RELOAD_PERIOD;
+
+    if (isOutdated) {
+        // get metadata again if outdated
+        const meta = await getFramesMeta('job', jobID, true);
+        if (new Date(meta.chunksUpdatedDate) > new Date(cached.meta.chunksUpdatedDate)) {
+            // chunks were re-defined. Existing data not relevant anymore
+            // currently we only re-write meta, remove all cached frames from provider and clear cached context images
+            // other parameters (e.g. chunkSize) are not supposed to be changed
+            cached.meta = meta;
+            cached.provider.cleanup(Number.MAX_SAFE_INTEGER);
+            for (const frame of Object.keys(cached.contextCache)) {
+                for (const image of Object.values(cached.contextCache[+frame].data)) {
+                    // close images to immediate memory release
+                    image.close();
+                }
+            }
+            cached.contextCache = {};
+        }
+
+        cached.metaFetchedTimestamp = Date.now();
+    }
+}
+
 export function getContextImage(jobID: number, frame: number): Promise<Record<string, ImageBitmap>> {
     return new Promise<Record<string, ImageBitmap>>((resolve, reject) => {
         if (!(jobID in frameDataCache)) {
             reject(new Error(
                 'Frame data was not initialized for this job. Try first requesting any frame.',
             ));
         }
+
         const frameData = frameDataCache[jobID];
         const requestId = frame;
         const { startFrame } = frameData;
@@ -695,7 +733,9 @@ export async function getFrame(
     dimension: DimensionType,
     getChunk: (chunkIndex: number, quality: ChunkQuality) => Promise<ArrayBuffer>,
 ): Promise<FrameData> {
-    if (!(jobID in frameDataCache)) {
+    const dataCacheExists = jobID in frameDataCache;
+
+    if (!dataCacheExists) {
         const blockType = chunkType === 'video' ? BlockType.MP4VIDEO : BlockType.ARCHIVE;
         const meta = await getFramesMeta('job', jobID);
 
@@ -718,6 +758,7 @@ export async function getFrame(
 
         frameDataCache[jobID] = {
             meta,
+            metaFetchedTimestamp: Date.now(),
             chunkSize,
             mode,
             startFrame,
@@ -743,6 +784,22 @@ export async function getFrame(
         };
     }
 
+    // basically the following functions may be affected if job cache is outdated
+    // - getFrame
+    // - getContextImage
+    // - getCachedChunks
+    // And from this idea we should call refreshJobCacheIfOutdated from each one
+    // Hovewer, following from the order, these methods are usually called
+    // it may lead to even more confusing behaviour
+    //
+    // Usually user first receives frame, then user receives ranges and finally user receives context images
+    // In this case (extremely rare, but nevertheless possible) user may get context images related to another frame
+    // - if cache gets outdated after getFrame() call
+    // - and before getContextImage() call
+    // - and both calls refer to the same frame that is refreshed honeypot frame and this frame has context images
+    // Thus, it is better to only call `refreshJobCacheIfOutdated` from getFrame()
+    await refreshJobCacheIfOutdated(jobID);
+
     const frameMeta = getFrameMeta(jobID, frame);
     frameDataCache[jobID].provider.setRenderSize(frameMeta.width, frameMeta.height);
     frameDataCache[jobID].decodeForward = isPlaying;
@@ -759,7 +816,7 @@ export async function getFrame(
     });
 }
 
-export async function getDeletedFrames(instanceType: 'job' | 'task', id): Promise<Record<number, boolean>> {
+export async function getDeletedFrames(instanceType: 'job' | 'task', id: number): Promise<Record<number, boolean>> {
     if (instanceType === 'job') {
         const { meta } = frameDataCache[id];
         return meta.deletedFrames;

diff --git a/cvat-core/src/index.ts b/cvat-core/src/index.ts
@@ -32,6 +32,7 @@ import QualityConflict from './quality-conflict';
 import QualitySettings from './quality-settings';
 import AnalyticsReport from './analytics-report';
 import AnnotationGuide from './guide';
+import ValidationLayout from './validation-layout';
 import { Request } from './request';
 import BaseSingleFrameAction, { listActions, registerAction, runActions } from './annotations-actions';
 import {
@@ -215,6 +216,7 @@ export default interface CVATCore {
         AnalyticsReport: typeof AnalyticsReport;
         Request: typeof Request;
         FramesMetaData: typeof FramesMetaData;
+        ValidationLayout: typeof ValidationLayout;
     };
     utils: {
         mask2Rle: typeof mask2Rle;

diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts
@@ -19,7 +19,7 @@ import {
     SerializedInvitationData, SerializedCloudStorage, SerializedFramesMetaData, SerializedCollection,
     SerializedQualitySettingsData, APIQualitySettingsFilter, SerializedQualityConflictData, APIQualityConflictsFilter,
     SerializedQualityReportData, APIQualityReportsFilter, SerializedAnalyticsReport, APIAnalyticsReportFilter,
-    SerializedRequest,
+    SerializedRequest, SerializedValidationLayout,
 } from './server-response-types';
 import { PaginatedResource } from './core-types';
 import { Request } from './request';
@@ -1382,6 +1382,24 @@ async function deleteJob(jobID: number): Promise<void> {
     }
 }
 
+const validationLayout = (instance: 'tasks' | 'jobs') => async (
+    id: number,
+): Promise<SerializedValidationLayout | null> => {
+    const { backendAPI } = config;
+
+    try {
+        const response = await Axios.get(`${backendAPI}/${instance}/${id}/validation_layout`, {
+            params: {
+                ...enableOrganization(),
+            },
+        });
+
+        return response.data;
+    } catch (errorData) {
+        throw generateError(errorData);
+    }
+};
+
 async function getUsers(filter = { page_size: 'all' }): Promise<SerializedUser[]> {
     const { backendAPI } = config;
 
@@ -2376,6 +2394,7 @@ export default Object.freeze({
         getPreview: getPreview('tasks'),
         backup: backupTask,
         restore: restoreTask,
+        validationLayout: validationLayout('tasks'),
     }),
 
     labels: Object.freeze({
@@ -2391,6 +2410,7 @@ export default Object.freeze({
         create: createJob,
         delete: deleteJob,
         exportDataset: exportDataset('jobs'),
+        validationLayout: validationLayout('jobs'),
     }),
 
     users: Object.freeze({

diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts
@@ -459,6 +459,7 @@ export interface SerializedFramesMetaData {
     deleted_frames: number[];
     included_frames: number[];
     frame_filter: string;
+    chunks_updated_date: string;
     frames: {
         width: number;
         height: number;
@@ -522,3 +523,9 @@ export interface SerializedRequest {
     expiry_date?: string;
     owner?: any;
 }
+
+export interface SerializedValidationLayout {
+    honeypot_count?: number;
+    honeypot_frames?: number[];
+    honeypot_real_frames?: number[];
+}
diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts
@@ -27,7 +27,7 @@ import {
     decodePreview,
 } from './frames';
 import Issue from './issue';
-import { SerializedLabel, SerializedTask } from './server-response-types';
+import { SerializedLabel, SerializedTask, SerializedValidationLayout } from './server-response-types';
 import { checkInEnum, checkObjectType } from './common';
 import {
     getCollection, getSaver, clearAnnotations, getAnnotations,
@@ -37,6 +37,7 @@ import AnnotationGuide from './guide';
 import requestsManager from './requests-manager';
 import { Request } from './request';
 import User from './user';
+import ValidationLayout from './validation-layout';
 
 // must be called with task/job context
 async function deleteFrameWrapper(jobID, frame): Promise<void> {
@@ -164,6 +165,19 @@ export function implementJob(Job: typeof JobClass): typeof JobClass {
         },
     });
 
+    Object.defineProperty(Job.prototype.validationLayout, 'implementation', {
+        value: async function validationLayoutImplementation(
+            this: JobClass,
+        ): ReturnType<typeof JobClass.prototype.validationLayout> {
+            const result = await serverProxy.jobs.validationLayout(this.id);
+            if (Object.keys(result).length) {
+                return new ValidationLayout(result as Required<SerializedValidationLayout>);
+            }
+
+            return null;
+        },
+    });
+
     Object.defineProperty(Job.prototype.frames.get, 'implementation', {
         value: function getFrameImplementation(
             this: JobClass,
@@ -624,6 +638,19 @@ export function implementTask(Task: typeof TaskClass): typeof TaskClass {
         },
     });
 
+    Object.defineProperty(Task.prototype.validationLayout, 'implementation', {
+        value: async function validationLayoutImplementation(
+            this: TaskClass,
+        ): ReturnType<typeof TaskClass.prototype.validationLayout> {
+            const result = await serverProxy.tasks.validationLayout(this.id);
+            if (Object.keys(result).length) {
+                return new ValidationLayout(result as Required<SerializedValidationLayout>);
+            }
+
+            return null;
+        },
+    });
+
     Object.defineProperty(Task.prototype.save, 'implementation', {
         value: async function saveImplementation(
             this: TaskClass,

diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts
@@ -28,6 +28,7 @@ import { Request } from './request';
 import logger from './logger';
 import Issue from './issue';
 import ObjectState from './object-state';
+import ValidationLayout from './validation-layout';
 
 function buildDuplicatedAPI(prototype) {
     Object.defineProperties(prototype, {
@@ -685,6 +686,11 @@ export class Job extends Session {
         return result;
     }
 
+    async validationLayout(): Promise<ValidationLayout | null> {
+        const result = await PluginRegistry.apiWrapper.call(this, Job.prototype.validationLayout);
+        return result;
+    }
+
     async openIssue(issue: Issue, message: string): Promise<Issue> {
         const result = await PluginRegistry.apiWrapper.call(this, Job.prototype.openIssue, issue, message);
         return result;
@@ -1179,6 +1185,11 @@ export class Task extends Session {
         const result = await PluginRegistry.apiWrapper.call(this, Task.prototype.guide);
         return result;
     }
+
+    async validationLayout(): Promise<ValidationLayout | null> {
+        const result = await PluginRegistry.apiWrapper.call(this, Task.prototype.validationLayout);
+        return result;
+    }
 }
 
 buildDuplicatedAPI(Job.prototype);

diff --git a/cvat-core/src/validation-layout.ts b/cvat-core/src/validation-layout.ts
@@ -0,0 +1,44 @@
+// Copyright (C) 2024 CVAT.ai Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import { SerializedValidationLayout } from 'server-response-types';
+import PluginRegistry from './plugins';
+
+export default class ValidationLayout {
+    #honeypotFrames: number[];
+    #honeypotRealFrames: number[];
+
+    public constructor(data: Required<SerializedValidationLayout>) {
+        this.#honeypotFrames = [...data.honeypot_frames];
+        this.#honeypotRealFrames = [...data.honeypot_real_frames];
+    }
+
+    public get honeypotFrames() {
+        return [...this.#honeypotFrames];
+    }
+
+    public get honeypotRealFrames() {
+        return [...this.#honeypotRealFrames];
+    }
+
+    async getRealFrame(frame: number): Promise<number | null> {
+        const result = await PluginRegistry.apiWrapper.call(this, ValidationLayout.prototype.getRealFrame, frame);
+        return result;
+    }
+}
+
+Object.defineProperties(ValidationLayout.prototype.getRealFrame, {
+    implementation: {
+        writable: false,
+        enumerable: false,
+        value: function implementation(this: ValidationLayout, frame: number): number | null {
+            const index = this.honeypotFrames.indexOf(frame);
+            if (index !== -1) {
+                return this.honeypotRealFrames[index];
+            }
+
+            return null;
+        },
+    },
+});