initial RoboflowObjectDetector implementation, Redux state updates

SkalskiP · Sep 27, 2022 · aa7bf80 · aa7bf80
1 parent dc9bb07
commit aa7bf80
Show file tree

Hide file tree

Showing 8 changed files with 197 additions and 5 deletions.
diff --git a/public/ico/roboflow-logo.png b/public/ico/roboflow-logo.png
diff --git a/src/ai/RoboflowObjectDetector.ts b/src/ai/RoboflowObjectDetector.ts
@@ -0,0 +1,49 @@
+import * as express from 'roboflow';
+
+
+export interface DetectedObject {
+    bbox: [number, number, number, number];
+    class: string;
+    score: number;
+}
+
+
+export class RoboflowObjectDetector {
+    private static model;
+
+    public static loadModel(
+        publishableKey: string,
+        modelId: string,
+        modelVersion: number,
+        onSuccess: () => unknown,
+        onError: (error: Error) => unknown
+    ) {
+        roboflow.auth({
+            publishable_key: publishableKey
+        }).load({
+            model: modelId,
+            version: modelVersion
+        }).then((model) => {
+            RoboflowObjectDetector.model = model
+            onSuccess()
+        }).catch((error) => {
+            onError(error)
+        })
+    }
+
+    public static predict(image: HTMLImageElement, callback?: (predictions: DetectedObject[]) => unknown) {
+        if (!RoboflowObjectDetector.model) return;
+
+        RoboflowObjectDetector.model
+            .detect(image)
+            .then((predictions: DetectedObject[]) => {
+                if (callback) {
+                    callback(predictions)
+                }
+            })
+            .catch((error) => {
+                // TODO
+                throw new Error(error as string);
+            })
+    }
+}
diff --git a/src/store/Actions.ts b/src/store/Actions.ts
@@ -4,6 +4,7 @@ export enum Action {
     UPDATE_REJECTED_SUGGESTED_LABEL_LIST = '@@UPDATE_REJECTED_SUGGESTED_LABEL_LIST',
     UPDATE_OBJECT_DETECTOR_STATUS = '@@UPDATE_OBJECT_DETECTOR_STATUS',
     UPDATE_POSE_DETECTOR_STATUS = '@@UPDATE_POSE_DETECTOR_STATUS',
+    UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS = '@@UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS',
     UPDATE_DISABLED_AI_FLAG = '@@UPDATE_DISABLED_AI_FLAG',
 
     // GENERAL

diff --git a/src/store/ai/actionCreators.ts b/src/store/ai/actionCreators.ts
@@ -37,11 +37,20 @@ export function updatePoseDetectorStatus(isPoseDetectorLoaded: boolean): AIActio
     }
 }
 
+export function updateRoboflowJSObjectDetectorStatus(isRoboflowJSObjectDetectorLoaded: boolean): AIActionTypes {
+    return {
+        type: Action.UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS,
+        payload: {
+            isRoboflowJSObjectDetectorLoaded,
+        }
+    }
+}
+
 export function updateDisabledAIFlag(isAIDisabled: boolean): AIActionTypes {
     return {
         type: Action.UPDATE_DISABLED_AI_FLAG,
         payload: {
             isAIDisabled,
         }
     }
-}
+}
diff --git a/src/store/ai/reducer.ts b/src/store/ai/reducer.ts
@@ -6,6 +6,7 @@ const initialState: AIState = {
     rejectedSuggestedLabelList: [],
     isObjectDetectorLoaded: false,
     isPoseDetectorLoaded: false,
+    isRoboflowJSObjectDetectorLoaded: false,
     isAIDisabled: false
 };
 
@@ -38,6 +39,12 @@ export function aiReducer(
                 isPoseDetectorLoaded: action.payload.isPoseDetectorLoaded
             }
         }
+        case Action.UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS: {
+            return {
+                ...state,
+                isRoboflowJSObjectDetectorLoaded: action.payload.isRoboflowJSObjectDetectorLoaded
+            }
+        }
         case Action.UPDATE_DISABLED_AI_FLAG: {
             return {
                 ...state,

diff --git a/src/store/ai/types.ts b/src/store/ai/types.ts
@@ -7,6 +7,9 @@ export type AIState = {
     // POSE NET
     isPoseDetectorLoaded: boolean;
 
+    // ROBOFLOW
+    isRoboflowJSObjectDetectorLoaded: boolean;
+
     // GENERAL
     suggestedLabelList: string[];
     rejectedSuggestedLabelList: string[];
@@ -41,6 +44,13 @@ interface UpdatePoseDetectorStatus {
     }
 }
 
+interface UpdateRoboflowJSObjectDetectorStatus {
+    type: typeof Action.UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS;
+    payload: {
+        isRoboflowJSObjectDetectorLoaded: boolean;
+    }
+}
+
 interface UpdateDisabledAIFlag {
     type: typeof Action.UPDATE_DISABLED_AI_FLAG;
     payload: {
@@ -52,4 +62,5 @@ export type AIActionTypes = UpdateSuggestedLabelList
     | UpdateRejectedSuggestedLabelList
     | UpdateObjectDetectorStatus
     | UpdatePoseDetectorStatus
+    | UpdateRoboflowJSObjectDetectorStatus
     | UpdateDisabledAIFlag
diff --git a/src/views/PopupView/LoadRoboflowModelPopup/LoadRoboflowModelPopup.scss b/src/views/PopupView/LoadRoboflowModelPopup/LoadRoboflowModelPopup.scss
@@ -0,0 +1,34 @@
+@import '../../../settings/Settings';
+
+.load-roboflow-model-popup {
+  display: flex;
+  flex-direction: column;
+  flex-wrap: nowrap;
+  justify-content: center;
+  align-items: center;
+  align-content: center;
+  padding: 40px 0;
+  flex: 1;
+
+  .message {
+    align-self: stretch;
+    color: white;
+    font-size: 15px;
+    padding: 0 40px 30px 40px;
+
+    display: flex;
+    flex-direction: column;
+    flex-wrap: nowrap;
+    align-items: center;
+
+    > a {
+      max-width: 120px;
+      margin-bottom: 30px;
+
+      > img {
+        max-width: 120px;
+        user-select: none;
+      }
+    }
+  }
+}
diff --git a/src/views/PopupView/LoadRoboflowModelPopup/LoadRoboflowModelPopup.tsx b/src/views/PopupView/LoadRoboflowModelPopup/LoadRoboflowModelPopup.tsx
@@ -1,29 +1,110 @@
-import React from 'react';
+import React, {useState} from 'react';
 import './LoadRoboflowModelPopup.scss';
 import {GenericYesNoPopup} from '../GenericYesNoPopup/GenericYesNoPopup';
 import {PopupActions} from '../../../logic/actions/PopupActions';
 import {updateActivePopupType} from '../../../store/general/actionCreators';
 import {AppState} from '../../../store';
 import {connect} from 'react-redux';
+import {TextField} from '@mui/material';
+import {styled} from '@mui/system';
+import {Settings} from '../../../settings/Settings';
+import {RoboflowObjectDetector} from '../../../ai/RoboflowObjectDetector';
+
+const StyledTextField = styled(TextField)({
+    '& .MuiInputBase-root': {
+        color: 'white',
+    },
+    '& label': {
+        color: 'white',
+    },
+    '& .MuiInput-underline:before': {
+        borderBottomColor: 'white',
+    },
+    '& .MuiInput-underline:hover:before': {
+        borderBottomColor: 'white',
+    },
+    '& label.Mui-focused': {
+        color: Settings.SECONDARY_COLOR,
+    },
+    '& .MuiInput-underline:after': {
+        borderBottomColor: Settings.SECONDARY_COLOR,
+    }
+});
 
 
 const LoadRoboflowModelPopup: React.FC = () => {
+    const [publishableKey, setPublishableKey] = useState('');
+    const [modelId, setModelId] = useState('');
+    const [modelVersion, setModelVersion] = useState(1);
 
-    const onAccept = () => {
+    const onModelLoadSuccess = () => {
         PopupActions.close();
     }
 
+    const onModelLoadError = (error: Error) => {
+        // tslint:disable-next-line:no-console
+        console.log(error)
+    }
+
+    const onAccept = () => {
+        RoboflowObjectDetector.loadModel(publishableKey, modelId, modelVersion, onModelLoadSuccess, onModelLoadError);
+    }
+
     const onReject = () => {
         PopupActions.close();
     }
 
     const renderContent = () => {
-        return null;
+        return <div className='load-roboflow-model-popup'>
+            <div className='message'>
+                <a href={'https://roboflow.com/'} target='_blank' rel='noopener noreferrer'>
+                    <img
+                        draggable={false}
+                        alt={'upload'}
+                        src={'ico/roboflow-logo.png'}
+                    />
+                </a>
+                <p>Use your pretrained object detection models to speed up annotation.</p>
+            </div>
+
+            <StyledTextField
+                variant='standard'
+                id={'key'}
+                autoComplete={'off'}
+                autoFocus={true}
+                type={'password'}
+                margin={'dense'}
+                label={'Publishable key'}
+                value={publishableKey}
+                onChange={(event: React.ChangeEvent<HTMLInputElement>) => setPublishableKey(event.target.value)}
+                style={{ width: 280 }}
+                InputLabelProps={{
+                    shrink: true
+                }}
+            />
+
+            <StyledTextField
+                variant='standard'
+                id={'key'}
+                autoComplete={'off'}
+                autoFocus={false}
+                type={'text'}
+                margin={'dense'}
+                label={'Model id'}
+                value={modelId}
+                onChange={(event: React.ChangeEvent<HTMLInputElement>) => setModelId(event.target.value)}
+                style={{ width: 280 }}
+                InputLabelProps={{
+                    shrink: true
+                }}
+            />
+
+        </div>
     }
 
     return (
         <GenericYesNoPopup
-            title={'Say hello to AI'}
+            title={'Load roboflow.js model'}
             renderContent={renderContent}
             acceptLabel={'Use model!'}
             onAccept={onAccept}