diff --git a/BabbleApp/algo_settings_widget.py b/BabbleApp/algo_settings_widget.py index 6e4cec9..abd75c5 100644 --- a/BabbleApp/algo_settings_widget.py +++ b/BabbleApp/algo_settings_widget.py @@ -198,6 +198,7 @@ def render(self, window, event, values): except ValueError: pass # Ignore invalid float conversion + if self.config.gui_model_file != values[self.gui_model_file]: self.config.gui_model_file = values[self.gui_model_file] changed = True @@ -210,6 +211,7 @@ def render(self, window, event, values): except ValueError: pass # Ignore invalid float conversion + if self.config.gui_use_gpu != values[self.gui_use_gpu]: self.config.gui_use_gpu = values[self.gui_use_gpu] changed = True @@ -222,6 +224,7 @@ def render(self, window, event, values): except ValueError: pass # Ignore invalid int conversion + if self.config.gui_runtime != str(values[self.gui_runtime]): self.config.gui_runtime = str(values[self.gui_runtime]) changed = True diff --git a/BabbleApp/babble_processor.py b/BabbleApp/babble_processor.py index 615e460..2eddb4c 100644 --- a/BabbleApp/babble_processor.py +++ b/BabbleApp/babble_processor.py @@ -68,6 +68,7 @@ def __init__( self.current_image_gray = None self.current_frame_number = None self.current_fps = None + self.FRAMESIZE = [0,0,1] self.calibration_frame_counter = None @@ -129,6 +130,7 @@ def capture_crop_rotate_image(self): try: # Get frame from capture source, crop to ROI + self.FRAMESIZE = self.current_image.shape self.current_image = self.current_image[ int(self.config.roi_window_y): int( self.config.roi_window_y + self.config.roi_window_h @@ -223,15 +225,14 @@ def run(self): new_blue_channel = red_channel new_green_channel = red_channel self.current_image = cv2.merge((new_blue_channel, new_green_channel, red_channel)) - self.current_image_gray = cv2.cvtColor( - self.current_image, cv2.COLOR_BGR2GRAY + self.current_image, cv2.COLOR_BGR2GRAY ) self.current_image_gray_clean = self.current_image_gray.copy() #copy this frame to have a clean image for blink algo run_model(self) - if self.config.use_calibration: + if self.settings.use_calibration: self.output = cal.cal_osc(self, self.output) #else: @@ -239,3 +240,5 @@ def run(self): #print(self.output) self.output_images_and_update(CamInfo(self.current_algo, self.output)) + def get_framesize(self): + return self.FRAMESIZE \ No newline at end of file diff --git a/BabbleApp/babbleapp.py b/BabbleApp/babbleapp.py index a5abead..e979dcd 100644 --- a/BabbleApp/babbleapp.py +++ b/BabbleApp/babbleapp.py @@ -28,6 +28,7 @@ from osc import VRChatOSCReceiver, VRChatOSC from general_settings_widget import SettingsWidget from algo_settings_widget import AlgoSettingsWidget +from calib_settings_widget import CalibSettingsWidget from utils.misc_utils import is_nt if is_nt: from winotify import Notification @@ -41,9 +42,11 @@ CAM_NAME = "-CAMWIDGET-" SETTINGS_NAME = "-SETTINGSWIDGET-" ALGO_SETTINGS_NAME = "-ALGOSETTINGSWIDGET-" +CALIB_SETTINGS_NAME = "-CALIBSETTINGSWIDGET-" CAM_RADIO_NAME = "-CAMRADIO-" SETTINGS_RADIO_NAME = "-SETTINGSRADIO-" ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-" +CALIB_SETTINGS_RADIO_NAME = "-CALIBSETTINGSRADIO-" page_url = "https://github.com/SummerSigh/ProjectBabble/releases/latest" appversion = "Babble v2.0.6 Alpha" @@ -108,6 +111,7 @@ def main(): settings = [ SettingsWidget(Tab.SETTINGS, config, osc_queue), AlgoSettingsWidget(Tab.ALGOSETTINGS, config, osc_queue), + CalibSettingsWidget(Tab.CALIBRATION, config, osc_queue), ] layout = [ @@ -133,6 +137,13 @@ def main(): default=(config.cam_display_id == Tab.ALGOSETTINGS), key=ALGO_SETTINGS_RADIO_NAME, ), + sg.Radio( + "Calibration", + "TABSELECTRADIO", + background_color="#292929", + default=(config.cam_display_id == Tab.CALIBRATION), + key=CALIB_SETTINGS_RADIO_NAME, + ), ], [ @@ -157,6 +168,13 @@ def main(): visible=(config.cam_display_id in [Tab.ALGOSETTINGS]), background_color="#424042", ), + sg.Column( + settings[2].widget_layout, + vertical_alignment="top", + key=CALIB_SETTINGS_NAME, + visible=(config.cam_display_id in [Tab.CALIBRATION]), + background_color="#424042", + ), ], ] @@ -167,6 +185,8 @@ def main(): settings[0].start() if config.cam_display_id in [Tab.ALGOSETTINGS]: settings[1].start() + if config.cam_display_id in [Tab.CALIBRATION]: + settings[2].start() # the cam needs to be running before it is passed to the OSC if config.settings.gui_ROSC: @@ -206,9 +226,11 @@ def main(): cams[0].start() settings[0].stop() settings[1].stop() + settings[2].stop() window[CAM_NAME].update(visible=True) window[SETTINGS_NAME].update(visible=False) window[ALGO_SETTINGS_NAME].update(visible=False) + window[CALIB_SETTINGS_NAME].update(visible=False) config.cam_display_id = Tab.CAM config.save() @@ -217,10 +239,12 @@ def main(): elif values[SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.SETTINGS: cams[0].stop() settings[1].stop() + settings[2].stop() settings[0].start() window[CAM_NAME].update(visible=False) window[SETTINGS_NAME].update(visible=True) window[ALGO_SETTINGS_NAME].update(visible=False) + window[CALIB_SETTINGS_NAME].update(visible=False) config.cam_display_id = Tab.SETTINGS config.save() @@ -228,12 +252,26 @@ def main(): elif values[ALGO_SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.ALGOSETTINGS: cams[0].stop() settings[0].stop() + settings[2].stop() settings[1].start() window[CAM_NAME].update(visible=False) window[SETTINGS_NAME].update(visible=False) window[ALGO_SETTINGS_NAME].update(visible=True) + window[CALIB_SETTINGS_NAME].update(visible=False) config.cam_display_id = Tab.ALGOSETTINGS config.save() + + elif values[CALIB_SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.CALIBRATION: + cams[0].start() # Allow tracking to continue in calibration tab + settings[0].stop() + settings[1].stop() + settings[2].start() + window[CAM_NAME].update(visible=False) + window[SETTINGS_NAME].update(visible=False) + window[ALGO_SETTINGS_NAME].update(visible=False) + window[CALIB_SETTINGS_NAME].update(visible=True) + config.cam_display_id = Tab.CALIBRATION + config.save() # Otherwise, render all diff --git a/BabbleApp/calib_settings_values.py b/BabbleApp/calib_settings_values.py new file mode 100644 index 0000000..453ea6a --- /dev/null +++ b/BabbleApp/calib_settings_values.py @@ -0,0 +1,108 @@ + +def set_shapes(widget_id): + shape_index = ["cheekPuffLeft", "cheekPuffRight", "cheekSuckLeft", "cheekSuckRight", "jawOpen", "jawForward", "jawLeft", "jawRight", "noseSneerLeft", "noseSneerRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", + "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmileLeft", + "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthUpperUpLeft", + "mouthUpperUpRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", "mouthPressRight", "mouthStretchLeft", + "mouthStretchRight", "tongueOut", "tongueUp", "tongueDown", "tongueLeft", "tongueRight", "tongueRoll", "tongueBendDown", "tongueCurlUp", "tongueSquish", "tongueFlat", "tongueTwistLeft", "tongueTwistRight"] + + shape = [ + [ + f"-CHEEKPUFFLEFTMIN{widget_id}-", + f"-CHEEKPUFFRIGHTMIN{widget_id}-", + f"-CHEEKSUCKLEFTMIN{widget_id}-", + f"-CHEEKSUCKRIGHTMIN{widget_id}-", + f"-JAWOPENMIN{widget_id}-", + f"-JAWFORWARDMIN{widget_id}-", + f"-JAWLEFTMIN{widget_id}-", + f"-JAWRIGHTMIN{widget_id}-", + f"-NOSESNEERLEFTMIN{widget_id}-", + f"-NOSESNEERRIGHTMIN{widget_id}-", + f"-MOUTHFUNNELMIN{widget_id}-", + f"-MOUTHPUCKERMIN{widget_id}-", + f"-MOUTHLEFTMIN{widget_id}-", + f"-MOUTHRIGHTMIN{widget_id}-", + f"-MOUTHROLLUPPERMIN{widget_id}-", + f"-MOUTHROLLLOWERMIN{widget_id}-", + f"-MOUTHSHRUGUPPERMIN{widget_id}-", + f"-MOUTHSHRUGLOWERMIN{widget_id}-", + f"-MOUTHCLOSEMIN{widget_id}-", + f"-MOUTHSMILELEFTMIN{widget_id}-", + f"-MOUTHSMILERIGHTMIN{widget_id}-", + f"-MOUTHFROWNLEFTMIN{widget_id}-", + f"-MOUTHFROWNRIGHTMIN{widget_id}-", + f"-MOUTHDIMPLELEFTMIN{widget_id}-", + f"-MOUTHDIMPLERIGHTMIN{widget_id}-", + f"-MOUTHUPPERUPLEFTMIN{widget_id}-", + f"-MOUTHUPPERUPRIGHTMIN{widget_id}-", + f"-MOUTHLOWERDOWNLEFTMIN{widget_id}-", + f"-MOUTHLOWERDOWNRIGHTMIN{widget_id}-", + f"-MOUTHPRESSLEFTMIN{widget_id}-", + f"-MOUTHPRESSRIGHTMIN{widget_id}-", + f"-MOUTHSTRETCHLEFTMIN{widget_id}-", + f"-MOUTHSTRETCHRIGHTMIN{widget_id}-", + f"-TONGUEOUTMIN{widget_id}-", + f"-TONGUEUPMIN{widget_id}-", + f"-TONGUEDOWNMIN{widget_id}-", + f"-TONGUELEFTMIN{widget_id}-", + f"-TONGUERIGHTMIN{widget_id}-", + f"-TONGUEROLLMIN{widget_id}-", + f"-TONGUEBENDDOWNMIN{widget_id}-", + f"-TONGUECURLUPMIN{widget_id}-", + f"-TONGUESQUISHMIN{widget_id}-", + f"-TONGUEFLATMIN{widget_id}-", + f"-TONGUETWISTLEFTMIN{widget_id}-", + f"-TONGUETWISTRIGHTMIN{widget_id}-", + ], + [ + f"-CHEEKPUFFLEFTMAX{widget_id}-", + f"-CHEEKPUFFRIGHTMAX{widget_id}-", + f"-CHEEKSUCKLEFTMAX{widget_id}-", + f"-CHEEKSUCKRIGHTMAX{widget_id}-", + f"-JAWOPENMAX{widget_id}-", + f"-JAWFORWARDMAX{widget_id}-", + f"-JAWLEFTMAX{widget_id}-", + f"-JAWRIGHTMAX{widget_id}-", + f"-NOSESNEERLEFTMAX{widget_id}-", + f"-NOSESNEERRIGHTMAX{widget_id}-", + f"-MOUTHFUNNELMAX{widget_id}-", + f"-MOUTHPUCKERMAX{widget_id}-", + f"-MOUTHLEFTMAX{widget_id}-", + f"-MOUTHRIGHTMAX{widget_id}-", + f"-MOUTHROLLUPPERMAX{widget_id}-", + f"-MOUTHROLLLOWERMAX{widget_id}-", + f"-MOUTHSHRUGUPPERMAX{widget_id}-", + f"-MOUTHSHRUGLOWERMAX{widget_id}-", + f"-MOUTHCLOSEMAX{widget_id}-", + f"-MOUTHSMILELEFTMAX{widget_id}-", + f"-MOUTHSMILERIGHTMAX{widget_id}-", + f"-MOUTHFROWNLEFTMAX{widget_id}-", + f"-MOUTHFROWNRIGHTMAX{widget_id}-", + f"-MOUTHDIMPLELEFTMAX{widget_id}-", + f"-MOUTHDIMPLERIGHTMAX{widget_id}-", + f"-MOUTHUPPERUPLEFTMAX{widget_id}-", + f"-MOUTHUPPERUPRIGHTMAX{widget_id}-", + f"-MOUTHLOWERDOWNLEFTMAX{widget_id}-", + f"-MOUTHLOWERDOWNRIGHTMAX{widget_id}-", + f"-MOUTHPRESSLEFTMAX{widget_id}-", + f"-MOUTHPRESSRIGHTMAX{widget_id}-", + f"-MOUTHSTRETCHLEFTMAX{widget_id}-", + f"-MOUTHSTRETCHRIGHTMAX{widget_id}-", + f"-TONGUEOUTMAX{widget_id}-", + f"-TONGUEUPMAX{widget_id}-", + f"-TONGUEDOWNMAX{widget_id}-", + f"-TONGUELEFTMAX{widget_id}-", + f"-TONGUERIGHTMAX{widget_id}-", + f"-TONGUEROLLMAX{widget_id}-", + f"-TONGUEBENDDOWNMAX{widget_id}-", + f"-TONGUECURLUPMAX{widget_id}-", + f"-TONGUESQUISHMAX{widget_id}-", + f"-TONGUEFLATMAX{widget_id}-", + f"-TONGUETWISTLEFTMAX{widget_id}-", + f"-TONGUETWISTRIGHTMAX{widget_id}-", + ], + + ] + return shape_index, shape + + diff --git a/BabbleApp/calib_settings_widget.py b/BabbleApp/calib_settings_widget.py new file mode 100644 index 0000000..3c2df43 --- /dev/null +++ b/BabbleApp/calib_settings_widget.py @@ -0,0 +1,221 @@ +import PySimpleGUI as sg + +from config import BabbleSettingsConfig +from osc import Tab +from queue import Queue +from threading import Event +import numpy as np +from calib_settings_values import set_shapes + + +class CalibSettingsWidget: + def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue: Queue): + + + self.gui_general_settings_layout = f"-GENERALSETTINGSLAYOUT{widget_id}-" + self.gui_reset_min = f"-RESETMIN{widget_id}-" + self.gui_reset_max = f"-RESETMAX{widget_id}-" + self.gui_multiply = f"-MULTIPLY{widget_id}-" + self.gui_calibration_mode = f"-CALIBRATIONMODE{widget_id}-" + self.main_config = main_config + self.config = main_config.settings + self.array = np.fromstring(self.config.calib_array.replace('[', '').replace(']', ''), sep=',').reshape(2, 45) + self.calibration_list = ['Neutral', 'Full'] + self.osc_queue = osc_queue + self.shape_index, self.shape = set_shapes(widget_id) + self.refreshed = False + + + + # Define the window's contents + s = self.single_shape + d = self.double_shape + self.general_settings_layout = [ + + d('cheekPuff'), + d('cheekSuck'), + s('jawOpen'), + s('jawForward'), + d('jaw'), + d('noseSneer'), + s('mouthFunnel'), + s('mouthPucker'), + d('mouth'), + s('mouthRollUpper'), + s('mouthRollLower'), + s('mouthShrugUpper'), + s('mouthShrugLower'), + s('mouthClose'), + d('mouthSmile'), + d('mouthFrown'), + d('mouthDimple'), + d('mouthUpperUp'), + d('mouthLowerDown'), + d('mouthPress'), + d('mouthStretch'), + s('tongueOut'), + s('tongueUp'), + s('tongueDown'), + d('tongue'), + s('tongueRoll'), + s('tongueBendDown'), + s('tongueCurlUp'), + s('tongueSquish'), + s('tongueFlat'), + d('tongueTwist'), + + + ] + + self.widget_layout = [ + [ + sg.Text("Calibration Settings:", background_color='#242224'), + sg.Text("Calibration Mode:", background_color='#424042'), + sg.OptionMenu( + self.calibration_list, + self.config.calibration_mode, + key=self.gui_calibration_mode, + tooltip='Neutral = Only Min values are set when starting and stopping calibration. Full = Min and Max values are set based on recorded values when starting calibration.', + ), + ], + [ + sg.Text("Left ", expand_x=True, justification='left'), + sg.Text("Shape", expand_x=True, justification='center'), + sg.Text("Right", expand_x=True, justification='right') + ], + [ + sg.Text("Min", expand_x=True, justification='center'), + sg.Text("Max", expand_x=True, justification='center'), + sg.HSeparator(pad=(50,0)), + sg.Text("Max", expand_x=True, justification='center'), + sg.Text("Min", expand_x=True, justification='center'), + ], + [ + sg.Column( + self.general_settings_layout, + key=self.gui_general_settings_layout, + scrollable=True, + vertical_scroll_only=True, + element_justification='center', + background_color='#424042' ), + ], + [ + sg.Button("Reset Min", key=self.gui_reset_min, button_color='#FF0000', tooltip = "Reset minimum values",), + sg.Button("Reset Max", key=self.gui_reset_max, button_color='#FF0000', tooltip = "Reset maximum values",), + ], + ] + + self.cancellation_event = Event() # Set the event until start is called, otherwise we can block if shutdown is called. + self.cancellation_event.set() + self.image_queue = Queue(maxsize=2) + + def double_shape(self, shapename): + indexl = self.shape_index.index(f"{shapename}Left") + indexr = self.shape_index.index(f"{shapename}Right") + double_shape = [ + sg.InputText( + default_text=self.array[0][indexl], + key=self.shape[0][indexl], + size=(8), + tooltip="Min Left Value", + ), + sg.InputText( + default_text=self.array[1][indexl], + key=self.shape[1][indexl], + size=(8), + tooltip="Max Left Value", + ), + sg.Text(f"{shapename}Left/Right", background_color='#424042', expand_x=True), + sg.InputText( + default_text=self.array[1][indexr], + key=self.shape[1][indexr], + size=(8), + tooltip="Max Right Value", + ), + sg.InputText( + default_text=self.array[0][indexr], + key=self.shape[0][indexr], + size=(8), + tooltip="Min Right Value", + ), + ] + return double_shape + + def single_shape(self, shapename): + index = self.shape_index.index(f"{shapename}") + single_shape = [ + sg.InputText( + default_text=self.array[0][index], + key=self.shape[0][index], + size=(8), + tooltip="Min Left Value", + ), + sg.InputText( + default_text=self.array[1][index], + key=self.shape[1][index], + size=(8), + tooltip="Max Left Value", + ), + sg.Text(f"{shapename}", background_color='#424042', expand_x=True), + ] + return single_shape + + def started(self): + return not self.cancellation_event.is_set() + + def start(self): + # If we're already running, bail + if not self.cancellation_event.is_set(): + return + self.cancellation_event.clear() + self.array = np.fromstring(self.config.calib_array.replace('[', '').replace(']', ''), sep=',').reshape(2, 45) # Reload the array from the config + self.refreshed = False + + def stop(self): + # If we're not running yet, bail + if self.cancellation_event.is_set(): + return + self.cancellation_event.set() + + def render(self, window, event, values): + # If anything has changed in our configuration settings, change/update those. + changed = False + if not self.refreshed: + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + window[element2].update(float(self.array[count1][count2])) + #values[element2] = float(self.array[count1][count2]) + self.refreshed = True + print('DEBUG: Refreshed') + + if self.config.calibration_mode != str(values[self.gui_calibration_mode]): + self.config.calibration_mode = str(values[self.gui_calibration_mode]) + changed = True + + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + if values[element2] != '': + try: + if float(self.array[count1][count2]) != float(values[element2]): + self.array[count1][count2] = float(values[element2]) + changed = True + except: print("Not a float") + + if event == self.gui_reset_min: + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + self.array[0][count2] = float(0) + changed = True + self.refreshed = False + elif event == self.gui_reset_max: + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + self.array[1][count2] = float(1) + changed = True + self.refreshed = False + + if changed: + self.config.calib_array = np.array2string(self.array, separator=',') + self.main_config.save() + #print(self.main_config) + self.osc_queue.put(Tab.CALIBRATION) diff --git a/BabbleApp/camera.py b/BabbleApp/camera.py index 839e6b1..213a26c 100644 --- a/BabbleApp/camera.py +++ b/BabbleApp/camera.py @@ -66,6 +66,7 @@ def __init__( self.prevft = 0 self.newft = 0 self.fl = [0] + self.FRAME_SIZE = [0,0] self.error_message = f"{Fore.YELLOW}[WARN] Capture source {{}} not found, retrying...{Fore.RESET}" @@ -83,13 +84,11 @@ def run(self): return should_push = True # If things aren't open, retry until they are. Don't let read requests come in any earlier - # than this, otherwise we can deadlock ourselves. + # than this, otherwise we can deadlock (valve reference) ourselves. if ( self.config.capture_source is not None and self.config.capture_source != "" ): - self.current_capture_source = self.config.capture_source - - if "COM" in str(self.config.capture_source) and self.config.capture_source not in self.camera_list: + if "COM" in str(self.config.capture_source): if ( self.serial_connection is None or self.camera_status == CameraState.DISCONNECTED @@ -114,7 +113,7 @@ def run(self): if self.config.capture_source not in self.camera_list: self.current_capture_source = self.config.capture_source else: - self.current_capture_source = get_camera_index_by_name(self.current_capture_source) + self.current_capture_source = get_camera_index_by_name(self.config.capture_source) if self.config.use_ffmpeg: self.cv2_camera = cv2.VideoCapture(self.current_capture_source, cv2.CAP_FFMPEG) @@ -139,9 +138,11 @@ def run(self): if should_push and not self.capture_event.wait(timeout=0.02): continue if self.config.capture_source is not None: - if "COM" in str(self.config.capture_source): + ports = ("COM", "/dev/tty") + if any(x in str(self.config.capture_source) for x in ports): self.get_serial_camera_picture(should_push) else: + self.__del__() self.get_cv2_camera_picture(should_push) if not should_push: # if we get all the way down here, consider ourselves connected @@ -153,6 +154,7 @@ def get_cv2_camera_picture(self, should_push): if not ret: self.cv2_camera.set(cv2.CAP_PROP_POS_FRAMES, 0) raise RuntimeError("Problem while getting frame") + self.FRAME_SIZE = image.shape frame_number = self.cv2_camera.get(cv2.CAP_PROP_POS_FRAMES) # Calculate the fps. yeah = time.time() @@ -175,7 +177,7 @@ def get_cv2_camera_picture(self, should_push): #self.bps = image.nbytes if should_push: self.push_image_to_queue(image, frame_number, self.fps) - except: + except Exception as e: print( f"{Fore.YELLOW}[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}") self.camera_status = CameraState.DISCONNECTED diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index d349505..66a3c47 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -8,9 +8,8 @@ from babble_processor import BabbleProcessor, CamInfoOrigin from camera import Camera, CameraState from config import BabbleConfig -from landmark_processor import LandmarkProcessor from osc import Tab -from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names +from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names, get_camera_index_by_name class CameraWidget: @@ -35,7 +34,7 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue): self.gui_vertical_flip = f"-VERTICALFLIP{widget_id}-" self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-" self.use_calibration = f"-USECALIBRATION{widget_id}-" - self.use_n_calibration = f"-USENCALIBRATION{widget_id}-" + self.gui_refresh_button = f"-REFRESHCAMLIST{widget_id}-" self.osc_queue = osc_queue self.main_config = main_config self.cam_id = widget_id @@ -43,6 +42,7 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue): self.config = main_config.cam self.settings = main_config.settings self.camera_list = list_camera_names() + self.maybe_image = None if self.cam_id == Tab.CAM: self.config = main_config.cam else: @@ -67,16 +67,6 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue): self.cam_id, ) - self.babble_landmark = LandmarkProcessor( - self.config, - self.settings_config, - self.main_config, - self.cancellation_event, - self.capture_event, - self.capture_queue, - self.image_queue, - self.cam_id, - ) self.camera_status_queue = Queue(maxsize=2) self.camera = Camera( @@ -91,7 +81,7 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue): self.roi_layout = [ [ - sg.Button("Auto ROI", key=self.gui_autoroi, button_color='#539e8a', tooltip="Automatically set ROI", ), + sg.Button("Select Entire Frame", key=self.gui_autoroi, button_color='#539e8a', tooltip="Automatically set ROI", ), ], [ sg.Graph( @@ -121,24 +111,19 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue): ], [ sg.Button("Start Calibration", key=self.gui_restart_calibration, button_color='#539e8a', - tooltip="Start calibration. Look all arround to all extreams without blinking until sound is heard.", ), + tooltip="Neutural Calibration: Hold a relaxed face, press [Start Calibration] and then press [Stop Calibraion]. \nFull Calibration: Press [Start Calibration] and make as many face movements as you can until it switches back to tracking mode or press [Stop Calibration]", disabled=True), + sg.Button("Stop Calibration", key=self.gui_stop_calibration, button_color='#539e8a', - tooltip="Stop calibration manualy.", ), + tooltip="Stop calibration manualy.", disabled=True), ], [ sg.Checkbox( - "Use Neutral Calibration:", - default=self.config.use_n_calibration, - key=self.use_n_calibration, - background_color='#424042', - tooltip="Toggle use of calibration using minimum values found during a neutral pose calibration step.", - ), - sg.Checkbox( - "Use Full Calibration:", - default=self.config.use_calibration, + "Enable Calibration:", + default=self.settings_config.use_calibration, key=self.use_calibration, background_color='#424042', - tooltip="Toggle use of calibration.", + tooltip="Checked = Calibrated model output. Unchecked = Raw model output", + enable_events=True ), ], [ @@ -172,10 +157,11 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue): self.widget_layout = [ [ sg.Text("Camera Address", background_color='#424042'), - sg.InputCombo(self.camera_list, default_value=self.config.capture_source, + sg.InputCombo(values=self.camera_list, default_value=self.config.capture_source, key=self.gui_camera_addr, tooltip="Enter the IP address or UVC port of your camera. (Include the 'http://')", - enable_events=True) + enable_events=True), + sg.Button("Refresh List", key=self.gui_refresh_button, button_color='#539e8a') ], [ sg.Button("Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#539e8a'), @@ -202,7 +188,7 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue): def _movavg_fps(self, next_fps): self.movavg_fps_queue.append(next_fps) - fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue)) + fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue)) millisec = round((1 / fps if fps else 0) * 1000) return f"{fps} Fps {millisec} ms" @@ -220,8 +206,8 @@ def start(self): self.cancellation_event.clear() self.babble_cnn_thread = Thread(target=self.babble_cnn.run) self.babble_cnn_thread.start() - self.babble_landmark_thread = Thread(target=self.babble_landmark.run) - self.babble_landmark_thread.start() + #self.babble_landmark_thread = Thread(target=self.babble_landmark.run) + #self.babble_landmark_thread.start() self.camera_thread = Thread(target=self.camera.run) self.camera_thread.start() @@ -231,7 +217,7 @@ def stop(self): return self.cancellation_event.set() self.babble_cnn_thread.join() - self.babble_landmark_thread.join() + #self.babble_landmark_thread.join() self.camera_thread.join() def render(self, window, event, values): @@ -251,10 +237,18 @@ def render(self, window, event, values): try: self.config.use_ffmpeg = False # Try storing ints as ints, for those using wired cameras. - if value not in self.camera_list: - self.config.capture_source = int(value) - else: + #if value not in self.camera_list: + # self.config.capture_source = value + #if "COM" not in value: + ports = ("COM", "/dev/tty") + if any(x in str(value) for x in ports): self.config.capture_source = value + else: + cam = get_camera_index_by_name(value) # Set capture_source to the UVC index. Otherwise treat value like an ipcam if we return none + if cam != None: + self.config.capture_source = get_camera_index_by_name(value) + else: + self.config.capture_source = value except ValueError: if value == "": self.config.capture_source = None @@ -280,7 +274,6 @@ def render(self, window, event, values): self.config.rotation_angle = int(values[self.gui_rotation_slider]) changed = True - #print(self.config.gui_vertical_flip) if self.config.gui_vertical_flip != values[self.gui_vertical_flip]: self.config.gui_vertical_flip = values[self.gui_vertical_flip] changed = True @@ -289,12 +282,8 @@ def render(self, window, event, values): self.config.gui_horizontal_flip = values[self.gui_horizontal_flip] changed = True - if self.config.use_calibration != values[self.use_calibration]: - self.config.use_calibration = values[self.use_calibration] - changed = True - - if self.config.use_n_calibration != values[self.use_n_calibration]: - self.config.use_n_calibration = values[self.use_n_calibration] + if self.settings_config.use_calibration != values[self.use_calibration]: + self.settings_config.use_calibration = values[self.use_calibration] changed = True if changed: @@ -314,6 +303,17 @@ def render(self, window, event, values): window[self.gui_roi_layout].update(visible=True) window[self.gui_tracking_layout].update(visible=False) + if event == self.use_calibration: + print("toggle event") + if self.settings_config.use_calibration == True: + window[self.gui_restart_calibration].update(disabled = False) + window[self.gui_stop_calibration].update(disabled = False) + print("Enabled") + else: + window[self.gui_restart_calibration].update(disabled = True) + window[self.gui_stop_calibration].update(disabled = True) + print("Disabled") + if event == "{}+UP".format(self.gui_roi_selection): # Event for mouse button up in ROI mode self.is_mouse_up = True @@ -328,27 +328,6 @@ def render(self, window, event, values): self.config.roi_window_h = abs(self.y0 - self.y1) self.main_config.save() - if event == self.gui_autoroi: - print("Auto ROI") - #image = self.image_queue.get() - #image = self.babble_landmark.get_frame() # Get image for pfld - #print(image) - #print(len(image)) - #print(image) - #cv2.imwrite("yeah.png", image) - self.babble_landmark.infer_frame() - output = self.babble_landmark.output - print(f"Output: {output}") - self.x1 = output[2] - self.y1 = output[3] - self.x0 = output[0] - self.y0 = output[1] - self.config.roi_window_x = min([self.x0, self.x1]) - self.config.roi_window_y = min([self.y0, self.y1]) - self.config.roi_window_w = abs(self.x0 - self.x1) - self.config.roi_window_h = abs(self.y0 - self.y1) - self.main_config.save() - if event == self.gui_roi_selection: # Event for mouse button down or mouse drag in ROI mode if self.is_mouse_up: @@ -356,12 +335,30 @@ def render(self, window, event, values): self.x0, self.y0 = values[self.gui_roi_selection] self.x1, self.y1 = values[self.gui_roi_selection] + if event == self.gui_autoroi: + print("Set ROI") + output = self.babble_cnn.get_framesize() + self.config.roi_window_x = 0 + self.config.roi_window_y = 0 + self.config.roi_window_w = output[0] + self.config.roi_window_h = output[1] + self.main_config.save() + + if (event == self.gui_refresh_button): + print("\033[94m[INFO] Refreshed Camera List\033[0m") + self.camera_list = list_camera_names() + print(self.camera_list) + window[self.gui_camera_addr].update(values=self.camera_list) + + if event == self.gui_restart_calibration: - self.babble_cnn.calibration_frame_counter = 1500 - PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) + if values[self.use_calibration] == True: # Don't start recording if the calibration filter is disabled. + self.babble_cnn.calibration_frame_counter = 1500 + PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) if event == self.gui_stop_calibration: - self.babble_cnn.calibration_frame_counter = 0 + if self.babble_cnn.calibration_frame_counter != None: # Only assign the variable if we are in calibration mode. + self.babble_cnn.calibration_frame_counter = 0 needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 @@ -390,6 +387,7 @@ def render(self, window, event, values): if self.roi_queue.empty(): self.capture_event.set() maybe_image = self.roi_queue.get(block=False) + self.maybe_image = maybe_image imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes() graph = window[self.gui_roi_selection] if self.figure: @@ -419,4 +417,4 @@ def render(self, window, event, values): if cam_info.info_type != CamInfoOrigin.FAILURE: self.osc_queue.put((self.cam_id, cam_info)) except Empty: - pass + pass \ No newline at end of file diff --git a/BabbleApp/config.py b/BabbleApp/config.py index 6b66051..2b740b0 100644 --- a/BabbleApp/config.py +++ b/BabbleApp/config.py @@ -16,8 +16,6 @@ class BabbleCameraConfig(BaseModel): roi_window_w: int = 0 roi_window_h: int = 0 capture_source: Union[int, str, None] = None - use_calibration: bool = False - use_n_calibration: bool = True gui_vertical_flip: bool = False gui_horizontal_flip: bool = False use_ffmpeg: bool = False @@ -44,6 +42,8 @@ class BabbleSettingsConfig(BaseModel): gui_cam_resolution_x: int = 0 gui_cam_resolution_y: int = 0 gui_cam_framerate: int = 0 + use_calibration: bool = False + calibration_mode: str = 'Neutral' class BabbleConfig(BaseModel): version: int = 1 @@ -87,5 +87,5 @@ def save(self): # No backup because the saved settings file is broken. pass with open(CONFIG_FILE_NAME, "w") as settings_file: - json.dump(obj=self.dict(), fp=settings_file) + json.dump(obj=self.dict(), fp=settings_file, indent=2) print("[INFO] Config Saved Successfully") diff --git a/BabbleApp/general_settings_widget.py b/BabbleApp/general_settings_widget.py index b7714e5..c2e8e6f 100644 --- a/BabbleApp/general_settings_widget.py +++ b/BabbleApp/general_settings_widget.py @@ -225,17 +225,23 @@ def render(self, window, event, values): self.config.gui_osc_location = values[self.gui_osc_location] changed = True if values[self.gui_cam_resolution_x] != '': - if self.config.gui_cam_resolution_x != int(values[self.gui_cam_resolution_x]): - self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x]) - changed = True + if str(self.config.gui_cam_resolution_x) != values[self.gui_cam_resolution_x]: + try: + self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x]) + changed = True + except: print("Not an Int") if values[self.gui_cam_resolution_y] != '': - if self.config.gui_cam_resolution_y != int(values[self.gui_cam_resolution_y]): - self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y]) - changed = True + if str(self.config.gui_cam_resolution_y) != values[self.gui_cam_resolution_y]: + try: + self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y]) + changed = True + except: print("Not an Int") if values[self.gui_cam_framerate] != '': - if self.config.gui_cam_framerate != int(values[self.gui_cam_framerate]): - self.config.gui_cam_framerate = int(values[self.gui_cam_framerate]) - changed = True + if str(self.config.gui_cam_framerate) != values[self.gui_cam_framerate]: + try: + self.config.gui_cam_framerate = int(values[self.gui_cam_framerate]) + changed = True + except: print("Not an Int") if self.config.gui_use_red_channel != bool(values[self.gui_use_red_channel]): self.config.gui_use_red_channel = bool(values[self.gui_use_red_channel]) diff --git a/BabbleApp/landmark_model_loader.py b/BabbleApp/landmark_model_loader.py index e391f37..1da5f19 100644 --- a/BabbleApp/landmark_model_loader.py +++ b/BabbleApp/landmark_model_loader.py @@ -28,8 +28,8 @@ def run_model(self): # Replace transforms n shit for the pfld model self.output = output -def write_image(self): # Debug function for development, remove once pfld is implemented. - frame = cv2.resize(self.current_image_gray, (256, 256)) - cv2.imwrite("yeah.png", frame) +def write_image(self, image): # Placeholder function for development, replace all instances with run_model() once pfld is implemented. + frame = cv2.resize(image, (256, 256)) + cv2.imwrite("yeah.png", frame) # Write the clean, untransformed frame to show that the frame is clean for pfld print("Image Wrote") - self.output = (0, 0, 100, 100) \ No newline at end of file + self.output = ((0, 0, 100, 100), 25) # Return ROI box and Rotation information. \ No newline at end of file diff --git a/BabbleApp/landmark_processor.py b/BabbleApp/landmark_processor.py index 14568a1..d642a47 100644 --- a/BabbleApp/landmark_processor.py +++ b/BabbleApp/landmark_processor.py @@ -115,8 +115,8 @@ def __init__( def get_frame(self): return self.current_image_gray_clean - def infer_frame(self): - return write_image(self) + def infer_frame(self, image): + return write_image(self, image) def output_images_and_update(self, output_information: CamInfo): try: diff --git a/BabbleApp/osc.py b/BabbleApp/osc.py index 96bef21..70351c7 100644 --- a/BabbleApp/osc.py +++ b/BabbleApp/osc.py @@ -5,12 +5,14 @@ from enum import IntEnum import time from config import BabbleConfig +import traceback import math class Tab(IntEnum): CAM = 0 SETTINGS = 1 ALGOSETTINGS = 2 + CALIBRATION = 3 import numpy as np @@ -98,6 +100,7 @@ def __init__(self, cancellation_event: threading.Event, main_config: BabbleConfi self.cancellation_event = cancellation_event self.dispatcher = dispatcher.Dispatcher() self.cams = cams # we cant import CameraWidget so any type it is + print("OSC_INIT") try: self.server = osc_server.OSCUDPServer((self.config.gui_osc_address, int(self.config.gui_osc_receiver_port)), self.dispatcher) except: @@ -114,17 +117,18 @@ def recalibrate_mouth(self, address, osc_value): if type(osc_value) != bool: return # just incase we get anything other than bool if osc_value: for cam in self.cams: - cam.ransac.calibration_frame_counter = 300 + cam.babble_cnn.calibration_frame_counter = 300 PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) def run(self): # bind what function to run when specified OSC message is received try: - self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_cams) + self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_mouth) # start the server print("\033[92m[INFO] VRChatOSCReceiver serving on {}\033[0m".format(self.server.server_address)) self.server.serve_forever() except: + traceback.print_exc() print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m") \ No newline at end of file diff --git a/BabbleApp/osc_calibrate_filter.py b/BabbleApp/osc_calibrate_filter.py index 195e9a3..714eaa3 100644 --- a/BabbleApp/osc_calibrate_filter.py +++ b/BabbleApp/osc_calibrate_filter.py @@ -9,10 +9,15 @@ class CamId(IntEnum): class cal(): + def __init__(self): + self.calibrated_array = None + self.raw_array = None + def cal_osc(self, array): + self.raw_array = array #print(self.calibration_frame_counter) if self.calibration_frame_counter == 0: - if not self.config.use_n_calibration: + if self.settings.calibration_mode == 'Full': self.calibration_frame_counter = None values = np.array(self.val_list) @@ -42,19 +47,27 @@ def cal_osc(self, array): PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC) - if self.config.use_n_calibration: + if self.settings.calibration_mode == 'Neutral': + self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',') + self.min_max_array = self.min_max_array.reshape((2, 45)) + print(f'minmax {self.min_max_array}') + max_values = self.min_max_array[1] self.calibration_frame_counter = None values = np.array(self.val_list) + #print(f'values: {values}') deadzone_value = self.settings.calib_deadzone # Initialize the min_max_array with shape (2, num_outputs) num_outputs = values.shape[1] self.min_max_array = np.zeros((2, num_outputs)) - lower_threshold = np.clip([np.mean(values, axis = 0) + deadzone_value], 0, 1) - upper_threshold = np.ones((1, num_outputs)) # We don't need to adjust the max values. + lower_threshold = np.clip([np.mean(values, axis = 0) + deadzone_value], 0, 1).tolist()[0] + upper_threshold = max_values.tolist() + #upper_threshold = np.ones((1, num_outputs)) # We don't need to adjust the max values. + #upper_threshold = self.min_max_array[1] print(lower_threshold) print(upper_threshold) - self.min_max_array = np.array([lower_threshold, upper_threshold]) - self.settings.calib_array = np.array2string(self.min_max_array, separator=',') + self.min_max_array = np.append(lower_threshold, upper_threshold) + #self.min_max_array = np.array([lower_threshold.tolist(), upper_threshold.tolist()]) + self.settings.calib_array = str([lower_threshold, upper_threshold]) self.config_class.save() print("[INFO] Calibration completed.") @@ -67,8 +80,8 @@ def cal_osc(self, array): self.val_list.append(array) self.calibration_frame_counter -= 1 - - if self.settings.calib_array is not None and self.config.use_calibration: + self.raw_array = array + if self.settings.calib_array is not None and self.settings.use_calibration and self.settings.calibration_mode == 'Full': self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',') self.min_max_array = self.min_max_array.reshape((2, 45)) @@ -85,17 +98,22 @@ def cal_osc(self, array): calibrated_array[i] = calibrated_value array = calibrated_array - if self.settings.calib_array is not None and self.config.use_n_calibration: + if self.settings.calib_array is not None and self.settings.use_calibration and self.settings.calibration_mode == 'Neutral': self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',') self.min_max_array = self.min_max_array.reshape((2, 45)) calibrated_array = np.zeros_like(array) for i, value in enumerate(array): min_value = self.min_max_array[0, i] + max_value = self.min_max_array[1, i] - calibrated_value = (value - min_value) / (1.0 - min_value) + calibrated_value = (value - min_value) / (max_value - min_value) calibrated_array[i] = calibrated_value array = calibrated_array #array[4] = log((np.clip(array[4]*10,0,10))+1.0, 11) Log Filter: Move to filter system. - return np.clip(array,0,1) # Clamp outputs between 0-1 + array = np.clip(array,0,1) + return array # Clamp outputs between 0-1 + + def get_outputs(self): + return self.calibrated_array, self.raw_array diff --git a/BabbleApp/tab.py b/BabbleApp/tab.py index e57d414..21074d3 100644 --- a/BabbleApp/tab.py +++ b/BabbleApp/tab.py @@ -5,6 +5,7 @@ class Tab(IntEnum): CAM = 0 SETTINGS = 1 ALGOSETTINGS = 2 + CALIBRATION = 3 class CamInfoOrigin(Enum): diff --git a/BabbleApp/utils/misc_utils.py b/BabbleApp/utils/misc_utils.py index 4709a79..a69823b 100644 --- a/BabbleApp/utils/misc_utils.py +++ b/BabbleApp/utils/misc_utils.py @@ -1,7 +1,24 @@ +import typing +import serial +import sys +import glob import os import platform import cv2 import subprocess +from pygrabber.dshow_graph import FilterGraph + +is_nt = True if sys.platform.startswith('win') else False +graph = FilterGraph() + + +def list_camera_names(): + cam_list = graph.get_input_devices() + cam_names = [] + for index, name in enumerate(cam_list): + cam_names.append(name) + cam_names = cam_names + list_serial_ports() + return cam_names # Detect the operating system is_nt = True if os.name == "nt" else False @@ -73,19 +90,48 @@ def list_camera_names(): if is_nt: # On Windows, use pygrabber to list devices cam_list = graph.get_input_devices() - return cam_list + return cam_list + list_serial_ports() elif os_type == "Linux": # On Linux, return UVC device paths like '/dev/video0' - return list_linux_uvc_devices() + return list_linux_uvc_devices() + list_serial_ports() elif os_type == "Darwin": # On macOS, fallback to OpenCV (device names aren't fetched) - return list_cameras_opencv() + return list_cameras_opencv() + list_serial_ports() else: return ["Unsupported operating system"] +def list_serial_ports(): + print("DEBUG: Listed Serial Ports") + """ Lists serial port names + + :raises EnvironmentError: + On unsupported or unknown platforms + :returns: + A list of the serial ports available on the system + """ + if sys.platform.startswith('win'): + ports = ['COM%s' % (i + 1) for i in range(256)] + elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'): + # this excludes your current terminal "/dev/tty" + ports = glob.glob('/dev/tty[A-Za-z]*') + elif sys.platform.startswith('darwin'): + ports = glob.glob('/dev/tty.*') + else: + raise EnvironmentError('Unsupported platform') + + result = [] + for port in ports: + try: + s = serial.Serial(port) + s.close() + result.append(port) + except (OSError, serial.SerialException): + pass + return result + def get_camera_index_by_name(name): """ Cross-platform function to get the camera index by its name or path """ @@ -111,6 +157,10 @@ def get_camera_index_by_name(name): return None +#def get_serial_port(name): +# for i, device in enumerate(cam_list): + + # Placeholder for sound functions on Windows def PlaySound(*args, **kwargs):