Do not delay getUserMedia promise resolution on VPIO unit prewarming

https://bugs.webkit.org/show_bug.cgi?id=274203 rdar://128112002 Reviewed by Eric Carlson. Before this patch, we were delaying the resolution of the getUserMedia promise upon VPIO creation prewarming. This was ensuring that there would be no freeze and audio capture would start right away. But this delays the web page processing. We are now doing the following when starting capture: - Prewarm the VPIO unit when GPU process is instructed that it will start audio capture soon. - Create sources in GPU process, do not wait for warming up to finish.. - Resolve getUserMedia promise. - Start the sources. When starting microphone capture, complete the prewarming if needed before starting the VPIO unit. This change allows the web page to manipulate the getUserMedia tracks sooner. It can then use the tracks to set up a VC call for instance, in parallel to the prewarming of the VPIO unit. We keep prewarming of the VPIO unit as fast as we can. Manually tested on macOS, should be a no op on iOS. * Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp: (WebCore::BaseAudioSharedUnit::startProducingData): (WebCore::BaseAudioSharedUnit::prepareForNewCapture): * Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.h: * Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp: (WebCore::CoreAudioCaptureSource::whenReady): Deleted. * Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.h: * Source/WebCore/platform/mediastream/mac/CoreAudioSharedUnit.cpp: (WebCore::CoreAudioSharedUnit::prewarmAudioUnitCreation): * Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp: (WebKit::UserMediaCaptureManagerProxy::createMediaSourceForCaptureDeviceWithConstraints): Canonical link: https://commits.webkit.org/278851@main
WebKit · May 16, 2024 · 63db443 · 63db443
1 parent a6b4a4b
commit 63db443
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 28 deletions.
diff --git a/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp b/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp
@@ -93,6 +93,21 @@ void BaseAudioSharedUnit::startProducingData()
     if (++m_producingCount != 1)
         return;
 
+#if PLATFORM(MAC)
+    prewarmAudioUnitCreation([weakThis = WeakPtr { *this }] {
+        if (weakThis)
+            weakThis->continueStartProducingData();
+    });
+#else
+    continueStartProducingData();
+#endif
+}
+
+void BaseAudioSharedUnit::continueStartProducingData()
+{
+    if (!m_producingCount)
+        return;
+
     if (isProducingData())
         return;
 

diff --git a/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.h b/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.h
@@ -102,6 +102,7 @@ class BaseAudioSharedUnit : public RealtimeMediaSourceCenterObserver {
 protected:
     void forEachClient(const Function<void(CoreAudioCaptureSource&)>&) const;
     void captureFailed();
+    void continueStartProducingData();
 
     virtual void cleanupAudioUnit() = 0;
     virtual OSStatus startInternal() = 0;

diff --git a/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp b/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp
@@ -343,15 +343,6 @@ void CoreAudioCaptureSource::setIsInBackground(bool value)
 }
 #endif
 
-#if PLATFORM(MAC)
-void CoreAudioCaptureSource::whenReady(CompletionHandler<void(CaptureSourceError&&)>&& callback)
-{
-    unit().prewarmAudioUnitCreation([callback = WTFMove(callback)] () mutable {
-        callback({ });
-    });
-}
-#endif
-
 void CoreAudioCaptureSource::audioUnitWillStart()
 {
     forEachObserver([](auto& observer) {

diff --git a/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.h b/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.h
@@ -88,9 +88,6 @@ class CoreAudioCaptureSource : public RealtimeMediaSource, public ThreadSafeRefC
 #if PLATFORM(IOS_FAMILY)
     void setIsInBackground(bool) final;
 #endif
-#if PLATFORM(MAC)
-    void whenReady(CompletionHandler<void(CaptureSourceError&&)>&&) final;
-#endif
 
     std::optional<Vector<int>> discreteSampleRates() const final { return { { 8000, 16000, 32000, 44100, 48000, 96000 } }; }
 

diff --git a/Source/WebCore/platform/mediastream/mac/CoreAudioSharedUnit.cpp b/Source/WebCore/platform/mediastream/mac/CoreAudioSharedUnit.cpp
@@ -713,17 +713,23 @@ bool CoreAudioSharedUnit::migrateToNewDefaultDevice(const CaptureDevice& capture
 
 void CoreAudioSharedUnit::prewarmAudioUnitCreation(CompletionHandler<void()>&& callback)
 {
-    if (!m_audioUnitCreationWarmupPromise) {
-        m_audioUnitCreationWarmupPromise = invokeAsync(WorkQueue::create("CoreAudioSharedUnit AudioUnit creation"_s).get(), [] {
-            return createAudioUnit(true);
-        })->whenSettled(RunLoop::main(), [weakThis = WeakPtr { *this }] (auto&& vpioUnitOrError) {
-            if (weakThis && vpioUnitOrError.has_value())
-                weakThis->setStoredVPIOUnit(WTFMove(vpioUnitOrError.value()));
-            return GenericNonExclusivePromise::createAndResolve();
-        });
+    if (m_audioUnitCreationWarmupPromise) {
+        m_audioUnitCreationWarmupPromise->whenSettled(RunLoop::main(), WTFMove(callback));
+        return;
     }
 
-    m_audioUnitCreationWarmupPromise->whenSettled(RunLoop::main(), WTFMove(callback));
+    if (!enableEchoCancellation()) {
+        callback();
+        return;
+    }
+
+    m_audioUnitCreationWarmupPromise = invokeAsync(WorkQueue::create("CoreAudioSharedUnit AudioUnit creation"_s).get(), [] {
+        return createAudioUnit(true);
+    })->whenSettled(RunLoop::main(), [weakThis = WeakPtr { *this }] (auto&& vpioUnitOrError) {
+        if (weakThis && vpioUnitOrError.has_value())
+            weakThis->setStoredVPIOUnit(WTFMove(vpioUnitOrError.value()));
+        return GenericNonExclusivePromise::createAndResolve();
+    });
 }
 
 void CoreAudioSharedUnit::deallocateStoredVPIOUnit()

diff --git a/Source/WebKit/GPUProcess/GPUConnectionToWebProcess.cpp b/Source/WebKit/GPUProcess/GPUConnectionToWebProcess.cpp
@@ -153,6 +153,7 @@
 #endif
 
 #if ENABLE(MEDIA_STREAM)
+#include <WebCore/CoreAudioSharedUnit.h>
 #include <WebCore/SecurityOrigin.h>
 #endif
 
@@ -1107,6 +1108,11 @@ void GPUConnectionToWebProcess::setOrientationForMediaCapture(IntDegrees orienta
 
 void GPUConnectionToWebProcess::updateCaptureAccess(bool allowAudioCapture, bool allowVideoCapture, bool allowDisplayCapture)
 {
+#if PLATFORM(MAC) && ENABLE(MEDIA_STREAM)
+    if (allowAudioCapture)
+        CoreAudioSharedUnit::singleton().prewarmAudioUnitCreation([] { });
+#endif
+
     m_allowsAudioCapture |= allowAudioCapture;
     m_allowsVideoCapture |= allowVideoCapture;
     m_allowsDisplayCapture |= allowDisplayCapture;

diff --git a/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp b/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp
@@ -552,13 +552,7 @@ void UserMediaCaptureManagerProxy::createMediaSourceForCaptureDeviceWithConstrai
             return;
         }
 
-        proxy->source().whenReady([completionHandler = WTFMove(completionHandler), proxy = WeakPtr { *proxy }] (auto&& error) mutable {
-            if (!!error || !proxy) {
-                completionHandler(error, { }, { });
-                return;
-            }
-            completionHandler({ }, proxy->settings(), proxy->source().capabilities());
-        });
+        completionHandler({ }, proxy->settings(), proxy->source().capabilities());
         m_proxies.add(id, WTFMove(proxy));
     };