From 20f3203f396e442e39798e5ef3cc4d0d6994fecb Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 21:01:24 -0700 Subject: [PATCH 01/10] (Java) Add support for resolution alignment during encoding --- sdk/android/BUILD.gn | 4 + .../DefaultAlignedVideoEncoderFactory.java | 68 +++++ .../webrtc/HardwareVideoEncoderWrapper.java | 255 ++++++++++++++++++ .../api/org/webrtc/ResolutionAdjustment.java | 39 +++ .../SimulcastAlignedVideoEncoderFactory.java | 189 +++++++++++++ 5 files changed, 555 insertions(+) create mode 100644 sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java create mode 100644 sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java create mode 100644 sdk/android/api/org/webrtc/ResolutionAdjustment.java create mode 100644 sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 9d82264d79..b2aea680c0 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -529,6 +529,10 @@ if (is_android) { "api/org/webrtc/SoftwareVideoEncoderFactory.java", "api/org/webrtc/SimulcastVideoEncoder.java", "api/org/webrtc/SimulcastVideoEncoderFactory.java", + "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", + "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", + "api/org/webrtc/HardwareVideoEncoderWrapper.java", + "api/org/webrtc/ResolutionAdjustment.java", ] deps = [ diff --git a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java new file mode 100644 index 0000000000..2ee9d496dd --- /dev/null +++ b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +import java.util.LinkedHashSet; + +/** + * The main difference with the standard [DefaultAlignedVideoEncoderFactory] is that this fixes + * issues with resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting [resolutionAdjustment]. Internally the resolution during streaming + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard [DefaultVideoEncoderFactory] and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnco + * derFactoryWrapper.kt#L18 + */ +public class DefaultAlignedVideoEncoderFactory implements VideoEncoderFactory { + private final VideoEncoderFactory hardwareVideoEncoderFactory; + private final VideoEncoderFactory softwareVideoEncoderFactory; + + public DefaultAlignedVideoEncoderFactory( + EglBase.Context eglContext, + boolean enableIntelVp8Encoder, + boolean enableH264HighProfile, + ResolutionAdjustment resolutionAdjustment + ) { + VideoEncoderFactory defaultFactory = + new HardwareVideoEncoderFactory(eglContext, enableIntelVp8Encoder, enableH264HighProfile); + hardwareVideoEncoderFactory = (resolutionAdjustment == ResolutionAdjustment.NONE) ? + defaultFactory : + new HardwareVideoEncoderWrapperFactory(defaultFactory, resolutionAdjustment.getValue()); + softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory(); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + VideoEncoder softwareEncoder = softwareVideoEncoderFactory.createEncoder(info); + VideoEncoder hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info); + if (hardwareEncoder != null && softwareEncoder != null) { + return new VideoEncoderFallback(softwareEncoder, hardwareEncoder); + } + return hardwareEncoder != null ? hardwareEncoder : softwareEncoder; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + LinkedHashSet supportedCodecInfos = new LinkedHashSet<>(); + supportedCodecInfos.addAll(Arrays.asList(softwareVideoEncoderFactory.getSupportedCodecs())); + supportedCodecInfos.addAll(Arrays.asList(hardwareVideoEncoderFactory.getSupportedCodecs())); + return supportedCodecInfos.toArray(new VideoCodecInfo[0]); + } +} + diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java new file mode 100644 index 0000000000..707d528e04 --- /dev/null +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207 + * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco + * derWrapperFactory.kt + */ +class HardwareVideoEncoderWrapper implements VideoEncoder { + + private static final String TAG = "HardwareVideoEncoderWrapper"; + + private final VideoEncoder internalEncoder; + private final int alignment; + + public HardwareVideoEncoderWrapper(VideoEncoder internalEncoder, int alignment) { + this.internalEncoder = internalEncoder; + this.alignment = alignment; + } + + private static class CropSizeCalculator { + + private static final String TAG = "CropSizeCalculator"; + + private final int alignment; + private final int originalWidth; + private final int originalHeight; + private final int cropX; + private final int cropY; + + public CropSizeCalculator(int alignment, int originalWidth, int originalHeight) { + this.alignment = alignment; + this.originalWidth = originalWidth; + this.originalHeight = originalHeight; + this.cropX = originalWidth % alignment; + this.cropY = originalHeight % alignment; + if (originalWidth != 0 && originalHeight != 0) { + Logging.v(TAG + " init(): alignment=" + alignment + + " size=" + originalWidth + "x" + originalHeight + " => " + croppedWidth + "x" + croppedHeight); + } + } + + public int getCroppedWidth() { + return originalWidth - cropX; + } + + public int getCroppedHeight() { + return originalHeight - cropY; + } + + public boolean isCropRequired() { + return cropX != 0 || cropY != 0; + } + + public boolean hasFrameSizeChanged(int nextWidth, int nextHeight) { + if (originalWidth == nextWidth && originalHeight == nextHeight) { + return false; + } else { + Logging.v(TAG + " frame size has changed: " + + originalWidth + "x" + originalHeight + " => " + nextWidth + "x" + nextHeight); + return true; + } + return originalWidth != nextWidth || originalHeight != nextHeight; + } + } + + private CropSizeCalculator calculator = new CropSizeCalculator(1, 0, 0); + + private VideoCodecStatus retryWithoutCropping(int width, int height, Runnable retryFunc) { + Logging.v(TAG, "retrying without resolution adjustment"); + calculator = new CropSizeCalculator(1, width, height); + retryFunc.run(); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus initEncode(VideoEncoder.Settings originalSettings, VideoEncoder.Callback callback) { + calculator = new CropSizeCalculator(alignment, originalSettings.width, originalSettings.height); + if (!calculator.isCropRequired()) { + return internalEncoder.initEncode(originalSettings, callback); + } else { + VideoEncoder.Settings croppedSettings = new VideoEncoder.Settings( + originalSettings.numberOfCores, + calculator.getCroppedWidth(), + calculator.getCroppedHeight(), + originalSettings.startBitrate, + originalSettings.maxFramerate, + originalSettings.numberOfSimulcastStreams, + originalSettings.automaticResizeOn, + originalSettings.capabilities + ); + try { + VideoCodecStatus result = internalEncoder.initEncode(croppedSettings, callback); + if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e(TAG, "internalEncoder.initEncode() returned FALLBACK_SOFTWARE: " + + "croppedSettings " + croppedSettings); + return retryWithoutCropping( + originalSettings.width, + originalSettings.height, + () -> internalEncoder.initEncode(originalSettings, callback) + ); + } else { + return result; + } + } catch (Exception e) { + Logging.e(TAG, "internalEncoder.initEncode() failed", e); + return retryWithoutCropping( + originalSettings.width, + originalSettings.height, + () -> internalEncoder.initEncode(originalSettings, callback) + ); + } + } + } + + @Override + public VideoCodecStatus release() { + return internalEncoder.release(); + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) { + if (calculator.hasFrameSizeChanged(frame.getBuffer().getWidth(), frame.getBuffer().getHeight())) { + calculator = new CropSizeCalculator(alignment, frame.getBuffer().getWidth(), frame.getBuffer().getHeight()); + } + if (!calculator.isCropRequired()) { + return internalEncoder.encode(frame, encodeInfo); + } else { + int croppedWidth = calculator.getCroppedWidth(); + int croppedHeight = calculator.getCroppedHeight(); + VideoFrame.Buffer croppedBuffer = frame.getBuffer().cropAndScale( + calculator.cropX / 2, + calculator.cropY / 2, + croppedWidth, + croppedHeight, + croppedWidth, + croppedHeight + ); + VideoFrame croppedFrame = new VideoFrame(croppedBuffer, frame.getRotation(), frame.getTimestampNs()); + try { + VideoCodecStatus result = internalEncoder.encode(croppedFrame, encodeInfo); + if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e(TAG, "internalEncoder.encode() returned FALLBACK_SOFTWARE"); + return retryWithoutCropping( + frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), + () -> internalEncoder.encode(frame, encodeInfo) + ); + } else { + return result; + } + } catch (Exception e) { + Logging.e(TAG, "internalEncoder.encode() failed", e); + return retryWithoutCropping( + frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), + () -> internalEncoder.encode(frame, encodeInfo) + ); + } finally { + croppedBuffer.release(); + } + } + } + + @Override + public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) { + return internalEncoder.setRateAllocation(allocation, frameRate); + } + + @Override + public VideoEncoder.ScalingSettings getScalingSettings() { + return internalEncoder.getScalingSettings(); + } + + @Override + public String getImplementationName() { + return internalEncoder.getImplementationName(); + } + + @Override + public long createNativeVideoEncoder() { + return internalEncoder.createNativeVideoEncoder(); + } + + @Override + public boolean isHardwareEncoder() { + return internalEncoder.isHardwareEncoder(); + } + + @Override + public VideoCodecStatus setRates(VideoEncoder.RateControlParameters rcParameters) { + return internalEncoder.setRates(rcParameters); + } + + @Override + public VideoEncoder.ResolutionBitrateLimits[] getResolutionBitrateLimits() { + return internalEncoder.getResolutionBitrateLimits(); + } + + @Override + public VideoEncoder.EncoderInfo getEncoderInfo() { + return internalEncoder.getEncoderInfo(); + } +} + +class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory { + + private static final String TAG = "HardwareVideoEncoderWrapperFactory"; + + private final HardwareVideoEncoderFactory factory; + private final int resolutionPixelAlignment; + + public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) { + this.factory = factory; + this.resolutionPixelAlignment = resolutionPixelAlignment; + if (resolutionPixelAlignment == 0) { + throw new IllegalArgumentException("resolutionPixelAlignment should not be 0"); + } + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { + try { + VideoEncoder encoder = factory.createEncoder(videoCodecInfo); + if (encoder == null) { + return null; + } + return new HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment); + } catch (Exception e) { + Logging.e(TAG, "createEncoder failed", e); + return null; + } + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return factory.getSupportedCodecs(); + } +} + + diff --git a/sdk/android/api/org/webrtc/ResolutionAdjustment.java b/sdk/android/api/org/webrtc/ResolutionAdjustment.java new file mode 100644 index 0000000000..0ed741db74 --- /dev/null +++ b/sdk/android/api/org/webrtc/ResolutionAdjustment.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Resolution alignment values. Generally the MULTIPLE_OF_16 is recommended + * for both VP8 and H264 + */ +public enum ResolutionAdjustment { + NONE(1), + MULTIPLE_OF_2(2), + MULTIPLE_OF_4(4), + MULTIPLE_OF_8(8), + MULTIPLE_OF_16(16); + + private final int value; + + private ResolutionAdjustment(int value) { + this.value = value; + } + + public int getValue() { + return value; + } +} + diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java new file mode 100644 index 0000000000..3cf14ac9c0 --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * The main difference with the standard SimulcastVideoEncoderFactory is that this fixes issues + * with simulcasting resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting resolutionAdjustment. Internally the resolutions during simulcast + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard SimulcastVideoEncoderFactory and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnc + * oderFactoryWrapper.kt#L18 + */ +public class SimulcastAlignedVideoEncoderFactory implements VideoEncoderFactory { + private static class StreamEncoderWrapper implements VideoEncoder { + private static final String TAG = StreamEncoderWrapper.class.getSimpleName(); + private final VideoEncoder encoder; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private VideoEncoder.Settings streamSettings; + + public StreamEncoderWrapper(VideoEncoder encoder) { + this.encoder = encoder; + } + + @Override + public VideoCodecStatus initEncode(VideoEncoder.Settings settings, VideoEncoder.Callback callback) { + streamSettings = settings; + Callable callable = () -> { + Logging.v(TAG, "initEncode() thread=" + Thread.currentThread().getName() + " [" + Thread.currentThread().getId() + "]"); + Logging.v(TAG, " encoder=" + encoder.getImplementationName()); + Logging.v(TAG, " streamSettings:"); + Logging.v(TAG, " numberOfCores=" + settings.numberOfCores); + Logging.v(TAG, " width=" + settings.width); + Logging.v(TAG, " height=" + settings.height); + Logging.v(TAG, " startBitrate=" + settings.startBitrate); + Logging.v(TAG, " maxFramerate=" + settings.maxFramerate); + Logging.v(TAG, " automaticResizeOn=" + settings.automaticResizeOn); + Logging.v(TAG, " numberOfSimulcastStreams=" + settings.numberOfSimulcastStreams); + Logging.v(TAG, " lossNotification=" + settings.capabilities.lossNotification); + return encoder.initEncode(settings, callback); + }; + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus release() { + Callable callable = () -> encoder.release(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) { + Callable callable = () -> { + if (streamSettings != null) { + if (frame.getBuffer().getWidth() == streamSettings.width) { + return encoder.encode(frame, encodeInfo); + } else { + int originalWidth = frame.getBuffer().getWidth(); + int originalHeight = frame.getBuffer().getHeight(); + VideoFrame.Buffer scaledBuffer = frame.getBuffer().cropAndScale( + 0, 0, originalWidth, originalHeight, + streamSettings.width, streamSettings.height + ); + VideoFrame scaledFrame = new VideoFrame(scaledBuffer, frame.getRotation(), frame.getTimestampNs()); + VideoCodecStatus result = encoder.encode(scaledFrame, encodeInfo); + scaledBuffer.release(); + return result; + } + } else { + return VideoCodecStatus.ERROR; + } + }; + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) { + Callable callable = () -> encoder.setRateAllocation(allocation, frameRate); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoEncoder.ScalingSettings getScalingSettings() { + Callable callable = () -> encoder.getScalingSettings(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return null; + } + } + + @Override + public String getImplementationName() { + Callable callable = () -> encoder.getImplementationName(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return null; + } + } + } + + private static class StreamEncoderWrapperFactory implements VideoEncoderFactory { + private final VideoEncoderFactory factory; + + public StreamEncoderWrapperFactory(VideoEncoderFactory factory) { + this.factory = factory; + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { + VideoEncoder encoder = factory.createEncoder(videoCodecInfo); + if (encoder == null) { + return null; + } + return new StreamEncoderWrapper(encoder); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return factory.getSupportedCodecs(); + } + } + + private final VideoEncoderFactory primary; + private final VideoEncoderFactory fallback; + private final SimulcastVideoEncoderFactory native; + + public SimulcastAlignedVideoEncoderFactory(EglBase.Context sharedContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile, ResolutionAdjustment resolutionAdjustment) { + HardwareVideoEncoderFactory hardwareVideoEncoderFactory = new HardwareVideoEncoderFactory(sharedContext, enableIntelVp8Encoder, enableH264HighProfile); + VideoEncoderFactory encoderFactory; + if (resolutionAdjustment == ResolutionAdjustment.NONE) { + encoderFactory = hardwareVideoEncoderFactory; + } else { + encoderFactory = new HardwareVideoEncoderWrapperFactory(hardwareVideoEncoderFactory, resolutionAdjustment.getValue()); + } + primary = new StreamEncoderWrapperFactory(encoderFactory); + fallback = new SoftwareVideoEncoderFactory(); + native = new SimulcastVideoEncoderFactory(primary, fallback); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + return native.createEncoder(info); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return native.getSupportedCodecs(); + } +} + + From 2d52d5718542fa23f9504e00d9c5d32ce9df912f Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 21:21:46 -0700 Subject: [PATCH 02/10] rename native to delegate --- .../org/webrtc/SimulcastAlignedVideoEncoderFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java index 3cf14ac9c0..ac1e34c607 100644 --- a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java @@ -160,7 +160,7 @@ public VideoCodecInfo[] getSupportedCodecs() { private final VideoEncoderFactory primary; private final VideoEncoderFactory fallback; - private final SimulcastVideoEncoderFactory native; + private final SimulcastVideoEncoderFactory delegate; public SimulcastAlignedVideoEncoderFactory(EglBase.Context sharedContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile, ResolutionAdjustment resolutionAdjustment) { HardwareVideoEncoderFactory hardwareVideoEncoderFactory = new HardwareVideoEncoderFactory(sharedContext, enableIntelVp8Encoder, enableH264HighProfile); @@ -172,17 +172,17 @@ public SimulcastAlignedVideoEncoderFactory(EglBase.Context sharedContext, boolea } primary = new StreamEncoderWrapperFactory(encoderFactory); fallback = new SoftwareVideoEncoderFactory(); - native = new SimulcastVideoEncoderFactory(primary, fallback); + delegate = new SimulcastVideoEncoderFactory(primary, fallback); } @Override public VideoEncoder createEncoder(VideoCodecInfo info) { - return native.createEncoder(info); + return delegate.createEncoder(info); } @Override public VideoCodecInfo[] getSupportedCodecs() { - return native.getSupportedCodecs(); + return delegate.getSupportedCodecs(); } } From e5001d8121d8d5686040bf79e3fa4d377f5d77e2 Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 21:51:41 -0700 Subject: [PATCH 03/10] extract HardwareVideoEncoderWrapperFactory --- sdk/android/BUILD.gn | 1 + .../webrtc/HardwareVideoEncoderWrapper.java | 35 ------------ .../HardwareVideoEncoderWrapperFactory.java | 56 +++++++++++++++++++ 3 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index b2aea680c0..eb0aa958ae 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -531,6 +531,7 @@ if (is_android) { "api/org/webrtc/SimulcastVideoEncoderFactory.java", "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", + "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java", "api/org/webrtc/HardwareVideoEncoderWrapper.java", "api/org/webrtc/ResolutionAdjustment.java", ] diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java index 707d528e04..f34eb7c924 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java @@ -217,39 +217,4 @@ public VideoEncoder.EncoderInfo getEncoderInfo() { } } -class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory { - - private static final String TAG = "HardwareVideoEncoderWrapperFactory"; - - private final HardwareVideoEncoderFactory factory; - private final int resolutionPixelAlignment; - - public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) { - this.factory = factory; - this.resolutionPixelAlignment = resolutionPixelAlignment; - if (resolutionPixelAlignment == 0) { - throw new IllegalArgumentException("resolutionPixelAlignment should not be 0"); - } - } - - @Override - public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { - try { - VideoEncoder encoder = factory.createEncoder(videoCodecInfo); - if (encoder == null) { - return null; - } - return new HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment); - } catch (Exception e) { - Logging.e(TAG, "createEncoder failed", e); - return null; - } - } - - @Override - public VideoCodecInfo[] getSupportedCodecs() { - return factory.getSupportedCodecs(); - } -} - diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java new file mode 100644 index 0000000000..2b5b1c8f7f --- /dev/null +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207 + * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco + * derWrapperFactory.kt + */ +class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory { + + private static final String TAG = "HardwareVideoEncoderWrapperFactory"; + + private final HardwareVideoEncoderFactory factory; + private final int resolutionPixelAlignment; + + public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) { + this.factory = factory; + this.resolutionPixelAlignment = resolutionPixelAlignment; + if (resolutionPixelAlignment == 0) { + throw new IllegalArgumentException("resolutionPixelAlignment should not be 0"); + } + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { + try { + VideoEncoder encoder = factory.createEncoder(videoCodecInfo); + if (encoder == null) { + return null; + } + return new HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment); + } catch (Exception e) { + Logging.e(TAG, "createEncoder failed", e); + return null; + } + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return factory.getSupportedCodecs(); + } +} \ No newline at end of file From 407155e49a7532425466cf687ee6d0e4138b0ae7 Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 22:10:09 -0700 Subject: [PATCH 04/10] change class order --- sdk/android/BUILD.gn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index eb0aa958ae..d52d64b4cf 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -529,11 +529,11 @@ if (is_android) { "api/org/webrtc/SoftwareVideoEncoderFactory.java", "api/org/webrtc/SimulcastVideoEncoder.java", "api/org/webrtc/SimulcastVideoEncoderFactory.java", - "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", - "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", - "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java", - "api/org/webrtc/HardwareVideoEncoderWrapper.java", "api/org/webrtc/ResolutionAdjustment.java", + "api/org/webrtc/HardwareVideoEncoderWrapper.java", + "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java", + "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", + "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", ] deps = [ From da7040bbda97649606d3b124ecd5e948b777a926 Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 22:29:16 -0700 Subject: [PATCH 05/10] add missing import --- .../api/org/webrtc/DefaultAlignedVideoEncoderFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java index 2ee9d496dd..65720ef6a0 100644 --- a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java @@ -15,6 +15,7 @@ */ package org.webrtc; +import java.util.Arrays; import java.util.LinkedHashSet; /** @@ -39,7 +40,7 @@ public DefaultAlignedVideoEncoderFactory( boolean enableH264HighProfile, ResolutionAdjustment resolutionAdjustment ) { - VideoEncoderFactory defaultFactory = + HardwareVideoEncoderFactory defaultFactory = new HardwareVideoEncoderFactory(eglContext, enableIntelVp8Encoder, enableH264HighProfile); hardwareVideoEncoderFactory = (resolutionAdjustment == ResolutionAdjustment.NONE) ? defaultFactory : From c1daa68c079bbd893cb67165bc123334c730c0fd Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 22:58:48 -0700 Subject: [PATCH 06/10] compile fixes --- sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java | 2 +- .../api/org/webrtc/HardwareVideoEncoderWrapperFactory.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java index f34eb7c924..07446b2393 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java @@ -50,7 +50,7 @@ public CropSizeCalculator(int alignment, int originalWidth, int originalHeight) this.cropY = originalHeight % alignment; if (originalWidth != 0 && originalHeight != 0) { Logging.v(TAG + " init(): alignment=" + alignment + - " size=" + originalWidth + "x" + originalHeight + " => " + croppedWidth + "x" + croppedHeight); + " size=" + originalWidth + "x" + originalHeight + " => " + getCroppedWidth() + "x" + getCroppedHeight()); } } diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java index 2b5b1c8f7f..11f7c73198 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java @@ -24,10 +24,10 @@ class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory { private static final String TAG = "HardwareVideoEncoderWrapperFactory"; - private final HardwareVideoEncoderFactory factory; + private final VideoEncoderFactory factory; private final int resolutionPixelAlignment; - public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) { + public HardwareVideoEncoderWrapperFactory(VideoEncoderFactory factory, int resolutionPixelAlignment) { this.factory = factory; this.resolutionPixelAlignment = resolutionPixelAlignment; if (resolutionPixelAlignment == 0) { From 2f2a580c8d233d0c14b4ddcd81994bc363c87d80 Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 23:14:02 -0700 Subject: [PATCH 07/10] fix logging --- sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java index 07446b2393..128e916c20 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java @@ -49,7 +49,7 @@ public CropSizeCalculator(int alignment, int originalWidth, int originalHeight) this.cropX = originalWidth % alignment; this.cropY = originalHeight % alignment; if (originalWidth != 0 && originalHeight != 0) { - Logging.v(TAG + " init(): alignment=" + alignment + + Logging.v(TAG, "init(): alignment=" + alignment + " size=" + originalWidth + "x" + originalHeight + " => " + getCroppedWidth() + "x" + getCroppedHeight()); } } @@ -70,7 +70,7 @@ public boolean hasFrameSizeChanged(int nextWidth, int nextHeight) { if (originalWidth == nextWidth && originalHeight == nextHeight) { return false; } else { - Logging.v(TAG + " frame size has changed: " + + Logging.v(TAG, "frame size has changed: " + originalWidth + "x" + originalHeight + " => " + nextWidth + "x" + nextHeight); return true; } From 41c2875bff2db5daf0c7b55a769f072e1832463c Mon Sep 17 00:00:00 2001 From: kanat Date: Mon, 25 Mar 2024 23:32:28 -0700 Subject: [PATCH 08/10] fix unreachable return statement --- sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java index 128e916c20..ac3b0c4684 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java @@ -74,7 +74,6 @@ public boolean hasFrameSizeChanged(int nextWidth, int nextHeight) { originalWidth + "x" + originalHeight + " => " + nextWidth + "x" + nextHeight); return true; } - return originalWidth != nextWidth || originalHeight != nextHeight; } } From c968c3d12aad4b17a79e5d29c219871417dc8c10 Mon Sep 17 00:00:00 2001 From: kanat Date: Tue, 26 Mar 2024 11:55:46 -0700 Subject: [PATCH 09/10] fix dependencies --- sdk/android/BUILD.gn | 10 +++++----- .../org/webrtc/HardwareVideoEncoderWrapperFactory.java | 4 ++-- .../webrtc/SimulcastAlignedVideoEncoderFactory.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index d52d64b4cf..3197cbe6d6 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -154,6 +154,7 @@ if (is_android) { sources = [ "api/org/webrtc/Predicate.java", "api/org/webrtc/RefCounted.java", + "api/org/webrtc/ResolutionAdjustment.java", "src/java/org/webrtc/ApplicationContextProvider.java", "src/java/org/webrtc/CalledByNative.java", "src/java/org/webrtc/CalledByNativeUnchecked.java", @@ -363,6 +364,8 @@ if (is_android) { "api/org/webrtc/DefaultVideoDecoderFactory.java", "api/org/webrtc/DefaultVideoEncoderFactory.java", "api/org/webrtc/WrappedVideoDecoderFactory.java", + "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", + "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", ] deps = [ @@ -394,6 +397,8 @@ if (is_android) { sources = [ "api/org/webrtc/HardwareVideoDecoderFactory.java", "api/org/webrtc/HardwareVideoEncoderFactory.java", + "api/org/webrtc/HardwareVideoEncoderWrapper.java", + "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java", "api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java", "src/java/org/webrtc/AndroidVideoDecoder.java", "src/java/org/webrtc/BaseBitrateAdjuster.java", @@ -529,11 +534,6 @@ if (is_android) { "api/org/webrtc/SoftwareVideoEncoderFactory.java", "api/org/webrtc/SimulcastVideoEncoder.java", "api/org/webrtc/SimulcastVideoEncoderFactory.java", - "api/org/webrtc/ResolutionAdjustment.java", - "api/org/webrtc/HardwareVideoEncoderWrapper.java", - "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java", - "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", - "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", ] deps = [ diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java index 11f7c73198..2b5b1c8f7f 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java @@ -24,10 +24,10 @@ class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory { private static final String TAG = "HardwareVideoEncoderWrapperFactory"; - private final VideoEncoderFactory factory; + private final HardwareVideoEncoderFactory factory; private final int resolutionPixelAlignment; - public HardwareVideoEncoderWrapperFactory(VideoEncoderFactory factory, int resolutionPixelAlignment) { + public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) { this.factory = factory; this.resolutionPixelAlignment = resolutionPixelAlignment; if (resolutionPixelAlignment == 0) { diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java index ac1e34c607..ab9faaa0c4 100644 --- a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java @@ -33,7 +33,7 @@ */ public class SimulcastAlignedVideoEncoderFactory implements VideoEncoderFactory { private static class StreamEncoderWrapper implements VideoEncoder { - private static final String TAG = StreamEncoderWrapper.class.getSimpleName(); + private static final String TAG = "StreamEncoderWrapper"; private final VideoEncoder encoder; private final ExecutorService executor = Executors.newSingleThreadExecutor(); private VideoEncoder.Settings streamSettings; From 50db26e60be05e841a4fc427343676d4226a8c90 Mon Sep 17 00:00:00 2001 From: kanat Date: Tue, 26 Mar 2024 12:11:18 -0700 Subject: [PATCH 10/10] move ResolutionAdjustment to video_java --- sdk/android/BUILD.gn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 3197cbe6d6..2e29916261 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -154,7 +154,6 @@ if (is_android) { sources = [ "api/org/webrtc/Predicate.java", "api/org/webrtc/RefCounted.java", - "api/org/webrtc/ResolutionAdjustment.java", "src/java/org/webrtc/ApplicationContextProvider.java", "src/java/org/webrtc/CalledByNative.java", "src/java/org/webrtc/CalledByNativeUnchecked.java", @@ -233,6 +232,7 @@ if (is_android) { "api/org/webrtc/WrappedNativeVideoEncoder.java", "api/org/webrtc/YuvConverter.java", "api/org/webrtc/YuvHelper.java", + "api/org/webrtc/ResolutionAdjustment.java", "src/java/org/webrtc/EglBase10Impl.java", "src/java/org/webrtc/EglBase14Impl.java", "src/java/org/webrtc/GlGenericDrawer.java",