Hardware VP8 encoding: Use QP as metric for resize.

Add vp8 frame header parser to get QP from vp8 bitstream. BUG= 4273 R=glaznev@webrtc.org, marpan@google.com, pbos@webrtc.org TBR=stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/49259004 Cr-Commit-Position: refs/heads/master@{#9256}
GetStream · May 21, 2015 · 98d8cf5 · 98d8cf5
1 parent 5fdcdf6
commit 98d8cf5
Show file tree

Hide file tree

Showing 9 changed files with 305 additions and 66 deletions.
diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
@@ -35,6 +35,7 @@
 #include "webrtc/base/thread.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
+#include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h"
 #include "webrtc/system_wrappers/interface/logcat_trace_context.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
@@ -196,8 +197,6 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
   scoped_ptr<webrtc::QualityScaler> quality_scaler_;
-  // Target frame size in bytes.
-  int target_framesize_;
   // Dynamic resolution change, off by default.
   bool scale_;
 };
@@ -280,6 +279,11 @@ int32_t MediaCodecVideoEncoder::InitEncode(
     size_t /* max_payload_size */) {
   const int kMinWidth = 320;
   const int kMinHeight = 180;
+  // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
+  // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
+  // always = 127. Note that in SW, QP is that of the user-level range [0, 63].
+  const int kMaxQP = 127;
+  const int kLowQpThresholdDenominator = 3;
   if (codec_settings == NULL) {
     ALOGE("NULL VideoCodec instance");
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
@@ -290,14 +294,10 @@ int32_t MediaCodecVideoEncoder::InitEncode(
 
   ALOGD("InitEncode request");
   scale_ = false;
-  quality_scaler_->Init(0);
-  quality_scaler_->SetMinResolution(kMinWidth, kMinHeight);
-  quality_scaler_->ReportFramerate(codec_settings->maxFramerate);
-  if (codec_settings->maxFramerate > 0) {
-    target_framesize_ = codec_settings->startBitrate * 1000 /
-        codec_settings->maxFramerate / 8;
-  } else {
-    target_framesize_ = 0;
+  if (codecType_ == kVideoCodecVP8) {
+    quality_scaler_->Init(kMaxQP / kLowQpThresholdDenominator);
+    quality_scaler_->SetMinResolution(kMinWidth, kMinHeight);
+    quality_scaler_->ReportFramerate(codec_settings->maxFramerate);
   }
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
@@ -337,12 +337,8 @@ int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
 
 int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
                                          uint32_t frame_rate) {
-  quality_scaler_->ReportFramerate(frame_rate);
-  if (frame_rate > 0) {
-    target_framesize_ = new_bit_rate * 1000 / frame_rate / 8;
-  } else {
-    target_framesize_ = 0;
-  }
+  if (codecType_ == kVideoCodecVP8)
+    quality_scaler_->ReportFramerate(frame_rate);
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
            this,
@@ -716,16 +712,8 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
         last_input_timestamp_ms_ - last_output_timestamp_ms_,
         frame_encoding_time_ms);
 
-    if (payload_size) {
-      double framesize_deviation = 0.0;
-      if (target_framesize_ > 0) {
-        framesize_deviation =
-          (double)abs((int)payload_size - target_framesize_) /
-          target_framesize_;
-      }
-      quality_scaler_->ReportNormalizedFrameSizeFluctuation(
-          framesize_deviation);
-    }
+    if (payload_size && codecType_ == kVideoCodecVP8)
+      quality_scaler_->ReportQP(webrtc::vp8::GetQP(payload));
 
     // Calculate and print encoding statistics - every 3 seconds.
     frames_encoded_++;
@@ -872,7 +860,8 @@ int32_t MediaCodecVideoEncoder::NextNaluPosition(
 }
 
 void MediaCodecVideoEncoder::OnDroppedFrame() {
-  quality_scaler_->ReportDroppedFrame();
+  if (codecType_ == kVideoCodecVP8)
+    quality_scaler_->ReportDroppedFrame();
 }
 
 MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() {

diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn
@@ -96,7 +96,9 @@ source_set("video_coding_utility") {
     "utility/include/frame_dropper.h",
     "utility/include/moving_average.h",
     "utility/include/quality_scaler.h",
+    "utility/include/vp8_header_parser.h",
     "utility/quality_scaler.cc",
+    "utility/vp8_header_parser.cc",
   ]
 
   configs += [ "../..:common_config" ]

diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
@@ -603,7 +603,7 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
   }
 
   rps_.Init();
-  quality_scaler_.Init(codec_.qpMax);
+  quality_scaler_.Init(codec_.qpMax / kDefaultLowQpDenominator);
   quality_scaler_.ReportFramerate(codec_.maxFramerate);
 
   return InitAndSetControlSettings();
@@ -1035,7 +1035,7 @@ int VP8EncoderImpl::GetEncodedPartitions(
     if (encoded_images_[0]._length > 0) {
       int qp;
       vpx_codec_control(&encoders_[0], VP8E_GET_LAST_QUANTIZER_64, &qp);
-      quality_scaler_.ReportNormalizedQP(qp);
+      quality_scaler_.ReportQP(qp);
     } else {
       quality_scaler_.ReportDroppedFrame();
     }

diff --git a/webrtc/modules/video_coding/utility/include/quality_scaler.h b/webrtc/modules/video_coding/utility/include/quality_scaler.h
@@ -15,6 +15,7 @@
 #include "webrtc/modules/video_coding/utility/include/moving_average.h"
 
 namespace webrtc {
+const int kDefaultLowQpDenominator = 3;
 class QualityScaler {
  public:
   struct Resolution {
@@ -23,15 +24,10 @@ class QualityScaler {
   };
 
   QualityScaler();
-  void Init(int max_qp);
+  void Init(int low_qp_threshold);
   void SetMinResolution(int min_width, int min_height);
   void ReportFramerate(int framerate);
-
-  // Report QP for SW encoder, report framesize fluctuation for HW encoder,
-  // only one of these two functions should be called, framesize fluctuation
-  // is to be used only if qp isn't available.
-  void ReportNormalizedQP(int qp);
-  void ReportNormalizedFrameSizeFluctuation(double framesize_deviation);
+  void ReportQP(int qp);
   void ReportDroppedFrame();
   void Reset(int framerate, int bitrate, int width, int height);
   Resolution GetScaledResolution(const I420VideoFrame& frame);
@@ -45,10 +41,9 @@ class QualityScaler {
   I420VideoFrame scaled_frame_;
 
   size_t num_samples_;
-  int target_framesize_;
   int low_qp_threshold_;
   MovingAverage<int> framedrop_percent_;
-  MovingAverage<double> frame_quality_;
+  MovingAverage<int> average_qp_;
 
   int downscale_shift_;
   int min_width_;

diff --git a/webrtc/modules/video_coding/utility/include/vp8_header_parser.h b/webrtc/modules/video_coding/utility/include/vp8_header_parser.h
@@ -0,0 +1,75 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_VP8_PARSE_HEADER_H_
+#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_VP8_PARSE_HEADER_H_
+
+namespace webrtc {
+
+namespace vp8 {
+
+enum {
+  MB_FEATURE_TREE_PROBS = 3,
+  NUM_MB_SEGMENTS = 4,
+  NUM_REF_LF_DELTAS = 4,
+  NUM_MODE_LF_DELTAS = 4,
+};
+
+typedef struct VP8BitReader VP8BitReader;
+struct VP8BitReader {
+  // Boolean decoder.
+  uint32_t value_;            // Current value.
+  uint32_t range_;            // Current range minus 1. In [127, 254] interval.
+  int bits_;                  // Number of valid bits left.
+  // Read buffer.
+  const uint8_t* buf_;        // Next byte to be read.
+  const uint8_t* buf_end_;    // End of read buffer.
+  int eof_;                   // True if input is exhausted.
+};
+
+const uint8_t kVP8Log2Range[128] = {
+  7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4,
+  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  0
+};
+
+// range = ((range - 1) << kVP8Log2Range[range]) + 1
+const uint8_t kVP8NewRange[128] = {
+  127, 127, 191, 127, 159, 191, 223, 127,
+  143, 159, 175, 191, 207, 223, 239, 127,
+  135, 143, 151, 159, 167, 175, 183, 191,
+  199, 207, 215, 223, 231, 239, 247, 127,
+  131, 135, 139, 143, 147, 151, 155, 159,
+  163, 167, 171, 175, 179, 183, 187, 191,
+  195, 199, 203, 207, 211, 215, 219, 223,
+  227, 231, 235, 239, 243, 247, 251, 127,
+  129, 131, 133, 135, 137, 139, 141, 143,
+  145, 147, 149, 151, 153, 155, 157, 159,
+  161, 163, 165, 167, 169, 171, 173, 175,
+  177, 179, 181, 183, 185, 187, 189, 191,
+  193, 195, 197, 199, 201, 203, 205, 207,
+  209, 211, 213, 215, 217, 219, 221, 223,
+  225, 227, 229, 231, 233, 235, 237, 239,
+  241, 243, 245, 247, 249, 251, 253, 127
+};
+
+int GetQP(uint8_t* buf);
+
+}  // namespace vp8
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_VIDEO_CODING_UTILITY_VP8_PARSE_HEADER_H_
diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc
@@ -14,42 +14,31 @@ namespace webrtc {
 static const int kMinFps = 10;
 static const int kMeasureSeconds = 5;
 static const int kFramedropPercentThreshold = 60;
-static const int kLowQpThresholdDenominator = 3;
-static const double kFramesizeFlucThreshold = 0.11;
 
 QualityScaler::QualityScaler()
     : num_samples_(0), low_qp_threshold_(-1), downscale_shift_(0),
       min_width_(0), min_height_(0) {
 }
 
-void QualityScaler::Init(int max_qp) {
+void QualityScaler::Init(int low_qp_threshold) {
   ClearSamples();
-  low_qp_threshold_ = max_qp / kLowQpThresholdDenominator;
+  low_qp_threshold_ = low_qp_threshold;
 }
 
 void QualityScaler::SetMinResolution(int min_width, int min_height) {
   min_width_ = min_width;
   min_height_ = min_height;
 }
 
-// TODO(jackychen): target_framesize should be calculated from average bitrate
-// in the measured period of time.
-// Report framerate(fps) and target_bitrate(kbit/s) to estimate # of samples
-// and get target_framesize_.
+// Report framerate(fps) to estimate # of samples.
 void QualityScaler::ReportFramerate(int framerate) {
   num_samples_ = static_cast<size_t>(
       kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate));
 }
 
-void QualityScaler::ReportNormalizedQP(int qp) {
+void QualityScaler::ReportQP(int qp) {
   framedrop_percent_.AddSample(0);
-  frame_quality_.AddSample(static_cast<double>(qp) / low_qp_threshold_);
-}
-
-void QualityScaler::ReportNormalizedFrameSizeFluctuation(
-    double framesize_deviation) {
-  framedrop_percent_.AddSample(0);
-  frame_quality_.AddSample(framesize_deviation / kFramesizeFlucThreshold);
+  average_qp_.AddSample(qp);
 }
 
 void QualityScaler::ReportDroppedFrame() {
@@ -67,13 +56,12 @@ QualityScaler::Resolution QualityScaler::GetScaledResolution(
   res.height = frame.height();
 
   // Update scale factor.
-  int avg_drop;
-  double avg_quality;
+  int avg_drop, avg_qp;
   if (framedrop_percent_.GetAverage(num_samples_, &avg_drop) &&
       avg_drop >= kFramedropPercentThreshold) {
     AdjustScale(false);
-  } else if (frame_quality_.GetAverage(num_samples_, &avg_quality) &&
-      avg_quality <= 1.0) {
+  } else if (average_qp_.GetAverage(num_samples_, &avg_qp) &&
+      avg_qp <= low_qp_threshold_) {
     AdjustScale(true);
   }
 
@@ -119,7 +107,7 @@ const I420VideoFrame& QualityScaler::GetScaledFrame(
 
 void QualityScaler::ClearSamples() {
   framedrop_percent_.Reset();
-  frame_quality_.Reset();
+  average_qp_.Reset();
 }
 
 void QualityScaler::AdjustScale(bool up) {

diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
@@ -31,7 +31,7 @@ class QualityScalerTest : public ::testing::Test {
   QualityScalerTest() {
     input_frame_.CreateEmptyFrame(
         kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth);
-    qs_.Init(kMaxQp);
+    qs_.Init(kMaxQp / kDefaultLowQpDenominator);
     qs_.ReportFramerate(kFramerate);
   }
 
@@ -40,7 +40,7 @@ class QualityScalerTest : public ::testing::Test {
     for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
       switch (scale_direction) {
         case kScaleUp:
-          qs_.ReportNormalizedQP(kLowQp);
+          qs_.ReportQP(kLowQp);
           break;
         case kScaleDown:
           qs_.ReportDroppedFrame();
@@ -93,7 +93,7 @@ TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
 
 TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
   for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) {
-    qs_.ReportNormalizedQP(kNormalQp);
+    qs_.ReportQP(kNormalQp);
     qs_.ReportDroppedFrame();
     qs_.ReportDroppedFrame();
     if (qs_.GetScaledResolution(input_frame_).width < input_frame_.width())
@@ -105,15 +105,15 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
 
 TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
-    qs_.ReportNormalizedQP(kNormalQp);
+    qs_.ReportQP(kNormalQp);
     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
         << "Unexpected scale on half framedrop.";
   }
 }
 
 TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
   for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
-    qs_.ReportNormalizedQP(kNormalQp);
+    qs_.ReportQP(kNormalQp);
     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
         << "Unexpected scale on half framedrop.";
 
@@ -153,7 +153,7 @@ void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() {
 
   // Verify we don't start upscaling after further low use.
   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
-    qs_.ReportNormalizedQP(kLowQp);
+    qs_.ReportQP(kLowQp);
     ExpectOriginalFrame();
   }
 }

diff --git a/webrtc/modules/video_coding/utility/video_coding_utility.gyp b/webrtc/modules/video_coding/utility/video_coding_utility.gyp
@@ -21,8 +21,10 @@
         'frame_dropper.cc',
         'include/frame_dropper.h',
         'include/moving_average.h',
-        'quality_scaler.cc',
         'include/quality_scaler.h',
+        'include/vp8_header_parser.h',
+        'quality_scaler.cc',
+        'vp8_header_parser.cc',
       ],
     },
   ], # targets