Accumulate frame tpl stats and pass through rate control api

Tpl stats is computed at the beginning of encoding the altref
frame. We aggregate tpl stats of all blocks for every frame of
the current group of picture.

After the altref frame is encoded, the tpl stats is passed through
the encode frame result to external environment.
Change-Id: I2284f8cf9c45d35ba02f3ea45f0187edbbf48294

vp9/encoder/vp9_encoder.c

@@ -1025,6 +1025,7 @@ static void dealloc_compressor_data(VP9_COMP *cpi) {
   free_partition_info(cpi);
   free_motion_vector_info(cpi);
   free_fp_motion_vector_info(cpi);
+  free_tpl_stats_info(cpi);
 #endif
 
   vp9_free_ref_frame_buffers(cm->buffer_pool);
@@ -2665,6 +2666,7 @@ VP9_COMP *vp9_create_compressor(const VP9EncoderConfig *oxcf,
   partition_info_init(cpi);
   motion_vector_info_init(cpi);
   fp_motion_vector_info_init(cpi);
+  tpl_stats_info_init(cpi);
 #endif
 
   return cpi;
@@ -5306,6 +5308,7 @@ static void update_encode_frame_result(
 #if CONFIG_RATE_CTRL
     const PARTITION_INFO *partition_info,
     const MOTION_VECTOR_INFO *motion_vector_info,
+    const TplDepStats *tpl_stats_info,
 #endif  // CONFIG_RATE_CTRL
     ENCODE_FRAME_RESULT *encode_frame_result);
 #endif  // !CONFIG_REALTIME_ONLY
@@ -5520,7 +5523,7 @@ static void encode_frame_to_data_rate(
         cpi->Source, coded_frame_buf, ref_frame_bufs, vp9_get_quantizer(cpi),
         cm->bit_depth, cpi->oxcf.input_bit_depth, cpi->td.counts,
 #if CONFIG_RATE_CTRL
-        cpi->partition_info, cpi->motion_vector_info,
+        cpi->partition_info, cpi->motion_vector_info, cpi->tpl_stats_info,
 #endif  // CONFIG_RATE_CTRL
         encode_frame_result);
   }
@@ -7371,6 +7374,48 @@ static void free_tpl_buffer(VP9_COMP *cpi) {
   }
 }
 
+#if CONFIG_RATE_CTRL
+static void accumulate_frame_tpl_stats(VP9_COMP *cpi) {
+  VP9_COMMON *const cm = &cpi->common;
+  const GF_GROUP *gf_group = &cpi->twopass.gf_group;
+  int show_frame_count = 0;
+  int frame_idx;
+  // Accumulate tpl stats for each frame in the current group of picture.
+  for (frame_idx = 1; frame_idx < gf_group->gf_group_size; ++frame_idx) {
+    TplDepFrame *tpl_frame = &cpi->tpl_stats[frame_idx];
+    if (!tpl_frame->is_valid) continue;
+
+    TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
+    const int tpl_stride = tpl_frame->stride;
+    int64_t intra_cost_base = 0;
+    int64_t inter_cost_base = 0;
+    int64_t mc_dep_cost_base = 0;
+    int64_t mc_ref_cost_base = 0;
+    int64_t mc_flow_base = 0;
+    int row, col;
+
+    for (row = 0; row < cm->mi_rows && tpl_frame->is_valid; ++row) {
+      for (col = 0; col < cm->mi_cols; ++col) {
+        TplDepStats *this_stats = &tpl_stats[row * tpl_stride + col];
+        intra_cost_base += this_stats->intra_cost;
+        inter_cost_base += this_stats->inter_cost;
+        mc_dep_cost_base += this_stats->mc_dep_cost;
+        mc_ref_cost_base += this_stats->mc_ref_cost;
+        mc_flow_base += this_stats->mc_flow;
+      }
+    }
+
+    cpi->tpl_stats_info[show_frame_count].intra_cost = intra_cost_base;
+    cpi->tpl_stats_info[show_frame_count].inter_cost = inter_cost_base;
+    cpi->tpl_stats_info[show_frame_count].mc_dep_cost = mc_dep_cost_base;
+    cpi->tpl_stats_info[show_frame_count].mc_ref_cost = mc_ref_cost_base;
+    cpi->tpl_stats_info[show_frame_count].mc_flow = mc_flow_base;
+
+    ++show_frame_count;
+  }
+}
+#endif  // CONFIG_RATE_CTRL
+
 static void setup_tpl_stats(VP9_COMP *cpi) {
   GF_PICTURE gf_picture[MAX_ARF_GOP_SIZE];
   const GF_GROUP *gf_group = &cpi->twopass.gf_group;
@@ -7393,6 +7438,10 @@ static void setup_tpl_stats(VP9_COMP *cpi) {
   dump_tpl_stats(cpi, tpl_group_frames, gf_group, gf_picture, cpi->tpl_bsize);
 #endif  // DUMP_TPL_STATS
 #endif  // CONFIG_NON_GREEDY_MV
+
+#if CONFIG_RATE_CTRL
+  accumulate_frame_tpl_stats(cpi);
+#endif  // CONFIG_RATE_CTRL
 }
 
 void vp9_get_ref_frame_info(FRAME_UPDATE_TYPE update_type, int ref_frame_flags,
@@ -7575,6 +7624,7 @@ static void update_encode_frame_result(
 #if CONFIG_RATE_CTRL
     const PARTITION_INFO *partition_info,
     const MOTION_VECTOR_INFO *motion_vector_info,
+    const TplDepStats *tpl_stats_info,
 #endif  // CONFIG_RATE_CTRL
     ENCODE_FRAME_RESULT *encode_frame_result) {
 #if CONFIG_RATE_CTRL
@@ -7598,6 +7648,7 @@ static void update_encode_frame_result(
   copy_frame_counts(counts, &encode_frame_result->frame_counts);
   encode_frame_result->partition_info = partition_info;
   encode_frame_result->motion_vector_info = motion_vector_info;
+  encode_frame_result->tpl_stats_info = tpl_stats_info;
   if (encode_frame_result->coded_frame.allocated) {
     yv12_buffer_to_image_buffer(&coded_frame_buf->buf,
                                 &encode_frame_result->coded_frame);

vp9/encoder/vp9_encoder.h

@@ -973,6 +973,7 @@ typedef struct VP9_COMP {
   PARTITION_INFO *partition_info;
   MOTION_VECTOR_INFO *motion_vector_info;
   MOTION_VECTOR_INFO *fp_motion_vector_info;
+  TplDepStats *tpl_stats_info;
 
   RATE_QSTEP_MODEL rq_model[ENCODE_FRAME_TYPES];
 #endif
@@ -1029,6 +1030,23 @@ static INLINE void free_motion_vector_info(struct VP9_COMP *cpi) {
   cpi->motion_vector_info = NULL;
 }
 
+// Allocates memory for the tpl stats information.
+// Only called once in vp9_create_compressor().
+static INLINE void tpl_stats_info_init(struct VP9_COMP *cpi) {
+  VP9_COMMON *const cm = &cpi->common;
+  CHECK_MEM_ERROR(
+      cm, cpi->tpl_stats_info,
+      (TplDepStats *)vpx_calloc(MAX_LAG_BUFFERS, sizeof(TplDepStats)));
+  memset(cpi->tpl_stats_info, 0, MAX_LAG_BUFFERS * sizeof(TplDepStats));
+}
+
+// Frees memory of the tpl stats information.
+// Only called once in dealloc_compressor_data().
+static INLINE void free_tpl_stats_info(struct VP9_COMP *cpi) {
+  vpx_free(cpi->tpl_stats_info);
+  cpi->tpl_stats_info = NULL;
+}
+
 // Allocates memory for the first pass motion vector information.
 // The unit size is each 16x16 block.
 // Only called once in vp9_create_compressor().
@@ -1091,6 +1109,7 @@ typedef struct ENCODE_FRAME_RESULT {
   FRAME_COUNTS frame_counts;
   const PARTITION_INFO *partition_info;
   const MOTION_VECTOR_INFO *motion_vector_info;
+  const TplDepStats *tpl_stats_info;
   IMAGE_BUFFER coded_frame;
   RATE_QINDEX_HISTORY rq_history;
 #endif  // CONFIG_RATE_CTRL

vp9/simple_encode.cc

@@ -207,6 +207,24 @@ static void update_motion_vector_info(
   }
 }
 
+static void update_tpl_stats_info(const TplDepStats *input_tpl_stats_info,
+                                  const int show_frame_count,
+                                  TplStatsInfo *output_tpl_stats_info) {
+  int frame_idx;
+  for (frame_idx = 0; frame_idx < show_frame_count; ++frame_idx) {
+    output_tpl_stats_info[frame_idx].intra_cost =
+        input_tpl_stats_info[frame_idx].intra_cost;
+    output_tpl_stats_info[frame_idx].inter_cost =
+        input_tpl_stats_info[frame_idx].inter_cost;
+    output_tpl_stats_info[frame_idx].mc_flow =
+        input_tpl_stats_info[frame_idx].mc_flow;
+    output_tpl_stats_info[frame_idx].mc_dep_cost =
+        input_tpl_stats_info[frame_idx].mc_dep_cost;
+    output_tpl_stats_info[frame_idx].mc_ref_cost =
+        input_tpl_stats_info[frame_idx].mc_ref_cost;
+  }
+}
+
 static void update_frame_counts(const FRAME_COUNTS *input_counts,
                                 FrameCounts *output_counts) {
   // Init array sizes.
@@ -486,6 +504,7 @@ static bool init_encode_frame_result(EncodeFrameResult *encode_frame_result,
                                              encode_frame_result->num_cols_4x4);
   encode_frame_result->motion_vector_info.resize(
       encode_frame_result->num_rows_4x4 * encode_frame_result->num_cols_4x4);
+  encode_frame_result->tpl_stats_info.resize(MAX_LAG_BUFFERS);
 
   if (encode_frame_result->coding_data.get() == nullptr) {
     return false;
@@ -507,7 +526,7 @@ static void encode_frame_result_update_rq_history(
 }
 
 static void update_encode_frame_result(
-    EncodeFrameResult *encode_frame_result,
+    EncodeFrameResult *encode_frame_result, const int show_frame_count,
     const ENCODE_FRAME_RESULT *encode_frame_info) {
   encode_frame_result->coding_data_bit_size =
       encode_frame_result->coding_data_byte_size * 8;
@@ -536,6 +555,10 @@ static void update_encode_frame_result(
                             kMotionVectorSubPixelPrecision);
   update_frame_counts(&encode_frame_info->frame_counts,
                       &encode_frame_result->frame_counts);
+  if (encode_frame_result->frame_type == kFrameTypeAltRef) {
+    update_tpl_stats_info(encode_frame_info->tpl_stats_info, show_frame_count,
+                          &encode_frame_result->tpl_stats_info[0]);
+  }
   encode_frame_result_update_rq_history(&encode_frame_info->rq_history,
                                         encode_frame_result);
 }
@@ -1169,7 +1192,10 @@ void SimpleEncode::EncodeFrame(EncodeFrameResult *encode_frame_result) {
       abort();
     }
 
-    update_encode_frame_result(encode_frame_result, &encode_frame_info);
+    const GroupOfPicture group_of_picture = this->ObserveGroupOfPicture();
+    const int show_frame_count = group_of_picture.show_frame_count;
+    update_encode_frame_result(encode_frame_result, show_frame_count,
+                               &encode_frame_info);
     PostUpdateState(*encode_frame_result);
   } else {
     // TODO(angiebird): Clean up encode_frame_result.

vp9/simple_encode.h

@@ -87,6 +87,24 @@ struct MotionVectorInfo {
   double mv_column[2];
 };
 
+// Accumulated tpl stats of all blocks in one frame.
+// For each frame, the tpl stats are computed per 32x32 block.
+struct TplStatsInfo {
+  // Intra complexity: the sum of absolute transform difference (SATD) of
+  // intra predicted residuals.
+  int64_t intra_cost;
+  // Inter complexity: the SATD of inter predicted residuals.
+  int64_t inter_cost;
+  // Motion compensated information flow. It measures how much information
+  // is propagated from the current frame to other frames.
+  int64_t mc_flow;
+  // Motion compensated dependency cost. It equals to its own intra_cost
+  // plus the mc_flow.
+  int64_t mc_dep_cost;
+  // Motion compensated reference cost.
+  int64_t mc_ref_cost;
+};
+
 struct RefFrameInfo {
   int coding_indexes[kRefFrameTypeMax];
 
@@ -261,6 +279,18 @@ struct EncodeFrameResult {
   // Similar to partition info, all 4x4 blocks inside the same partition block
   // share the same motion vector information.
   std::vector<MotionVectorInfo> motion_vector_info;
+  // A vector of the tpl stats information.
+  // The tpl stats measure the complexity of a frame, as well as the
+  // informatioin propagated along the motion trajactory between frames, in
+  // the reference frame structure.
+  // The tpl stats could be used as a more accurate spatial and temporal
+  // complexity measure in addition to the first pass stats.
+  // The vector contains tpl stats for all show frames in a GOP.
+  // The tpl stats stored in the vector is according to the encoding order.
+  // For example, suppose there are N show frames for the current GOP.
+  // Then tpl_stats_info[0] stores the information of the first frame to be
+  // encoded for this GOP, i.e, the AltRef frame.
+  std::vector<TplStatsInfo> tpl_stats_info;
   ImageBuffer coded_frame;
 
   // recode_count, q_index_history and rate_history are only available when