hwc/overlay: Fix decimation / bwc bugs

1) While setting BWC check for 90 transform and swap crop
2) Calculate decimation in the getDecimationFactor API itself.
   Currently, a residual downscale is calculated and the clients have
   to take logs and apply forcible decimation like in MdpCtrl
3) Fix bug where transform gets typecasted to bool

b/14225977

Change-Id: I3c99c571e02e2cf7822342516b6a87d97be553d1

libhwcomposer/hwc_mdpcomp.cpp

@@ -347,7 +347,7 @@ bool MDPComp::isValidDimension(hwc_context_t *ctx, hwc_layer_1_t *layer) {
 
     hwc_rect_t crop = integerizeSourceCrop(layer->sourceCropf);
     hwc_rect_t dst = layer->displayFrame;
-    bool rotated90 = (bool)layer->transform & HAL_TRANSFORM_ROT_90;
+    bool rotated90 = (bool)(layer->transform & HAL_TRANSFORM_ROT_90);
     int crop_w = rotated90 ? crop.bottom - crop.top : crop.right - crop.left;
     int crop_h = rotated90 ? crop.right - crop.left : crop.bottom - crop.top;
     int dst_w = dst.right - dst.left;

libhwcomposer/hwc_utils.cpp

@@ -2177,12 +2177,6 @@ void BwcPM::setBwc(const hwc_rect_t& crop,
     if(!qdutils::MDPVersion::getInstance().supportsBWC()) {
         return;
     }
-    //src width > MAX mixer supported dim
-    if((crop.right - crop.left) > qdutils::MAX_DISPLAY_DIM) {
-        return;
-    }
-    //Decimation necessary, cannot use BWC. H/W requirement.
-    if(qdutils::MDPVersion::getInstance().supportsDecimation()) {
     int src_w = crop.right - crop.left;
     int src_h = crop.bottom - crop.top;
     int dst_w = dst.right - dst.left;
@@ -2190,17 +2184,16 @@ void BwcPM::setBwc(const hwc_rect_t& crop,
     if(transform & HAL_TRANSFORM_ROT_90) {
         swap(src_w, src_h);
     }
-        float horDscale = 0.0f;
-        float verDscale = 0.0f;
-        int horzDeci = 0;
-        int vertDeci = 0;
-        ovutils::getDecimationFactor(src_w, src_h, dst_w, dst_h, horDscale,
-                verDscale);
-        //TODO Use log2f once math.h has it
-        if((int)horDscale)
-            horzDeci = (int)(log(horDscale) / log(2));
-        if((int)verDscale)
-            vertDeci = (int)(log(verDscale) / log(2));
+    //src width > MAX mixer supported dim
+    if(src_w > qdutils::MAX_DISPLAY_DIM) {
+        return;
+    }
+    //Decimation necessary, cannot use BWC. H/W requirement.
+    if(qdutils::MDPVersion::getInstance().supportsDecimation()) {
+        uint8_t horzDeci = 0;
+        uint8_t vertDeci = 0;
+        ovutils::getDecimationFactor(src_w, src_h, dst_w, dst_h, horzDeci,
+                vertDeci);
         if(horzDeci || vertDeci) return;
     }
     //Property

liboverlay/overlayMdp.cpp

@@ -154,32 +154,9 @@ void MdpCtrl::doDownscale() {
         mOVInfo.src_rect.w >>= mDownscale;
         mOVInfo.src_rect.h >>= mDownscale;
     } else if(MDPVersion::getInstance().supportsDecimation()) {
-        //Decimation + MDP Downscale
-        mOVInfo.horz_deci = 0;
-        mOVInfo.vert_deci = 0;
-        int minHorDeci = 0;
-        if(mOVInfo.src_rect.w > 2048) {
-            //If the client sends us something > what a layer mixer supports
-            //then it means it doesn't want to use split-pipe but wants us to
-            //decimate. A minimum decimation of 2 will ensure that the width is
-            //always within layer mixer limits.
-            minHorDeci = 2;
-        }
-
-        float horDscale = 0.0f;
-        float verDscale = 0.0f;
-
         utils::getDecimationFactor(mOVInfo.src_rect.w, mOVInfo.src_rect.h,
-                mOVInfo.dst_rect.w, mOVInfo.dst_rect.h, horDscale, verDscale);
-
-        if(horDscale < minHorDeci)
-            horDscale = minHorDeci;
-
-        if((int)horDscale)
-            mOVInfo.horz_deci = (int)log2f(horDscale);
-
-        if((int)verDscale)
-            mOVInfo.vert_deci = (int)log2f(verDscale);
+                mOVInfo.dst_rect.w, mOVInfo.dst_rect.h, mOVInfo.horz_deci,
+                mOVInfo.vert_deci);
     }
 }
 

liboverlay/overlayUtils.cpp

@@ -285,10 +285,12 @@ int getDownscaleFactor(const int& src_w, const int& src_h,
 }
 
 void getDecimationFactor(const int& src_w, const int& src_h,
-        const int& dst_w, const int& dst_h, float& horDscale,
-        float& verDscale) {
-    horDscale = ceilf((float)src_w / (float)dst_w);
-    verDscale = ceilf((float)src_h / (float)dst_h);
+        const int& dst_w, const int& dst_h, uint8_t& horzDeci,
+        uint8_t& vertDeci) {
+    horzDeci = 0;
+    vertDeci = 0;
+    float horDscale = ceilf((float)src_w / (float)dst_w);
+    float verDscale = ceilf((float)src_h / (float)dst_h);
 
     //Next power of 2, if not already
     horDscale = powf(2.0f, ceilf(log2f(horDscale)));
@@ -298,6 +300,21 @@ void getDecimationFactor(const int& src_w, const int& src_h,
     //between decimator and MDP downscale
     horDscale /= 4.0f;
     verDscale /= 4.0f;
+
+    if((int)horDscale)
+        horzDeci = (uint8_t)log2f(horDscale);
+
+    if((int)verDscale)
+        vertDeci = (uint8_t)log2f(verDscale);
+
+    if(src_w > 2048) {
+        //If the client sends us something > what a layer mixer supports
+        //then it means it doesn't want to use split-pipe but wants us to
+        //decimate. A minimum decimation of 2 will ensure that the width is
+        //always within layer mixer limits.
+        if(horzDeci < 2)
+            horzDeci = 2;
+    }
 }
 
 static inline int compute(const uint32_t& x, const uint32_t& y,

liboverlay/overlayUtils.h

@@ -413,8 +413,8 @@ int getHALFormat(int mdpFormat);
 int getDownscaleFactor(const int& src_w, const int& src_h,
         const int& dst_w, const int& dst_h);
 void getDecimationFactor(const int& src_w, const int& src_h,
-        const int& dst_w, const int& dst_h, float& horDscale,
-        float& verDscale);
+        const int& dst_w, const int& dst_h, uint8_t& horzDeci,
+        uint8_t& vertDeci);
 
 /* flip is upside down and such. V, H flip
  * rotation is 90, 180 etc