Ticket #29: RDO_bit_estimate.patch

Lib/TLibCommon/TypeDef.h

@@ -47 +47 @@
 //! \ingroup TLibCommon
 //! \{
 
+#define FIX_RDOQ_BIT_ESTIMATE 1  // correct RDOQ uninitialized values in case of vertical scan 
+
 ///////////////////////////////////////////////////////////
 // KTA tools section start
 ///////////////////////////////////////////////////////////

Lib/TLibEncoder/TEncCavlc.cpp

@@ -1522 +1522 @@
   assert(0);
 }
 
-Void TEncCavlc::estBit( estBitsSbacStruct* /*pcEstBitsCabac*/, Int /*width*/, Int /*height*/, ChannelType /*chType*/ )
+Void TEncCavlc::estBit( estBitsSbacStruct* /*pcEstBitsCabac*/, Int /*width*/, Int /*height*/, ChannelType /*chType*/ 
+#if FIX_RDOQ_BIT_ESTIMATE
+  , UInt /*uiScanIdx*/
+#endif
+  )
 {
   // printf("error : no VLC mode support in this version\n");
   return;

Lib/TLibEncoder/TEncCavlc.h

@@ -177 +177 @@
   Void codeKLTFlags      (TComTU &rTu, ComponentID component);
 #endif
 
-  Void estBit            ( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType );
+  Void estBit            ( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+    , UInt uiScanIdx
+#endif
+    );
 
   Void xCodePredWeightTable          ( TComSlice* pcSlice );
 

Lib/TLibEncoder/TEncEntropy.cpp

@@ -1205 +1205 @@
   }
 }
 
-Void TEncEntropy::estimateBit (estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, const ChannelType chType)
+Void TEncEntropy::estimateBit (estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, const ChannelType chType
+#if FIX_RDOQ_BIT_ESTIMATE
+  , UInt uiScanIdx
+#endif
+  )
 {
 #if JVET_C0024_QTBT
   if (width==2 || height==2)
@@ -1213 +1217 @@
       return;   //don't use RDOQ for 2xn;
   }
 
-  m_pcEntropyCoderIf->estBit ( pcEstBitsSbac, width, height, chType );
+  m_pcEntropyCoderIf->estBit ( pcEstBitsSbac, width, height, chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+    , uiScanIdx
+#endif
+    );
 #else
   const UInt heightAtEntropyCoding = (width != height) ? (height >> 1) : height;
 

Lib/TLibEncoder/TEncEntropy.h

@@ -150 +150 @@
   virtual Void codeKLTFlags      (TComTU &rTu, ComponentID component) = 0;
 #endif
   virtual Void codeSAOBlkParam   (SAOBlkParam& saoBlkParam, const BitDepths &bitDepths, Bool* sliceEnabled, Bool leftMergeAvail, Bool aboveMergeAvail, Bool onlyEstMergeInfo = false)    =0;
-  virtual Void estBit               (estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType) = 0;
+  virtual Void estBit               (estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType
+#if FIX_RDOQ_BIT_ESTIMATE
+    , UInt uiScanIdx
+#endif
+    ) = 0;
 
   virtual Void codeExplicitRdpcmMode ( TComTU &rTu, const ComponentID compID ) = 0;
 
@@ -323 +327 @@
 
   Void encodeCoeffNxN         ( TComTU &rTu, TCoeff* pcCoef, const ComponentID compID );
 
-  Void estimateBit             ( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType );
+  Void estimateBit             ( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+    , UInt uiScanIdx
+#endif
+    );
 
   Void encodeSAOBlkParam(SAOBlkParam& saoBlkParam, const BitDepths &bitDepths, Bool* sliceEnabled, Bool leftMergeAvail, Bool aboveMergeAvail){m_pcEntropyCoderIf->codeSAOBlkParam(saoBlkParam, bitDepths, sliceEnabled, leftMergeAvail, aboveMergeAvail, false);}
 

Lib/TLibEncoder/TEncSbac.cpp

@@ -2803 +2803 @@
  *   estimate bit cost for CBP, significant map and significant coefficients
  ****************************************************************************
  */
-Void TEncSbac::estBit( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType )
+Void TEncSbac::estBit( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+  , UInt uiScanIdx
+#endif
+  )
 {
   estCBFBit( pcEstBitsSbac );
 
@@ -2813 +2817 @@
   estSignificantMapBit( pcEstBitsSbac, width, height, chType );
 
   // encode last significant position
-  estLastSignificantPositionBit( pcEstBitsSbac, width, height, chType );
+  estLastSignificantPositionBit( pcEstBitsSbac, width, height, chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+    , uiScanIdx
+#endif
+    );
 
   // encode significant coefficients
   estSignificantCoefficientsBit( pcEstBitsSbac, chType );
@@ -2963 +2971 @@
  ****************************************************************************
  */
 
-Void TEncSbac::estLastSignificantPositionBit( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType )
+Void TEncSbac::estLastSignificantPositionBit( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+  , UInt uiScanIdx
+#endif
+  )
 {
   //--------------------------------------------------------------------------------------------------.
+#if FIX_RDOQ_BIT_ESTIMATE
+  // swap
+  if (uiScanIdx == SCAN_VER)
+  {
+    swap(width, height);
+  }
+#endif
 
   //set up the number of channels
 

Lib/TLibEncoder/TEncSbac.h

@@ -214 +214 @@
   // for RD-optimizatioon
   // -------------------------------------------------------------------------------------------------------------------
 
-  Void estBit               (estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType);
+  Void estBit               (estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType
+#if FIX_RDOQ_BIT_ESTIMATE
+    , UInt uiScanIdx
+#endif
+    );
   Void estCBFBit                     ( estBitsSbacStruct* pcEstBitsSbac );
   Void estSignificantCoeffGroupMapBit( estBitsSbacStruct* pcEstBitsSbac, ChannelType chType );
   Void estSignificantMapBit          ( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType );
-  Void estLastSignificantPositionBit ( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType );
+  Void estLastSignificantPositionBit ( estBitsSbacStruct* pcEstBitsSbac, Int width, Int height, ChannelType chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+    , UInt uiScanIdx
+#endif
+    );
   Void estSignificantCoefficientsBit ( estBitsSbacStruct* pcEstBitsSbac, ChannelType chType );
 
   Void codeExplicitRdpcmMode            ( TComTU &rTu, const ComponentID compID );

Lib/TLibEncoder/TEncSearch.cpp

@@ -1881 +1881 @@
   if( useTransformSkip ? m_pcEncCfg->getUseRDOQTS() : m_pcEncCfg->getUseRDOQ() )
 #endif
   {
-    m_pcEntropyCoder->estimateBit( m_pcTrQuant->m_pcEstBitsSbac, uiWidth, uiHeight, chType );
+#if FIX_RDOQ_BIT_ESTIMATE
+    COEFF_SCAN_TYPE scanType = COEFF_SCAN_TYPE(pcCU->getCoefScanIdx(uiAbsPartIdx, uiWidth, uiHeight, compID));
+#endif
+    m_pcEntropyCoder->estimateBit( m_pcTrQuant->m_pcEstBitsSbac, uiWidth, uiHeight, chType 
+#if FIX_RDOQ_BIT_ESTIMATE
+      , scanType
+#endif
+      );
   }
 
   //--- transform and quantization ---
@@ -2192 +2199 @@
     if (useTransformSkip ? m_pcEncCfg->getUseRDOQTS() : m_pcEncCfg->getUseRDOQ())
 #endif
     {
-        m_pcEntropyCoder->estimateBit(m_pcTrQuant->m_pcEstBitsSbac, uiWidth, uiHeight, chType);
+#if FIX_RDOQ_BIT_ESTIMATE
+      COEFF_SCAN_TYPE scanType = COEFF_SCAN_TYPE(pcCU->getCoefScanIdx(uiAbsPartIdx, uiWidth, uiHeight, compID));
+#endif
+        m_pcEntropyCoder->estimateBit(m_pcTrQuant->m_pcEstBitsSbac, uiWidth, uiHeight, chType
+#if FIX_RDOQ_BIT_ESTIMATE
+          , scanType
+#endif
+          );
     }
 
     //--- transform and quantization ---
@@ -8720 +8734 @@
 
                           if ((compID != COMPONENT_Cr) && ((transformSkipModeId == 1) ? m_pcEncCfg->getUseRDOQTS() : m_pcEncCfg->getUseRDOQ()))
                           {
-                              m_pcEntropyCoder->estimateBit(m_pcTrQuant->m_pcEstBitsSbac, tuCompRect.width, tuCompRect.height, toChannelType(compID));
+#if FIX_RDOQ_BIT_ESTIMATE
+                            COEFF_SCAN_TYPE scanType = COEFF_SCAN_TYPE(pcCU->getCoefScanIdx(uiAbsPartIdx, tuCompRect.width, tuCompRect.height, compID));
+#endif
+                              m_pcEntropyCoder->estimateBit(m_pcTrQuant->m_pcEstBitsSbac, tuCompRect.width, tuCompRect.height, toChannelType(compID)
+#if FIX_RDOQ_BIT_ESTIMATE
+                                , scanType
+#endif
+                                );
                           }
 
 #if RDOQ_CHROMA_LAMBDA