Ticket #199: 0001-Reshaper-High-bitdepth-bug-fix.patch

File 0001-Reshaper-High-bitdepth-bug-fix.patch, 4.9 KB (added by LGE_VCC, 5 years ago)

patch for bug fix (LGE_BUG_FIX)

  • source/App/EncoderApp/EncAppCfg.cpp 

    From e9300d39033a26ef7b2e09ffed497c8761550b0d Mon Sep 17 00:00:00 2001
From: Jane Zhao <jie.zhao@lge.com>
Date: Fri, 22 Feb 2019 10:54:04 -0800
Subject: [PATCH] Reshaper High bitdepth bug fix

---
 source/App/EncoderApp/EncAppCfg.cpp  |  6 ++++++
 source/Lib/CommonLib/Reshape.cpp     |  7 +++++--
 source/Lib/CommonLib/TypeDef.h       |  2 ++
 source/Lib/EncoderLib/EncReshape.cpp | 12 ++++++++++++
 4 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/source/App/EncoderApp/EncAppCfg.cpp b/source/App/EncoderApp/EncAppCfg.cpp
index eb18b7e..75d476c 100644
    a b bool EncAppCfg::parseCfg( int argc, char* argv[] ) 
    15701570  {
    15711571    m_chromaFormatConstraint = (tmpConstraintChromaFormat == 0) ? m_chromaFormatIDC : numberToChromaFormat(tmpConstraintChromaFormat);
    15721572    m_bitDepthConstraint     = ( ( m_profile == Profile::MAIN10 || m_profile == Profile::NEXT ) ? 10 : 8 );
     1573#if LGE_TEST_HIGH_BITDEPTH
     1574    if (m_profile == Profile::NEXT) { m_bitDepthConstraint = 16; }
     1575#endif
    15731576  }
    15741577
    15751578
    … … bool EncAppCfg::xCheckParameter() 
    20942097  }
    20952098  else
    20962099  {
     2100#if !LGE_TEST_HIGH_BITDEPTH
    20972101    xConfirmPara(m_bitDepthConstraint!=((m_profile==Profile::MAIN10 || m_profile==Profile::NEXT)?10:8), "BitDepthConstraint must be 8 for MAIN profile and 10 for MAIN10 profile.");
     2102#endif
     2103
    20982104    xConfirmPara(m_chromaFormatConstraint!=CHROMA_420 && m_profile!=Profile::NEXT, "ChromaFormatConstraint must be 420 for non main-RExt and non-Next profiles.");
    20992105    xConfirmPara(m_intraConstraintFlag==true, "IntraConstraintFlag must be false for non main_RExt profiles.");
    21002106    xConfirmPara(m_lowerBitRateConstraintFlag==false, "LowerBitrateConstraintFlag must be true for non main-RExt profiles.");

  • source/Lib/CommonLib/Reshape.cpp 

    diff --git a/source/Lib/CommonLib/Reshape.cpp b/source/Lib/CommonLib/Reshape.cpp
index 31ecca8..b88aefb 100644
    a b void Reshape::constructReshaper() 
    198198    int16_t Y2 = m_reshapePivot[i + 1];
    199199
    200200    m_fwdLUT[i*pwlFwdBinLen] = Clip3((Pel)0, (Pel)((1 << m_lumaBD) - 1), (Pel)Y1);
    201 
     201#if LGE_BUG_FIX
     202    int log2PwlFwdBinLen = (pwlFwdBinLen>=MAX_CU_SIZE)? g_aucLog2[pwlFwdBinLen>> MAX_CU_DEPTH] + MAX_CU_DEPTH : g_aucLog2[pwlFwdBinLen];
     203#else
    202204    int log2PwlFwdBinLen = g_aucLog2[pwlFwdBinLen];
    203 
     205#endif
     206   
    204207    int32_t scale = ((int32_t)(Y2 - Y1) * (1 << FP_PREC) + (1 << (log2PwlFwdBinLen - 1))) >> (log2PwlFwdBinLen);
    205208    for (int j = 1; j < pwlFwdBinLen; j++)
    206209    {

  • source/Lib/CommonLib/TypeDef.h 

    diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h
index e4da03d..a70ef1f 100644
    a b  
    5050#include <assert.h>
    5151#include <cassert>
    5252
     53#define LGE_TEST_HIGH_BITDEPTH                          1
     54#define LGE_BUG_FIX                                     1
    5355#define JVET_M0055_DEBUG_CTU                              1 // DebugCTU encoder debug option
    5456
    5557#define JVET_M0297_32PT_MTS_ZERO_OUT                      1 // 32 point MTS based on skipping high frequency coefficients

  • source/Lib/EncoderLib/EncReshape.cpp 

    diff --git a/source/Lib/EncoderLib/EncReshape.cpp b/source/Lib/EncoderLib/EncReshape.cpp
index ce7d76a..62c7ffb 100644
    a b void EncReshape::initLUTfromdQPModel() 
    11021102    int16_t Y1 = m_reshapePivot[i];
    11031103    int16_t Y2 = m_reshapePivot[i + 1];
    11041104    m_fwdLUT[i*pwlFwdBinLen] = Clip3((Pel)0, (Pel)((1 << m_lumaBD) - 1), (Pel)Y1);
     1105#if LGE_BUG_FIX
     1106    int log2PwlFwdBinLen = (pwlFwdBinLen >= MAX_CU_SIZE) ? g_aucLog2[pwlFwdBinLen >> MAX_CU_DEPTH] + MAX_CU_DEPTH : g_aucLog2[pwlFwdBinLen];
     1107#else
    11051108    int log2PwlFwdBinLen = g_aucLog2[pwlFwdBinLen];
     1109#endif
    11061110    int32_t scale = ((int32_t)(Y2 - Y1) * (1 << FP_PREC) + (1 << (log2PwlFwdBinLen - 1))) >> (log2PwlFwdBinLen);
    11071111    for (int j = 1; j < pwlFwdBinLen; j++)
    11081112    {
    … … void EncReshape::constructReshaperSDR() 
    11421146  int totCW = bdShift != 0 ? (bdShift > 0 ? m_reshapeLUTSize / (1<<bdShift) : m_reshapeLUTSize * (1 << (-bdShift))) : m_reshapeLUTSize;
    11431147  int histBins = PIC_ANALYZE_CW_BINS;
    11441148  int histLenth = totCW/histBins;
     1149#if LGE_BUG_FIX
     1150  int log2HistLenth = (histLenth >= MAX_CU_SIZE) ? g_aucLog2[histLenth >> MAX_CU_DEPTH] + MAX_CU_DEPTH : g_aucLog2[histLenth];
     1151#else
    11451152  int log2HistLenth = g_aucLog2[histLenth];
     1153#endif
    11461154  int16_t *tempFwdLUT = new int16_t[m_reshapeLUTSize + 1]();
    11471155  int i, j;
    11481156  int cwScaleBins1, cwScaleBins2;
    … … void EncReshape::constructReshaperSDR() 
    12711279  m_sliceReshapeInfo.maxNbitsNeededDeltaCW = std::max(1, (int)g_aucLog2[maxAbsDeltaCW << 1]);
    12721280
    12731281  histLenth = m_initCW;
     1282#if LGE_BUG_FIX
     1283  log2HistLenth = (histLenth >= MAX_CU_SIZE) ? g_aucLog2[histLenth >> MAX_CU_DEPTH] + MAX_CU_DEPTH : g_aucLog2[histLenth];
     1284#else
    12741285  log2HistLenth = g_aucLog2[histLenth];
     1286#endif
    12751287
    12761288  int sumBins = 0;
    12771289  for (i = 0; i < PIC_CODE_CW_BINS; i++)   { sumBins += m_binCW[i];  }