Ticket #981: tsrdoqpatchAlibaba.diff

File tsrdoqpatchAlibaba.diff, 23.9 KB (added by m.sarwer, 19 months ago)
  • source/Lib/CommonLib/QuantRDOQ.cpp

    diff --git a/source/Lib/CommonLib/QuantRDOQ.cpp b/source/Lib/CommonLib/QuantRDOQ.cpp
    index a258d766..58b1bf23 100644
    a b void QuantRDOQ::quant(TransformUnit &tu, const ComponentID &compID, const CCoeff 
    533533    if (!m_useSelectiveRDOQ || xNeedRDOQ(tu, compID, piCoef, cQP))
    534534    {
    535535#endif
     536#if ALI_FIX_RDOQ_TS_BUG
     537      if (useTransformSkip && !tu.cs->slice->getTSResidualCodingDisabledFlag())
     538#else
    536539      if( useTransformSkip )
     540#endif
    537541      {
    538542        if( (tu.cu->bdpcmMode && isLuma(compID)) || (isChroma(compID) && tu.cu->bdpcmModeChroma ) )
    539543        {
    void QuantRDOQ::quant(TransformUnit &tu, const ComponentID &compID, const CCoeff 
    546550      }
    547551      else
    548552      {
     553#if ALI_FIX_RDOQ_TS_BUG
     554        if ((tu.cu->bdpcmMode && isLuma(compID)) || (isChroma(compID) && tu.cu->bdpcmModeChroma))
     555          forwardRDPCMRRC(tu, compID, pSrc, uiAbsSum, cQP, ctx);
     556        else
     557#endif
    549558        xRateDistOptQuant( tu, compID, pSrc, uiAbsSum, cQP, ctx );
    550559      }
    551560#if T0196_SELECTIVE_RDOQ
    void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID, 
    637646  const bool   enableScalingLists = getUseScalingList(uiWidth, uiHeight, isTransformSkip, tu.cu->lfnstIdx > 0, disableSMForLFNST);
    638647#endif
    639648  const int    defaultQuantisationCoefficient = g_quantScales[ needSqrtAdjustment ?1:0][cQP.rem(isTransformSkip)];
     649 
     650#if  ALI_FIX_RDOQ_TS_BUG
     651  const double defaultErrorScale = isTransformSkip ? xGetErrScaleCoeff(TU::needsSqrt2Scale(tu, compID), uiWidth, uiHeight, cQP.rem(isTransformSkip), maxLog2TrDynamicRange, channelBitDepth, isTransformSkip)
     652    : (xGetErrScaleCoeffNoScalingList(scalingListType, uiLog2BlockWidth, uiLog2BlockHeight, cQP.rem(isTransformSkip)));
     653  const int iQBits = QUANT_SHIFT + cQP.per(isTransformSkip) + (isTransformSkip ? 0 : iTransformShift) + (needSqrtAdjustment ? -1 : 0);                   // Right shift of non-RDOQ quantizer;  level = (coeff*uiQ + offset)>>q_bits
     654#else
    640655  const double defaultErrorScale              = xGetErrScaleCoeffNoScalingList(scalingListType, uiLog2BlockWidth, uiLog2BlockHeight, cQP.rem(isTransformSkip));
    641656  const int iQBits = QUANT_SHIFT + cQP.per(isTransformSkip) + iTransformShift + (needSqrtAdjustment?-1:0);                   // Right shift of non-RDOQ quantizer;  level = (coeff*uiQ + offset)>>q_bits
    642 
     657#endif
    643658
    644659  const TCoeff entropyCodingMinimum = -(1 << maxLog2TrDynamicRange);
    645660  const TCoeff entropyCodingMaximum =  (1 << maxLog2TrDynamicRange) - 1;
    void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID, 
    10471062
    10481063  if( cctx.signHiding() && uiAbsSum>=2)
    10491064  {
     1065   
     1066#if  ALI_FIX_RDOQ_TS_BUG
     1067    const double inverseQuantScale = double(g_invQuantScales[needSqrtAdjustment ? 1 : 0][cQP.rem(isTransformSkip)]);
     1068#else
    10501069    const double inverseQuantScale = double(g_invQuantScales[0][cQP.rem(isTransformSkip)]);
     1070#endif
    10511071    int64_t rdFactor = (int64_t)(inverseQuantScale * inverseQuantScale * (1 << (2 * cQP.per(isTransformSkip))) / m_dLambda / 16
    10521072                                  / (1 << (2 * DISTORTION_PRECISION_ADJUSTMENT(channelBitDepth)))
    10531073                             + 0.5);
    void QuantRDOQ::forwardRDPCM( TransformUnit &tu, const ComponentID &compID, cons 
    16261646    absSum += abs(level);
    16271647  }
    16281648}
     1649#if ALI_FIX_RDOQ_TS_BUG
     1650
     1651void QuantRDOQ::forwardRDPCMRRC(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf &coeffs, TCoeff &absSum, const QpParam &qp, const Ctx &ctx)
     1652{
     1653  const FracBitsAccess& fracBits = ctx.getFracBitsAcess();
     1654
     1655  const SPS &sps = *tu.cs->sps;
     1656  const CompArea &rect = tu.blocks[compID];
     1657  const uint32_t width = rect.width;
     1658  const uint32_t height = rect.height;
     1659  const ChannelType chType = toChannelType(compID);
     1660  const int channelBitDepth = sps.getBitDepth(chType);
     1661
     1662  const bool extendedPrecision = sps.getSpsRangeExtension().getExtendedPrecisionProcessingFlag();
     1663  const int  maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange(chType);
     1664  const int  dirMode = isLuma(compID) ? tu.cu->bdpcmMode : tu.cu->bdpcmModeChroma;
     1665  int transformShift = getTransformShift(channelBitDepth, rect.size(), maxLog2TrDynamicRange);
     1666
     1667  if (extendedPrecision)
     1668  {
     1669    transformShift = std::max<int>(0, transformShift);
     1670  }
     1671
     1672  double   blockUncodedCost = 0;
     1673  const uint32_t maxNumCoeff = rect.area();
     1674
     1675  CHECK(compID >= MAX_NUM_TBLOCKS, "Invalid component ID");
     1676
     1677  int scalingListType = getScalingListType(tu.cu->predMode, compID);
     1678  CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
     1679
     1680  const TCoeff *srcCoeff = coeffs.buf;
     1681  TCoeff *dstCoeff = tu.getCoeffs(compID).buf;
     1682
     1683  double *costCoeff = m_pdCostCoeff;
     1684  double *costSig = m_pdCostSig;
     1685  double *costCoeff0 = m_pdCostCoeff0;
     1686
     1687  memset(m_pdCostCoeff, 0, sizeof(double) *  maxNumCoeff);
     1688  memset(m_pdCostSig, 0, sizeof(double) *  maxNumCoeff);
     1689  memset(m_fullCoeff, 0, sizeof(TCoeff) * maxNumCoeff);
     1690
     1691  m_bdpcm = dirMode;
     1692
     1693  const bool   needsSqrt2Scale = TU::needsSqrt2Scale(tu, compID);  // should always be false - transform-skipped blocks don't require sqrt(2) compensation.
     1694  const bool   isTransformSkip = (tu.mtsIdx[compID] == MTS_SKIP);
     1695  const int    qBits = QUANT_SHIFT + qp.per(isTransformSkip) + (isTransformSkip ? 0 : transformShift) + (needsSqrt2Scale ? -1 : 0);  // Right shift of non-RDOQ quantizer;  level = (coeff*uiQ + offset)>>q_bits
     1696  const int    quantisationCoefficient = g_quantScales[needsSqrt2Scale ? 1 : 0][qp.rem(isTransformSkip)];
     1697  const double errorScale = xGetErrScaleCoeff(TU::needsSqrt2Scale(tu, compID), width, height, qp.rem(isTransformSkip), maxLog2TrDynamicRange, channelBitDepth, isTransformSkip);
     1698  TrQuantParams trQuantParams;
     1699  trQuantParams.rightShift = (IQUANT_SHIFT - ((isTransformSkip ? 0 : transformShift) + qp.per(isTransformSkip)));
     1700  trQuantParams.qScale = g_invQuantScales[needsSqrt2Scale ? 1 : 0][qp.rem(isTransformSkip)];
     1701
     1702  const TCoeff entropyCodingMaximum = (1 << maxLog2TrDynamicRange) - 1;
     1703
     1704  uint32_t coeffLevels[3];
     1705  double   coeffLevelError[4];
     1706
     1707  CoeffCodingContext cctx(tu, compID, tu.cs->picHeader->getSignDataHidingEnabledFlag(), true);
     1708
     1709
     1710  const int sbSizeM1 = (1 << cctx.log2CGSize()) - 1;
     1711  double    baseCost = 0;
     1712  uint32_t  goRiceParam = 0;
     1713  uint16_t  goRiceZero = 0;
     1714
     1715
     1716  double *costSigSubBlock = m_pdCostCoeffGroupSig;
     1717  memset(costSigSubBlock, 0, (maxNumCoeff >> cctx.log2CGSize()) * sizeof(double));
     1718
     1719  const int sbNum = width * height >> cctx.log2CGSize();
     1720  int scanPos;
     1721  coeffGroupRDStats rdStats;
     1722
     1723  bool anySigCG = false;
    16291724
     1725  int maxCtxBins = (cctx.maxNumCoeff() * 7) >> 2;
     1726  cctx.setNumCtxBins(maxCtxBins);
     1727
     1728  for (int sbId = 0; sbId < sbNum; sbId++)
     1729  {
     1730    cctx.initSubblock(sbId);
     1731
     1732    int noCoeffCoded = 0;
     1733    baseCost = 0.0;
     1734    memset(&rdStats, 0, sizeof(coeffGroupRDStats));
     1735    rdStats.iNumSbbCtxBins = 0;
     1736
     1737    for (int scanPosInSB = 0; scanPosInSB <= sbSizeM1; scanPosInSB++)
     1738    {
     1739      int lastPosCoded = sbSizeM1;
     1740      scanPos = cctx.minSubPos() + scanPosInSB;
     1741      //===== quantization =====
     1742      uint32_t blkPos = cctx.blockPos(scanPos);
     1743
     1744      const int posX = cctx.posX(scanPos);
     1745      const int posY = cctx.posY(scanPos);
     1746      const int posS = (1 == dirMode) ? posX : posY;
     1747      const int posNb = (1 == dirMode) ? (posX - 1) + posY * coeffs.stride : posX + (posY - 1) * coeffs.stride;
     1748      TCoeff predCoeff = (0 != posS) ? m_fullCoeff[posNb] : 0;
     1749
     1750      // set coeff
     1751      const int64_t          tmpLevel = int64_t(abs(srcCoeff[blkPos] - predCoeff)) * quantisationCoefficient;
     1752      const Intermediate_Int levelDouble = (Intermediate_Int)std::min<int64_t>(tmpLevel, std::numeric_limits<Intermediate_Int>::max() - (Intermediate_Int(1) << (qBits - 1)));
     1753      uint32_t roundAbsLevel = std::min<uint32_t>(uint32_t(entropyCodingMaximum), uint32_t((levelDouble + (Intermediate_Int(1) << (qBits - 1))) >> qBits));
     1754      uint32_t minAbsLevel = (roundAbsLevel > 1 ? roundAbsLevel - 1 : 1);
     1755
     1756      m_testedLevels = 0;
     1757      coeffLevels[m_testedLevels++] = roundAbsLevel;
     1758
     1759      if (minAbsLevel != roundAbsLevel)
     1760        coeffLevels[m_testedLevels++] = minAbsLevel;
     1761
     1762      double dErr = double(levelDouble);
     1763      coeffLevelError[0] = dErr * dErr * errorScale;
     1764
     1765      costCoeff0[scanPos] = coeffLevelError[0];
     1766      blockUncodedCost += costCoeff0[scanPos];
     1767      dstCoeff[blkPos] = coeffLevels[0];
     1768
     1769
     1770      //===== coefficient level estimation =====
     1771      unsigned    ctxIdSig = cctx.sigCtxIdAbsTS(scanPos, dstCoeff);
     1772      uint32_t    cLevel;
     1773      const BinFracBits fracBitsPar = fracBits.getFracBitsArray(cctx.parityCtxIdAbsTS());
     1774
     1775      goRiceParam = cctx.templateAbsSumTS(scanPos, dstCoeff);
     1776      goRiceZero = g_auiGoRicePosCoeff0(0, goRiceParam);
     1777      const uint8_t     sign = srcCoeff[blkPos] - predCoeff < 0 ? 1 : 0;
     1778      unsigned gt1CtxId = cctx.lrg1CtxIdAbsTS(scanPos, dstCoeff, dirMode);
     1779      const BinFracBits fracBitsGr1 = fracBits.getFracBitsArray(gt1CtxId);
     1780
     1781      const BinFracBits fracBitsGr2 = fracBits.getFracBitsArray(gt1CtxId); // need to implement
     1782
     1783      DTRACE_COND((dstCoeff[blkPos] != 0), g_trace_ctx, D_RDOQ_MORE, " uiCtxSig=%d", ctxIdSig);
     1784
     1785      const BinFracBits fracBitsSig = fracBits.getFracBitsArray(ctxIdSig);
     1786
     1787
     1788
     1789
     1790      bool lastCoeff = false; //
     1791      if (scanPosInSB == lastPosCoded && noCoeffCoded == 0)
     1792      {
     1793        lastCoeff = true;
     1794      }
     1795      int rightPixel, belowPixel;
     1796      cctx.neighTS(rightPixel, belowPixel, scanPos, dstCoeff);
     1797      int numUsedCtxBins = 0;
     1798      cLevel = xGetCodedLevelTSPredRRC(costCoeff[scanPos], costCoeff0[scanPos], costSig[scanPos], levelDouble, qBits, errorScale, coeffLevels, coeffLevelError,
     1799        &fracBitsSig, fracBitsPar, cctx, fracBits, fracBitsGr1, fracBitsGr2, goRiceZero, goRiceParam, lastCoeff, extendedPrecision, maxLog2TrDynamicRange, numUsedCtxBins);
     1800      cctx.decimateNumCtxBins(numUsedCtxBins);
     1801      rdStats.iNumSbbCtxBins += numUsedCtxBins;
     1802      if (cLevel > 0)
     1803      {
     1804        noCoeffCoded++;
     1805      }
     1806      dstCoeff[blkPos] = cLevel;
     1807
     1808      if (sign)
     1809      {
     1810        dstCoeff[blkPos] = -dstCoeff[blkPos];
     1811      }
     1812      xDequantSample(m_fullCoeff[blkPos], dstCoeff[blkPos], trQuantParams);
     1813      m_fullCoeff[blkPos] += predCoeff;
     1814
     1815      baseCost += costCoeff[scanPos];
     1816      rdStats.d64SigCost += costSig[scanPos];
     1817
     1818      if (dstCoeff[blkPos])
     1819      {
     1820        cctx.setSigGroup();
     1821        rdStats.d64CodedLevelandDist += costCoeff[scanPos] - costSig[scanPos];
     1822        rdStats.d64UncodedDist += costCoeff0[scanPos];
     1823      }
     1824    } //end for (iScanPosinCG)
     1825
     1826    if (!cctx.isSigGroup())
     1827    {
     1828      const BinFracBits fracBitsSigGroup = fracBits.getFracBitsArray(cctx.sigGroupCtxId(true));
     1829      baseCost += xGetRateSigCoeffGroup(fracBitsSigGroup, 0) - rdStats.d64SigCost;
     1830      costSigSubBlock[cctx.subSetId()] = xGetRateSigCoeffGroup(fracBitsSigGroup, 0);
     1831      cctx.increaseNumCtxBins(rdStats.iNumSbbCtxBins); // skip sub-block
     1832    }
     1833    else if (sbId != sbNum - 1 || anySigCG)
     1834    {
     1835      // rd-cost if SigCoeffGroupFlag = 0, initialization
     1836      double costZeroSB = baseCost;
     1837      const BinFracBits fracBitsSigGroup = fracBits.getFracBitsArray(cctx.sigGroupCtxId(true));
     1838
     1839      baseCost += xGetRateSigCoeffGroup(fracBitsSigGroup, 1);
     1840      costZeroSB += xGetRateSigCoeffGroup(fracBitsSigGroup, 0);
     1841      costSigSubBlock[cctx.subSetId()] = xGetRateSigCoeffGroup(fracBitsSigGroup, 1);
     1842
     1843      costZeroSB += rdStats.d64UncodedDist;         // distortion for resetting non-zero levels to zero levels
     1844      costZeroSB -= rdStats.d64CodedLevelandDist;   // distortion and level cost for keeping all non-zero levels
     1845      costZeroSB -= rdStats.d64SigCost;             // sig cost for all coeffs, including zero levels and non-zerl levels
     1846
     1847      if (costZeroSB < baseCost)
     1848      {
     1849        cctx.resetSigGroup();
     1850        baseCost = costZeroSB;
     1851        costSigSubBlock[cctx.subSetId()] = xGetRateSigCoeffGroup(fracBitsSigGroup, 0);
     1852        cctx.increaseNumCtxBins(rdStats.iNumSbbCtxBins); // skip sub-block
     1853
     1854        for (int scanPosInSB = 0; scanPosInSB <= sbSizeM1; scanPosInSB++)
     1855        {
     1856          scanPos = cctx.minSubPos() + scanPosInSB;
     1857          uint32_t blkPos = cctx.blockPos(scanPos);
     1858
     1859          const int posX = cctx.posX(scanPos);
     1860          const int posY = cctx.posY(scanPos);
     1861          const int posS = (1 == dirMode) ? posX : posY;
     1862          const int posNb = (1 == dirMode) ? (posX - 1) + posY * coeffs.stride : posX + (posY - 1) * coeffs.stride;
     1863          m_fullCoeff[scanPos] = (0 != posS) ? m_fullCoeff[posNb] : 0;
     1864
     1865          if (dstCoeff[blkPos])
     1866          {
     1867            dstCoeff[blkPos] = 0;
     1868            costCoeff[scanPos] = costCoeff0[scanPos];
     1869            costSig[scanPos] = 0;
     1870          }
     1871        }
     1872      }
     1873      else
     1874      {
     1875        anySigCG = true;
     1876      }
     1877    }
     1878  }
     1879
     1880  //===== estimate last position =====
     1881  for (int scanPos = 0; scanPos < maxNumCoeff; scanPos++)
     1882  {
     1883    int blkPos = cctx.blockPos(scanPos);
     1884    TCoeff level = dstCoeff[blkPos];
     1885    absSum += abs(level);
     1886  }
     1887}
     1888
     1889#endif
    16301890void QuantRDOQ::xDequantSample(TCoeff& pRes, TCoeff& coeff, const TrQuantParams& trQuantParams)
    16311891{
    16321892  // xDequant
    inline int QuantRDOQ::xGetICRateTS( const uint32_t absLevel, 
    19282188  return rate;
    19292189}
    19302190
     2191#if ALI_FIX_RDOQ_TS_BUG
     2192inline uint32_t QuantRDOQ::xGetCodedLevelTSPredRRC(double&            rd64CodedCost,
     2193  double&            rd64CodedCost0,
     2194  double&            rd64CodedCostSig,
     2195  Intermediate_Int    levelDouble,
     2196  int                 qBits,
     2197  double              errorScale,
     2198  uint32_t coeffLevels[],
     2199  double coeffLevelError[],
     2200  const BinFracBits* fracBitsSig,
     2201  const BinFracBits& fracBitsPar,
     2202  CoeffCodingContext& cctx,
     2203  const FracBitsAccess& fracBitsAccess,
     2204  // const BinFracBits& fracBitsSign,
     2205  const BinFracBits& fracBitsGt1,
     2206  const BinFracBits& fracBitsGt2,
     2207  //  const uint8_t      sign,
     2208   // int                rightPixel,
     2209   // int                belowPixel,
     2210  uint16_t      goRiceZero,
     2211  uint16_t           ricePar,
     2212  bool               isLast,
     2213  bool               useLimitedPrefixLength,
     2214  const int          maxLog2TrDynamicRange
     2215  , int&               numUsedCtxBins
     2216) const
     2217{
     2218  double currCostSig = 0;
     2219  uint32_t   bestAbsLevel = 0;
     2220  numUsedCtxBins = 0;
     2221  int numBestCtxBin = 0;
     2222  if (!isLast && coeffLevels[0] < 3)
     2223  {
     2224    if (cctx.numCtxBins() >= 4)
     2225      rd64CodedCostSig = xGetRateSigCoef(*fracBitsSig, 0);
     2226    else
     2227      rd64CodedCostSig = xGetICost(1 << SCALE_BITS);
     2228    rd64CodedCost = rd64CodedCost0 + rd64CodedCostSig;
     2229    if (cctx.numCtxBins() >= 4)
     2230      numUsedCtxBins++;
     2231    if (coeffLevels[0] == 0)
     2232    {
     2233      return bestAbsLevel;
     2234    }
     2235  }
     2236  else
     2237  {
     2238    rd64CodedCost = MAX_DOUBLE;
     2239  }
     2240
     2241  if (!isLast)
     2242  {
     2243    if (cctx.numCtxBins() >= 4)
     2244      currCostSig = xGetRateSigCoef(*fracBitsSig, 1);
     2245    else
     2246      currCostSig = xGetICost(1 << SCALE_BITS);
     2247    if (coeffLevels[0] >= 3 && cctx.numCtxBins() >= 4)
     2248      numUsedCtxBins++;
     2249  }
     2250
     2251  for (int errorInd = 1; errorInd <= m_testedLevels; errorInd++)
     2252  {
     2253    int absLevel = coeffLevels[errorInd - 1];
     2254    double dErr = 0.0;
     2255    dErr = double(levelDouble - (Intermediate_Int(absLevel) << qBits));
     2256    coeffLevelError[errorInd] = dErr * dErr * errorScale;
     2257    int modAbsLevel = absLevel;
     2258    int numCtxBins = 0;
     2259    double dCurrCost = coeffLevelError[errorInd] + xGetICost(xGetICRateTSRRC(modAbsLevel, fracBitsPar, cctx, fracBitsAccess, fracBitsGt1, fracBitsGt2, numCtxBins, goRiceZero, ricePar, useLimitedPrefixLength, maxLog2TrDynamicRange));
     2260
     2261    if (cctx.numCtxBins() >= 4)
     2262      dCurrCost += currCostSig; // if cctx.numCtxBins < 4, xGetICRateTS return rate including sign cost. dont need to add any more
     2263
     2264    if (dCurrCost < rd64CodedCost)
     2265    {
     2266      bestAbsLevel = absLevel;
     2267      rd64CodedCost = dCurrCost;
     2268      rd64CodedCostSig = currCostSig;
     2269      numBestCtxBin = numCtxBins;
     2270    }
     2271  }
     2272  numUsedCtxBins += numBestCtxBin;
     2273  return bestAbsLevel;
     2274}
     2275
     2276inline int QuantRDOQ::xGetICRateTSRRC(const uint32_t            absLevel,
     2277  const BinFracBits&        fracBitsPar,
     2278  const CoeffCodingContext& cctx,
     2279  const FracBitsAccess&     fracBitsAccess,
     2280  const BinFracBits&        fracBitsGt1,
     2281  const BinFracBits&        fracBitsGt2,
     2282  int&                      numCtxBins,
     2283  const uint16_t            goRiceZero,
     2284  const uint16_t            ricePar,
     2285  const bool                useLimitedPrefixLength,
     2286  const int                 maxLog2TrDynamicRange) const
     2287{
     2288
     2289  if (cctx.numCtxBins() < 4) // Full by-pass coding
     2290  {
     2291    int rate = absLevel ? (1 << SCALE_BITS) : 0; // 1 bit to signal sign of non-zero
     2292
     2293    uint32_t  symbol = (absLevel == 0 ? goRiceZero : absLevel <= goRiceZero ? absLevel - 1 : absLevel);
     2294
     2295
     2296    uint32_t length;
     2297    const int threshold = COEF_REMAIN_BIN_REDUCTION;
     2298    if (symbol < (threshold << ricePar))
     2299    {
     2300      length = symbol >> ricePar;
     2301      rate += (length + 1 + ricePar) << SCALE_BITS;
     2302    }
     2303    else if (useLimitedPrefixLength)
     2304    {
     2305      const uint32_t maximumPrefixLength = (32 - (COEF_REMAIN_BIN_REDUCTION + maxLog2TrDynamicRange));
     2306
     2307      uint32_t prefixLength = 0;
     2308      uint32_t suffix = (symbol >> ricePar) - COEF_REMAIN_BIN_REDUCTION;
     2309
     2310      while ((prefixLength < maximumPrefixLength) && (suffix > ((2 << prefixLength) - 2)))
     2311      {
     2312        prefixLength++;
     2313      }
     2314
     2315      const uint32_t suffixLength = (prefixLength == maximumPrefixLength) ? (maxLog2TrDynamicRange - ricePar) : (prefixLength + 1/*separator*/);
     2316
     2317      rate += (COEF_REMAIN_BIN_REDUCTION + prefixLength + suffixLength + ricePar) << SCALE_BITS;
     2318    }
     2319    else
     2320    {
     2321      length = ricePar;
     2322      symbol = symbol - (threshold << ricePar);
     2323      while (symbol >= (1 << length))
     2324      {
     2325        symbol -= (1 << (length++));
     2326      }
     2327      rate += (threshold + length + 1 - ricePar + length) << SCALE_BITS;
     2328    }
     2329
     2330    return rate;
     2331  }
     2332
     2333
     2334  int iRate = int(xGetIEPRate()); // cost of sign bit
     2335  const uint32_t cthres = 4;
     2336  if (absLevel >= cthres)
     2337  {
     2338    uint32_t symbol = (absLevel - cthres) >> 1;
     2339    uint32_t length;
     2340    const int threshold = COEF_REMAIN_BIN_REDUCTION;
     2341    if (symbol < (threshold << ricePar))
     2342    {
     2343      length = symbol >> ricePar;
     2344      iRate += (length + 1 + ricePar) << SCALE_BITS;
     2345    }
     2346    else if (useLimitedPrefixLength)
     2347    {
     2348      const uint32_t maximumPrefixLength = (32 - (COEF_REMAIN_BIN_REDUCTION + maxLog2TrDynamicRange));
     2349
     2350      uint32_t prefixLength = 0;
     2351      uint32_t suffix = (symbol >> ricePar) - COEF_REMAIN_BIN_REDUCTION;
     2352
     2353      while ((prefixLength < maximumPrefixLength) && (suffix > ((2 << prefixLength) - 2)))
     2354      {
     2355        prefixLength++;
     2356      }
     2357
     2358      const uint32_t suffixLength = (prefixLength == maximumPrefixLength) ? (maxLog2TrDynamicRange - ricePar) : (prefixLength + 1/*separator*/);
     2359
     2360      iRate += (COEF_REMAIN_BIN_REDUCTION + prefixLength + suffixLength + ricePar) << SCALE_BITS;
     2361    }
     2362    else
     2363    {
     2364      length = ricePar;
     2365      symbol = symbol - (threshold << ricePar);
     2366      while (symbol >= (1 << length))
     2367      {
     2368        symbol -= (1 << (length++));
     2369      }
     2370      iRate += (threshold + length + 1 - ricePar + length) << SCALE_BITS;
     2371    }
     2372
     2373    iRate += fracBitsGt1.intBits[1];
     2374    iRate += fracBitsPar.intBits[(absLevel - 2) & 1];
     2375    iRate += fracBitsGt2.intBits[1];
     2376
     2377    numCtxBins += 3;
     2378  }
     2379  else if (absLevel == 1)
     2380  {
     2381    iRate += fracBitsGt1.intBits[0];
     2382    numCtxBins++;
     2383  }
     2384  else if (absLevel == 2)
     2385  {
     2386    iRate += fracBitsGt1.intBits[1];
     2387    iRate += fracBitsPar.intBits[0];
     2388    iRate += fracBitsGt2.intBits[0];
     2389    numCtxBins += 3;
     2390  }
     2391  else if (absLevel == 3)
     2392  {
     2393    iRate += fracBitsGt1.intBits[1];
     2394    iRate += fracBitsPar.intBits[1];
     2395    iRate += fracBitsGt2.intBits[0];
     2396    numCtxBins += 3;
     2397  }
     2398  else
     2399  {
     2400    iRate = 0;
     2401  }
     2402  return  iRate;
     2403}
     2404
     2405
     2406
     2407#endif
    19312408//! \}
  • source/Lib/CommonLib/QuantRDOQ.h

    diff --git a/source/Lib/CommonLib/QuantRDOQ.h b/source/Lib/CommonLib/QuantRDOQ.h
    index ec3ca1c6..0b031875 100644
    a b public: 
    6666  // quantization
    6767  void quant                ( TransformUnit &tu, const ComponentID &compID, const CCoeffBuf &pSrc, TCoeff &uiAbsSum, const QpParam &cQP, const Ctx& ctx );
    6868  void forwardRDPCM         ( TransformUnit &tu, const ComponentID &compID, const CCoeffBuf &pSrc, TCoeff &uiAbsSum, const QpParam &cQP, const Ctx &ctx );
    69 
     69#if ALI_FIX_RDOQ_TS_BUG
     70  void forwardRDPCMRRC(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf &pSrc, TCoeff &uiAbsSum, const QpParam &cQP, const Ctx &ctx);
     71#endif
    7072private:
    7173  double* xGetErrScaleCoeffSL            ( uint32_t list, uint32_t sizeX, uint32_t sizeY, int qp ) { return m_errScale[sizeX][sizeY][list][qp]; };  //!< get Error Scale Coefficent
    7274  double  xGetErrScaleCoeff              ( const bool needsSqrt2, SizeType width, SizeType height, int qp, const int maxLog2TrDynamicRange, const int channelBitDepth, bool bTransformSkip);
    private: 
    151153                              const uint16_t            ricePar,
    152154                              const bool                useLimitedPrefixLength,
    153155                              const int                 maxLog2TrDynamicRange  ) const;
     156
     157
     158#if ALI_FIX_RDOQ_TS_BUG
     159  inline uint32_t xGetCodedLevelTSPredRRC(double&            rd64CodedCost,
     160    double&            rd64CodedCost0,
     161    double&            rd64CodedCostSig,
     162    Intermediate_Int    levelDouble,
     163    int                 qBits,
     164    double              errorScale,
     165    uint32_t coeffLevels[],
     166    double coeffLevelError[],
     167    const BinFracBits* fracBitsSig,
     168    const BinFracBits& fracBitsPar,
     169    CoeffCodingContext& cctx,
     170    const FracBitsAccess& fracBitsAccess,
     171    //    const BinFracBits& fracBitsSign,
     172    const BinFracBits& fracBitsGt1,
     173    const BinFracBits& fracBitsGt2,
     174    //    const uint8_t      sign,
     175    //    int                rightPixel,
     176    //    int                belowPixel,
     177    uint16_t      goRiceZero,
     178    uint16_t           ricePar,
     179    bool               isLast,
     180    bool               useLimitedPrefixLength,
     181    const int          maxLog2TrDynamicRange
     182    , int&               numUsedCtxBins
     183  ) const;
     184
     185  inline int xGetICRateTSRRC(const uint32_t            absLevel,
     186    const BinFracBits&        fracBitsPar,
     187    const CoeffCodingContext& cctx,
     188    const FracBitsAccess&     fracBitsAccess,
     189    const BinFracBits&        fracBitsGt1,
     190    const BinFracBits&        fracBitsGt2,
     191    int&                      numCtxBins,
     192    const uint16_t             goRiceZero,
     193    const uint16_t            ricePar,
     194    const bool                useLimitedPrefixLength,
     195    const int                 maxLog2TrDynamicRange) const;
     196
     197
     198#endif
     199
    154200private:
    155201  bool    m_isErrScaleListOwner;
    156202
  • source/Lib/CommonLib/TypeDef.h

    diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h
    index 4a811c8e..9dbd7f9c 100644
    a b  
    5050#include <assert.h>
    5151#include <cassert>
    5252
     53#define ALI_ALWAYS_USE_REGULAR_RESIDUAL_CODING            1
     54#define ALI_FIX_RDOQ_TS_BUG                               1
     55
     56
    5357#define CABAC_RETRAIN                                     1 // CABAC retraining based on VTM8rc1
    5458
    5559#define JVET_Q0044_SLICE_IDX_WITH_SUBPICS                 1 // JVET-Q0044: slice index with subpictures
  • source/Lib/EncoderLib/EncSlice.cpp

    diff --git a/source/Lib/EncoderLib/EncSlice.cpp b/source/Lib/EncoderLib/EncSlice.cpp
    index 2521811f..33314cf7 100644
    a b void EncSlice::initEncSlice(Picture* pcPic, const int pocLast, const int pocCurr 
    344344  }
    345345  rpcSlice->setPOC( pocCurr );
    346346#if JVET_Q0089_SLICE_LOSSLESS_CODING_CHROMA_BDPCM
     347#if ALI_ALWAYS_USE_REGULAR_RESIDUAL_CODING
     348  rpcSlice->setTSResidualCodingDisabledFlag(true);
     349#else
    347350  if( m_pcCfg->getCostMode() == COST_LOSSLESS_CODING )
    348351  {
    349352    rpcSlice->setTSResidualCodingDisabledFlag(true);
    void EncSlice::initEncSlice(Picture* pcPic, const int pocLast, const int pocCurr 
    353356    rpcSlice->setTSResidualCodingDisabledFlag(false);
    354357  }
    355358#endif
     359#endif
    356360
    357361#if SHARP_LUMA_DELTA_QP
    358362  pcPic->fieldPic = isField;