Ticket #59: CholFix_Bms_Tag1.patch

source/Lib/CommonLib/TypeDef.h

@@ -203 +203 @@
 // Tool Switches
 // ====================================================================================================================
 
-
+#define WHU_CHOL_FIX                                                         1
 // This can be enabled by the makefile
 #ifndef RExt__HIGH_BIT_DEPTH_SUPPORT
 #define RExt__HIGH_BIT_DEPTH_SUPPORT                      0 ///< 0 (default) use data type definitions for 8-10 bit video, 1 = use larger data types to allow for up to 16-bit video (originally developed as part of N0188)

source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp

@@ -2128 +2128 @@
   int count=0;
   while(quantCoeffSum!=targetCoeffSumInt && count < 10)
   {
+#if WHU_CHOL_FIX
+    count++;
+#endif
     if (quantCoeffSum>targetCoeffSumInt)
     {
       diff=quantCoeffSum-targetCoeffSumInt;
@@ -3382 +3385 @@
 #define REG_SQR          0.0000001
 
 //Find filter coeff related
+#if WHU_CHOL_FIX
+Int EncAdaptiveLoopFilter::gnsLDLTDec(Double **inpMatr, Double *x, Double *rhs, Int noEq)
+{
+
+  Int size = noEq;
+  Double LMatr[m_MAX_SQR_FILT_LENGTH][m_MAX_SQR_FILT_LENGTH];
+  Double DMatr[m_MAX_SQR_FILT_LENGTH];
+
+  Double aux[m_MAX_SQR_FILT_LENGTH];
+
+  for (Int j = 0; j < size; j++)
+  {
+    Double sumM2 = 0.0;
+    for (Int k = 0; k < j; k++)
+    {
+      sumM2 += DMatr[k] * LMatr[j][k] * LMatr[j][k];
+    }
+    DMatr[j] = inpMatr[j][j] - sumM2;
+
+    for (Int i = j + 1; i < size; i++)
+    {
+      Double sumM1 = 0.0;
+      for (Int k = 0; k < j; k++)
+      {
+        sumM1 += LMatr[i][k] * DMatr[k] * LMatr[j][k];
+      }
+
+      if (fabs(DMatr[j]) < REG_SQR)
+      {
+        LMatr[i][j] = 0;
+        //return (0);
+      }
+      else
+      {
+        LMatr[i][j] = (inpMatr[i][j] - sumM1) / DMatr[j];
+      }
+    }
+  }
+
+  /* Solve L*aux = rhs for aux */
+  aux[0] = rhs[0];
+  for (Int i =1; i < size; i++)
+  {
+    Double tmp = 0.0;
+    for (Int k = 0; k < i; k++)
+    {
+      tmp += LMatr[i][k] * aux[k];
+    }
+    aux[i] = rhs[i] - tmp;
+  }
+
+  /* Solve DLT*x = aux for x */
+  if (fabs(DMatr[size - 1] ) < REG_SQR)
+  {
+    x[size - 1] = 0;
+    //return (0);
+  }
+  else
+  {
+    x[size - 1] = aux[size - 1] / DMatr[size - 1];
+  }
+
+  for (Int i = size -2; i >=0; i--)
+  {
+    Double tmp = 0.0;
+    if (fabs(DMatr[i]) < REG_SQR)
+    {
+      x[i] = 0;
+      //return (0);
+    }
+    else
+    {
+      for (Int k = i + 1; k < size; k++)
+      {
+        tmp += LMatr[k][i] * x[k];
+      }
+      x[i] = aux[i] / DMatr[i] - tmp;
+    }
+  }
+
+  return(1); /* Signal that  factorization is successfully performed */
+}
+#endif
+
 Int EncAdaptiveLoopFilter::gnsCholeskyDec(double **inpMatr, double outMatr[m_MAX_SQR_FILT_LENGTH][m_MAX_SQR_FILT_LENGTH], int noEq)
 {
   int
@@ -3466 +3553 @@
 
 Int EncAdaptiveLoopFilter::gnsSolveByChol( Double **LHS, Double *rhs, Double *x, Int noEq)
 {
+#if WHU_CHOL_FIX
+  int  singular;
+#else
   double aux[m_MAX_SQR_FILT_LENGTH];     /* Auxiliary vector */
   double U[m_MAX_SQR_FILT_LENGTH][m_MAX_SQR_FILT_LENGTH];    /* Upper triangular Cholesky factor of LHS */
   int  i, singular;          /* Looping variable */
+#endif
 
   /* The equation to be solved is LHSx = rhs */
 
+#if WHU_CHOL_FIX
+  singular = gnsLDLTDec(LHS, x, rhs, noEq);
+#else
   /* Compute upper triangular U such that U'*U = LHS */
   if(gnsCholeskyDec(LHS, U, noEq)) /* If Cholesky decomposition has been successful */
   {
@@ -3505 +3599 @@
     /* Solve U*x = aux for x */
     gnsBacksubstitution(U, aux, noEq, x);
   }
+#endif
   return(singular);
 }
 

source/Lib/EncoderLib/EncAdaptiveLoopFilter.h

@@ -218 +218 @@
   Int  gnsSolveByChol              ( Double **LHS, double *rhs, double *x, int noEq);
   Void gnsBacksubstitution         ( Double R[m_MAX_SQR_FILT_LENGTH][m_MAX_SQR_FILT_LENGTH], Double z[m_MAX_SQR_FILT_LENGTH], Int R_size, Double A[m_MAX_SQR_FILT_LENGTH]);
   Void gnsTransposeBacksubstitution( Double U[m_MAX_SQR_FILT_LENGTH][m_MAX_SQR_FILT_LENGTH], Double rhs[], Double x[], Int order);
+#if WHU_CHOL_FIX
+  Int  gnsLDLTDec(Double **inpMatr, Double *x, double *rhs, Int noEq);
+#endif
   Int  gnsCholeskyDec              ( Double **inpMatr, double outMatr[m_MAX_SQR_FILT_LENGTH][m_MAX_SQR_FILT_LENGTH], Int noEq);