Do not overflow/underflow intermediate predictions

Author: kmilos

src/librawspeed/decompressors/AbstractLJpegDecompressor.h

@@ -187,9 +187,9 @@ class AbstractLJpegDecompressor : public AbstractDecompressor {
 
   template <int N_COMP>
   [[nodiscard]] [[nodiscard]] [[nodiscard]] __attribute__((pure))
-  std::array<uint16_t, N_COMP>
+  std::array<int32_t, N_COMP>
   getInitialPredictors() const {
-    std::array<uint16_t, N_COMP> pred;
+    std::array<int32_t, N_COMP> pred;
     if (frame.prec < (Pt + 1)) {
       ThrowRDE("Invalid precision (%u) and point transform (%u) combination!",
                frame.prec, Pt);

src/librawspeed/decompressors/LJpegDecompressor.cpp

@@ -173,7 +173,10 @@ template <int N_COMP, bool WeirdWidth> void LJpegDecompressor::decodeN() {
 
   auto ht = getHuffmanTables<N_COMP>();
   auto pred = getInitialPredictors<N_COMP>();
-  uint16_t* predNext = pred.data();
+  std::array<uint16_t, N_COMP> predNext;
+  for (int i = 0; i != N_COMP; ++i) {
+    predNext[i] = uint16_t(pred);
+  }
 
   BitPumpJPEG bitStream(input);
 
@@ -191,9 +194,9 @@ template <int N_COMP, bool WeirdWidth> void LJpegDecompressor::decodeN() {
   for (unsigned row = 0; row < h; ++row) {
     unsigned col = 0;
 
-    copy_n(predNext, N_COMP, pred.data());
-    // the predictor for the next line is the start of this line
-    predNext = &img(row, col);
+    for (int i = 0; i != N_COMP; ++i) {
+      pred[i] = predNext[i];
+    }
     // the predictor mode is always horizontal on the first line
     uint32_t predMode = row == 0 ? 1 : predictorMode;
 
@@ -204,32 +207,32 @@ template <int N_COMP, bool WeirdWidth> void LJpegDecompressor::decodeN() {
     for (; col < N_COMP * fullBlocks; col += N_COMP) {
       for (int i = 0; i != N_COMP; ++i) {
         img(row, col + i) =
-            uint16_t(pred[i] + ht[i]->decodeDifference(bitStream));
+            uint16_t(pred[i] + uint16_t(ht[i]->decodeDifference(bitStream)));
         if (col < N_COMP * (fullBlocks - 1)) {
-          int predA = img(row, col + i);
-          int predB = predMode > 1 ? img(row - 1, col + N_COMP + i) : 0;
-          int predC = predMode > 1 ? img(row - 1, col + i) : 0;
+          int32_t predA = img(row, col + i);
+          int32_t predB = predMode > 1 ? img(row - 1, col + N_COMP + i) : 0;
+          int32_t predC = predMode > 1 ? img(row - 1, col + i) : 0;
           switch (predMode) {
           case 1:
-            pred[i] = uint16_t(predA);
+            pred[i] = predA;
             break;
           case 2:
-            pred[i] = uint16_t(predB);
+            pred[i] = predB;
             break;
           case 3:
-            pred[i] = uint16_t(predC);
+            pred[i] = predC;
             break;
           case 4:
-            pred[i] = uint16_t(predA + predB - predC);
+            pred[i] = predA + predB - predC;
             break;
           case 5:
-            pred[i] = uint16_t(predA + ((predB - predC) >> 1));
+            pred[i] = predA + ((predB - predC) >> 1);
             break;
           case 6:
-            pred[i] = uint16_t(predB + ((predA - predC) >> 1));
+            pred[i] = predB + ((predA - predC) >> 1);
             break;
           case 7:
-            pred[i] = uint16_t((predA + predB) >> 1);
+            pred[i] = (predA + predB) >> 1;
             break;
           default:
             ThrowRDE("Unsupported predictor mode: %u", predMode);
@@ -238,6 +241,11 @@ template <int N_COMP, bool WeirdWidth> void LJpegDecompressor::decodeN() {
       }
     }
 
+    // the predictor for the next line is the start of this decoded line
+    for (int i = 0; i != N_COMP; ++i) {
+      predNext[i] = img(row, i);
+    }
+
     // Sometimes we also need to consume one more block, and produce part of it.
     if /*constexpr*/ (WeirdWidth) {
       // FIXME: evaluate i-cache implications due to this being compile-time.
@@ -250,36 +258,36 @@ template <int N_COMP, bool WeirdWidth> void LJpegDecompressor::decodeN() {
       unsigned c = 0;
       for (; c < trailingPixels; ++c) {
         // Continue predictor update skipped at last full block
-        int predA = img(row, col - N_COMP + c);
-        int predB = predMode > 1 ? img(row - 1, col + c) : 0;
-        int predC = predMode > 1 ? img(row - 1, col - N_COMP + c) : 0;
+        int32_t predA = img(row, col - N_COMP + c);
+        int32_t predB = predMode > 1 ? img(row - 1, col + c) : 0;
+        int32_t predC = predMode > 1 ? img(row - 1, col - N_COMP + c) : 0;
         switch (predMode) {
         case 1:
-          pred[c] = uint16_t(predA);
+          pred[c] = predA;
           break;
         case 2:
-          pred[c] = uint16_t(predB);
+          pred[c] = predB;
           break;
         case 3:
-          pred[c] = uint16_t(predC);
+          pred[c] = predC;
           break;
         case 4:
-          pred[c] = uint16_t(predA + predB - predC);
+          pred[c] = predA + predB - predC;
           break;
         case 5:
-          pred[c] = uint16_t(predA + ((predB - predC) >> 1));
+          pred[c] = predA + ((predB - predC) >> 1);
           break;
         case 6:
-          pred[c] = uint16_t(predB + ((predA - predC) >> 1));
+          pred[c] = predB + ((predA - predC) >> 1);
           break;
         case 7:
-          pred[c] = uint16_t((predA + predB) >> 1);
+          pred[c] = (predA + predB) >> 1;
           break;
         default:
           ThrowRDE("Unsupported predictor mode: %u", predMode);
         }
         img(row, col + c) =
-            uint16_t(pred[c] + ht[c]->decodeDifference(bitStream));
+            uint16_t(pred[c] + uint16_t(ht[c]->decodeDifference(bitStream)));
       }
       // Discard the rest of the block.
       assert(c < N_COMP);