Add out-of-memory checks to QImage

Some of the methods of QImage crashed on out-of-memory situation as
they referred to null pointer if creation of a new QImage instance had
failed. Many cases like this were already fixed by commit
a041e4e which was a fix for
QTBUG-1985, but few cases still remained. Now, all the lines creating a
new QImage instance in qimage.cpp have been gone through and sanity
checks have been added where it was necessary to avoid immediate
crashing.

Task-number: ou1cimx1#994957
Change-Id: I1f07e8890bc91de18af075be73b1a06c667f3776
Reviewed-by: Murray Read <[email protected]>
Reviewed-by: Pasi Pentikäinen <[email protected]>
(cherry picked from commit 35aff3e)

Author: Pauli Jarvinen

src/gui/image/qimage.cpp

@@ -3942,6 +3942,7 @@ static inline int closestMatch(QRgb pixel, const QVector<QRgb> &clut) {
 static QImage convertWithPalette(const QImage &src, QImage::Format format,
                                  const QVector<QRgb> &clut) {
     QImage dest(src.size(), format);
+    QIMAGE_SANITYCHECK_MEMORY(dest);
     dest.setColorTable(clut);
 
 #if !defined(QT_NO_IMAGE_TEXT)
@@ -4290,6 +4291,7 @@ QImage QImage::convertBitOrder(Endian bitOrder) const
         return *this;
 
     QImage image(d->width, d->height, d->format == Format_Mono ? Format_MonoLSB : Format_Mono);
+    QIMAGE_SANITYCHECK_MEMORY(image);
 
     const uchar *data = d->data;
     const uchar *end = data + d->nbytes;
@@ -4917,6 +4919,7 @@ QImage QImage::rgbSwapped() const
     case Format_MonoLSB:
     case Format_Indexed8:
         res = copy();
+        QIMAGE_SANITYCHECK_MEMORY(res);
         for (int i = 0; i < res.d->colortable.size(); i++) {
             QRgb c = res.d->colortable.at(i);
             res.d->colortable[i] = QRgb(((c << 16) & 0xff0000) | ((c >> 16) & 0xff) | (c & 0xff00ff00));
@@ -6172,6 +6175,10 @@ void QImage::setAlphaChannel(const QImage &alphaChannel)
 
     } else {
         const QImage sourceImage = alphaChannel.convertToFormat(QImage::Format_RGB32);
+        if (sourceImage.isNull()) {
+            qWarning("QImage::setAlphaChannel: out of memory");
+            return;
+        }
         const uchar *src_data = sourceImage.d->data;
         const uchar *dest_data = d->data;
         for (int y=0; y<h; ++y) {
@@ -6224,6 +6231,7 @@ QImage QImage::alphaChannel() const
     int h = d->height;
 
     QImage image(w, h, Format_Indexed8);
+    QIMAGE_SANITYCHECK_MEMORY(image);
     image.setColorCount(256);
 
     // set up gray scale table.
@@ -6253,6 +6261,7 @@ QImage QImage::alphaChannel() const
         QImage alpha32 = *this;
         if (d->format != Format_ARGB32 && d->format != Format_ARGB32_Premultiplied)
             alpha32 = convertToFormat(Format_ARGB32);
+        QIMAGE_SANITYCHECK_MEMORY(alpha32);
 
         const uchar *src_data = alpha32.d->data;
         uchar *dest_data = image.d->data;
@@ -6396,6 +6405,7 @@ static QImage smoothScaled(const QImage &source, int w, int h) {
 
 static QImage rotated90(const QImage &image) {
     QImage out(image.height(), image.width(), image.format());
+    QIMAGE_SANITYCHECK_MEMORY(out);
     if (image.colorCount() > 0)
         out.setColorTable(image.colorTable());
     int w = image.width();
@@ -6455,6 +6465,7 @@ static QImage rotated180(const QImage &image) {
 
 static QImage rotated270(const QImage &image) {
     QImage out(image.height(), image.width(), image.format());
+    QIMAGE_SANITYCHECK_MEMORY(out);
     if (image.colorCount() > 0)
         out.setColorTable(image.colorTable());
     int w = image.width();