Ensure a sized format is used when creating MSAA textures. Closes baldurk#1623

* If the application used an unsized format like GL_DEPTH_COMPONENT to create
  the texture with glTexImage2DMultisample then that's legal, but if we promote
  it to glTextureStorage2DMultisampleEXT for DSA we can't continue to use that
  unsized format so we need to convert to sized by picking a sensible default.

Author: baldurk

renderdoc/driver/gl/gl_driver.cpp

@@ -2739,11 +2739,17 @@ void WrappedOpenGL::CreateTextureImage(GLuint tex, GLenum internalFormat, GLenum
 
   if(textype == eGL_TEXTURE_2D_MULTISAMPLE)
   {
+    // we need a sized format for storage functions
+    internalFormat = GetSizedFormat(internalFormat);
+
     GL.glTextureStorage2DMultisampleEXT(tex, textype, samples, internalFormat, width, height,
                                         GL_TRUE);
   }
   else if(textype == eGL_TEXTURE_2D_MULTISAMPLE_ARRAY)
   {
+    // we need a sized format for storage functions
+    internalFormat = GetSizedFormat(internalFormat);
+
     GL.glTextureStorage3DMultisampleEXT(tex, textype, samples, internalFormat, width, height, depth,
                                         GL_TRUE);
   }

renderdoc/driver/gl/gl_initstate.cpp

@@ -1515,7 +1515,8 @@ bool GLResourceManager::Serialise_InitialState(SerialiserType &ser, ResourceId i
                                       eGL_CLAMP_TO_EDGE);
 
             // must use immutable tex storage here, for MSAA<->Array copies
-            GL.glTextureStorage3DEXT(tex, eGL_TEXTURE_2D_ARRAY, 1, TextureState.internalformat,
+            GL.glTextureStorage3DEXT(tex, eGL_TEXTURE_2D_ARRAY, 1,
+                                     GetSizedFormat(TextureState.internalformat),
                                      TextureState.width, TextureState.height, copySlices);
 
             // read back from the array we prepared

renderdoc/driver/gl/gl_msaa_array_conv.cpp

@@ -110,6 +110,8 @@ void WrappedOpenGL::CopyTex2DMSToArray(GLuint &destArray, GLuint srcMS, GLint wi
 {
   const ArrayMSPrograms &arrms = GetArrayMS();
 
+  intFormat = GetSizedFormat(intFormat);
+
   // create temporary texture array, which we'll initialise to be the width/height in same format,
   // with the same number of array slices as multi samples.
   GL.glGenTextures(1, &destArray);
@@ -267,11 +269,11 @@ void WrappedOpenGL::CopyDepthTex2DMSToArray(GLuint &destArray, GLuint srcMS, GLi
       numStencil = 256;
       attach = eGL_DEPTH_STENCIL_ATTACHMENT;
       break;
-    case eGL_DEPTH:
+    case eGL_DEPTH_COMPONENT:
       numStencil = 1;
       attach = eGL_DEPTH_ATTACHMENT;
       break;
-    case eGL_STENCIL:
+    case eGL_STENCIL_INDEX:
       numStencil = 256;
       attach = eGL_STENCIL_ATTACHMENT;
       break;
@@ -341,6 +343,8 @@ void WrappedOpenGL::CopyArrayToTex2DMS(GLuint destMS, GLuint srcArray, GLint wid
 {
   WrappedOpenGL &drv = *this;
 
+  intFormat = GetSizedFormat(intFormat);
+
   const ArrayMSPrograms &arrms = GetArrayMS();
 
   if(!HasExt[ARB_compute_shader])
@@ -499,11 +503,11 @@ void WrappedOpenGL::CopyDepthArrayToTex2DMS(GLuint destMS, GLuint srcArray, GLin
       numStencil = 256;
       attach = eGL_DEPTH_STENCIL_ATTACHMENT;
       break;
-    case eGL_DEPTH:
+    case eGL_DEPTH_COMPONENT:
       numStencil = 1;
       attach = eGL_DEPTH_ATTACHMENT;
       break;
-    case eGL_STENCIL:
+    case eGL_STENCIL_INDEX:
       numStencil = 256;
       attach = eGL_STENCIL_ATTACHMENT;
       break;

renderdoc/driver/gl/gl_resources.cpp

@@ -455,7 +455,7 @@ GLenum GetBaseFormat(GLenum internalFormat)
     case eGL_STENCIL_INDEX4:
     case eGL_STENCIL_INDEX8:
     case eGL_STENCIL_INDEX16:
-    case eGL_STENCIL: return eGL_STENCIL;
+    case eGL_STENCIL: return eGL_STENCIL_INDEX;
     default: break;
   }
 
@@ -891,6 +891,28 @@ bool EmulateLuminanceFormat(GLuint tex, GLenum target, GLenum &internalFormat, G
   return true;
 }
 
+GLenum GetSizedFormat(GLenum internalformat)
+{
+  switch(internalformat)
+  {
+    case eGL_DEPTH_COMPONENT: internalformat = eGL_DEPTH_COMPONENT32F; break;
+    case eGL_DEPTH_STENCIL: internalformat = eGL_DEPTH32F_STENCIL8; break;
+    case eGL_STENCIL:
+    case eGL_STENCIL_INDEX: internalformat = eGL_STENCIL_INDEX8; break;
+    case eGL_RGBA: internalformat = eGL_RGBA8; break;
+    case eGL_RGBA_INTEGER: internalformat = eGL_RGBA8I; break;
+    case eGL_RGB: internalformat = eGL_RGB8; break;
+    case eGL_RGB_INTEGER: internalformat = eGL_RGB8I; break;
+    case eGL_RG: internalformat = eGL_RG8; break;
+    case eGL_RG_INTEGER: internalformat = eGL_RG8I; break;
+    case eGL_RED: internalformat = eGL_R8; break;
+    case eGL_RED_INTEGER: internalformat = eGL_R8I; break;
+    default: break;
+  }
+
+  return internalformat;
+}
+
 bool IsCompressedFormat(GLenum internalFormat)
 {
   switch(internalFormat)
@@ -979,7 +1001,7 @@ bool IsDepthStencilFormat(GLenum internalFormat)
 
   GLenum fmt = GetBaseFormat(internalFormat);
 
-  return (fmt == eGL_DEPTH_COMPONENT || fmt == eGL_STENCIL || fmt == eGL_DEPTH_STENCIL);
+  return (fmt == eGL_DEPTH_COMPONENT || fmt == eGL_STENCIL_INDEX || fmt == eGL_DEPTH_STENCIL);
 }
 
 bool IsUIntFormat(GLenum internalFormat)

renderdoc/driver/gl/gl_resources.h

@@ -37,6 +37,7 @@ void GetTextureSwizzle(GLuint tex, GLenum target, GLenum *swizzleRGBA);
 void SetTextureSwizzle(GLuint tex, GLenum target, const GLenum *swizzleRGBA);
 
 bool EmulateLuminanceFormat(GLuint tex, GLenum target, GLenum &internalFormat, GLenum &dataFormat);
+GLenum GetSizedFormat(GLenum internalFormat);
 
 inline void EmulateGLClamp(GLenum pname, GLenum param)
 {

renderdoc/driver/gl/wrappers/gl_texture_funcs.cpp

@@ -4485,6 +4485,9 @@ bool WrappedOpenGL::Serialise_glTextureStorage2DMultisampleEXT(SerialiserType &s
     GLenum dummy = eGL_NONE;
     bool emulated = EmulateLuminanceFormat(texture.name, target, internalformat, dummy);
 
+    // if we promoted glTexImage2DMultisample to storage, we need a sized format
+    internalformat = GetSizedFormat(internalformat);
+
     ResourceId liveId = GetResourceManager()->GetID(texture);
     m_Textures[liveId].width = width;
     m_Textures[liveId].height = height;
@@ -4660,6 +4663,9 @@ bool WrappedOpenGL::Serialise_glTextureStorage3DMultisampleEXT(SerialiserType &s
     GLenum dummy = eGL_NONE;
     bool emulated = EmulateLuminanceFormat(texture.name, target, internalformat, dummy);
 
+    // if we promoted glTexImage3DMultisample to storage, we need a sized format
+    internalformat = GetSizedFormat(internalformat);
+
     ResourceId liveId = GetResourceManager()->GetID(texture);
     m_Textures[liveId].width = width;
     m_Textures[liveId].height = height;