Bug 1689978 - Fill out-of-bounds texelFetchPtr with zeroes rather tha…

…n clamping. r=jrmuizel Attempting to just clamping the base address returning from texelFetchPtr might be causing some crashes in the case the texture is actually smaller than the offset area. Instead, switch out the sampler with a zero buffer to ensure we have something sane to sample without having to do slow bounds checking on everything. Differential Revision: https://phabricator.services.mozilla.com/D132508 [ghsync] From https://hg.mozilla.org/mozilla-central/rev/2d1f7196e88a3aee7d084225f4381b6952b545ce
servo · Dec 1, 2021 · b02f081 · b02f081
1 parent 97c12c7
commit b02f081
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 54 deletions.
diff --git a/glsl-to-cxx/src/lib.rs b/glsl-to-cxx/src/lib.rs
@@ -2603,24 +2603,9 @@ fn define_texel_fetch_ptr(
     offsets: &hir::TexelFetchOffsets,
 ) {
     show_indent(state);
-    let ptr_type = if let hir::SymDecl::Global(_, _, ty, _) = &sampler_sym.decl {
-        if symbol_run_class(&base_sym.decl, state.vector_mask) == hir::RunClass::Scalar {
-            match ty.kind {
-                hir::TypeKind::Sampler2D
-                | hir::TypeKind::Sampler2DRect => "vec4_scalar*",
-                hir::TypeKind::ISampler2D => "ivec4_scalar*",
-                _ => panic!(),
-            }
-        } else {
-            "I32"
-        }
-    } else {
-        panic!();
-    };
     write!(
         state,
-        "{} {}_{}_fetch = texelFetchPtr({}, {}, {}, {}, {}, {});\n",
-        ptr_type,
+        "auto {}_{}_fetch = texelFetchPtr({}, {}, {}, {}, {}, {});\n",
         sampler_sym.name,
         base_sym.name,
         sampler_sym.name,

diff --git a/swgl/src/gl.cc b/swgl/src/gl.cc
@@ -1123,9 +1123,7 @@ GLenum GetError() {
 
 // Sets the error status to out-of-memory to indicate that a buffer
 // or texture re-allocation failed.
-static void out_of_memory() {
-  ctx->last_error = GL_OUT_OF_MEMORY;
-}
+static void out_of_memory() { ctx->last_error = GL_OUT_OF_MEMORY; }
 
 static const char* const extensions[] = {
     "GL_ARB_blend_func_extended",
@@ -1176,6 +1174,12 @@ void GetIntegerv(GLenum pname, GLint* params) {
     case GL_MINOR_VERSION:
       params[0] = 2;
       break;
+    case GL_MIN_PROGRAM_TEXEL_OFFSET:
+      params[0] = 0;
+      break;
+    case GL_MAX_PROGRAM_TEXEL_OFFSET:
+      params[0] = MAX_TEXEL_OFFSET;
+      break;
     default:
       debugf("unhandled glGetIntegerv parameter %x\n", pname);
       assert(false);
@@ -2823,7 +2827,7 @@ void DestroyContext(Context* c) {
   delete c;
 }
 
-size_t ReportMemory(Context *ctx, size_t (*size_of_op)(void*)) {
+size_t ReportMemory(Context* ctx, size_t (*size_of_op)(void*)) {
   size_t size = 0;
   if (ctx) {
     for (auto& t : ctx->textures) {

diff --git a/swgl/src/gl_defs.h b/swgl/src/gl_defs.h
@@ -131,6 +131,8 @@ typedef intptr_t GLintptr;
 #define GL_MAX_TEXTURE_IMAGE_UNITS 0x8872
 #define GL_MAX_TEXTURE_SIZE 0x0D33
 #define GL_MAX_ARRAY_TEXTURE_LAYERS 0x88FF
+#define GL_MIN_PROGRAM_TEXEL_OFFSET 0x8904
+#define GL_MAX_PROGRAM_TEXEL_OFFSET 0x8905
 
 #define GL_VERTEX_SHADER 0x8B31
 #define GL_FRAGMENT_SHADER 0x8B30

diff --git a/swgl/src/texture.h b/swgl/src/texture.h
@@ -160,11 +160,13 @@ vec4 texelFetchR16(S sampler, ivec2 P) {
   return vec4(fetchOffsetsR16(sampler, offset), 0.0f, 0.0f, 1.0f);
 }
 
-template <typename S>
-SI vec4 fetchOffsetsFloat(S sampler, I32 offset) {
-  return pixel_float_to_vec4(
-      *(Float*)&sampler->buf[offset.x], *(Float*)&sampler->buf[offset.y],
-      *(Float*)&sampler->buf[offset.z], *(Float*)&sampler->buf[offset.w]);
+SI vec4 fetchOffsetsFloat(const uint32_t* buf, I32 offset) {
+  return pixel_float_to_vec4(*(Float*)&buf[offset.x], *(Float*)&buf[offset.y],
+                             *(Float*)&buf[offset.z], *(Float*)&buf[offset.w]);
+}
+
+SI vec4 fetchOffsetsFloat(samplerCommon* sampler, I32 offset) {
+  return fetchOffsetsFloat(sampler->buf, offset);
 }
 
 vec4 texelFetchFloat(sampler2D sampler, ivec2 P) {
@@ -307,11 +309,13 @@ vec4 texelFetch(sampler2DRect sampler, ivec2 P) {
   }
 }
 
-template <typename S>
-SI ivec4 fetchOffsetsInt(S sampler, I32 offset) {
-  return pixel_int_to_ivec4(
-      *(I32*)&sampler->buf[offset.x], *(I32*)&sampler->buf[offset.y],
-      *(I32*)&sampler->buf[offset.z], *(I32*)&sampler->buf[offset.w]);
+SI ivec4 fetchOffsetsInt(const uint32_t* buf, I32 offset) {
+  return pixel_int_to_ivec4(*(I32*)&buf[offset.x], *(I32*)&buf[offset.y],
+                            *(I32*)&buf[offset.z], *(I32*)&buf[offset.w]);
+}
+
+SI ivec4 fetchOffsetsInt(samplerCommon* sampler, I32 offset) {
+  return fetchOffsetsInt(sampler->buf, offset);
 }
 
 ivec4 texelFetch(isampler2D sampler, ivec2 P, int lod) {
@@ -329,43 +333,70 @@ ivec4_scalar texelFetch(isampler2D sampler, ivec2_scalar P, int lod) {
   return *(ivec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
 }
 
-SI vec4_scalar* texelFetchPtr(sampler2D sampler, ivec2_scalar P, int min_x,
-                              int max_x, int min_y, int max_y) {
-  P.x = min(max(P.x, -min_x), int(sampler->width) - 1 - max_x);
-  P.y = min(max(P.y, -min_y), int(sampler->height) - 1 - max_y);
+constexpr int MAX_TEXEL_OFFSET = 8;
+
+// Fill texelFetchOffset outside the valid texture bounds with zeroes. The
+// stride will be set to 0 so that only one row of zeroes is needed.
+static const uint32_t
+    zeroFetchBuf[MAX_TEXEL_OFFSET * sizeof(Float) / sizeof(uint32_t)] = {0};
+
+struct FetchScalar {
+  const uint32_t* buf;
+  uint32_t stride;
+};
+
+template <typename S>
+SI FetchScalar texelFetchPtr(S sampler, ivec2_scalar P, int min_x, int max_x,
+                             int min_y, int max_y) {
+  assert(max_x < MAX_TEXEL_OFFSET);
+  if (P.x < -min_x || P.x >= int(sampler->width) - max_x || P.y < -min_y ||
+      P.y >= int(sampler->height) - max_y) {
+    return FetchScalar{zeroFetchBuf, 0};
+  }
+  return FetchScalar{&sampler->buf[P.x * 4 + P.y * sampler->stride],
+                     sampler->stride};
+}
+
+SI vec4_scalar texelFetchUnchecked(sampler2D sampler, FetchScalar ptr, int x,
+                                   int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32F);
-  return (vec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
+  return *(vec4_scalar*)&ptr.buf[x * 4 + y * ptr.stride];
 }
 
-SI ivec4_scalar* texelFetchPtr(isampler2D sampler, ivec2_scalar P, int min_x,
-                               int max_x, int min_y, int max_y) {
-  P.x = min(max(P.x, -min_x), int(sampler->width) - 1 - max_x);
-  P.y = min(max(P.y, -min_y), int(sampler->height) - 1 - max_y);
+SI ivec4_scalar texelFetchUnchecked(isampler2D sampler, FetchScalar ptr, int x,
+                                    int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32I);
-  return (ivec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
+  return *(ivec4_scalar*)&ptr.buf[x * 4 + y * ptr.stride];
 }
 
-template <typename S>
-SI I32 texelFetchPtr(S sampler, ivec2 P, int min_x, int max_x, int min_y,
-                     int max_y) {
-  P.x = clampCoord(P.x, int(sampler->width) - max_x, -min_x);
-  P.y = clampCoord(P.y, int(sampler->height) - max_y, -min_y);
-  return P.x * 4 + P.y * sampler->stride;
-}
+struct FetchVector {
+  const uint32_t* buf;
+  I32 offset;
+  uint32_t stride;
+};
 
-template <typename S, typename P>
-SI P texelFetchUnchecked(S sampler, P* ptr, int x, int y = 0) {
-  return ptr[x + y * (sampler->stride >> 2)];
+template <typename S>
+SI FetchVector texelFetchPtr(S sampler, ivec2 P, int min_x, int max_x,
+                             int min_y, int max_y) {
+  assert(max_x < MAX_TEXEL_OFFSET);
+  if (test_any(P.x < -min_x || P.x >= int(sampler->width) - max_x ||
+               P.y < -min_y || P.y >= int(sampler->height) - max_y)) {
+    return FetchVector{zeroFetchBuf, I32(0), 0};
+  }
+  return FetchVector{sampler->buf, P.x * 4 + P.y * sampler->stride,
+                     sampler->stride};
 }
 
-SI vec4 texelFetchUnchecked(sampler2D sampler, I32 offset, int x, int y = 0) {
+SI vec4 texelFetchUnchecked(sampler2D sampler, FetchVector ptr, int x,
+                            int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32F);
-  return fetchOffsetsFloat(sampler, offset + (x * 4 + y * sampler->stride));
+  return fetchOffsetsFloat(&ptr.buf[x * 4 + y * ptr.stride], ptr.offset);
 }
 
-SI ivec4 texelFetchUnchecked(isampler2D sampler, I32 offset, int x, int y = 0) {
+SI ivec4 texelFetchUnchecked(isampler2D sampler, FetchVector ptr, int x,
+                             int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32I);
-  return fetchOffsetsInt(sampler, offset + (x * 4 + y * sampler->stride));
+  return fetchOffsetsInt(&ptr.buf[x * 4 + y * ptr.stride], ptr.offset);
 }
 
 #define texelFetchOffset(sampler, P, lod, offset) \