Work around line loop streaming bug.

This forces a hard limit on the buffer size we allocate from D3D11. It
can work around a D3D11 driver bug on NVIDIA where we would get an
invalid map pointer. This seemed to happen when the buffer sizes were
close to MAX_UINT.

Bug: chromium:943087
Change-Id: I64aa9c55cbb82015101262c19c72741c140964a5
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1531374
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>

src/libANGLE/renderer/d3d/d3d11/ResourceManager11.cpp

@@ -23,6 +23,11 @@ constexpr FLOAT kDebugColorInitClearValue[4] = {0.3f, 0.5f, 0.7f, 0.5f};
 constexpr FLOAT kDebugDepthInitValue         = 0.2f;
 constexpr UINT8 kDebugStencilInitValue       = 3;
 
+// A hard limit on buffer size. This works around a problem in the NVIDIA drivers where buffer sizes
+// close to MAX_UINT would give undefined results. The limit of MAX_UINT/2 should be generous enough
+// for almost any demanding application.
+constexpr UINT kMaximumBufferSizeHardLimit = std::numeric_limits<UINT>::max() >> 1;
+
 uint64_t ComputeMippedMemoryUsage(unsigned int width,
                                   unsigned int height,
                                   unsigned int depth,
@@ -109,6 +114,12 @@ HRESULT CreateResource(ID3D11Device *device,
                        const D3D11_SUBRESOURCE_DATA *initData,
                        ID3D11Buffer **buffer)
 {
+    // Force buffers to be limited to a fixed max size.
+    if (desc->ByteWidth > kMaximumBufferSizeHardLimit)
+    {
+        return E_OUTOFMEMORY;
+    }
+
     return device->CreateBuffer(desc, initData, buffer);
 }
 

src/libANGLE/renderer/d3d/d3d9/Renderer9.cpp

@@ -89,6 +89,11 @@ static void DrawPoints(IDirect3DDevice9 *device, GLsizei count, const void *indi
         device->DrawPrimitive(D3DPT_POINTLIST, indexValue, 1);
     }
 }
+
+// A hard limit on buffer size. This works around a problem in the NVIDIA drivers where buffer sizes
+// close to MAX_UINT would give undefined results. The limit of MAX_UINT/2 should be generous enough
+// for almost any demanding application.
+constexpr UINT kMaximumBufferSizeHardLimit = std::numeric_limits<UINT>::max() >> 1;
 }  // anonymous namespace
 
 Renderer9::Renderer9(egl::Display *display) : RendererD3D(display), mStateManager(this)
@@ -865,6 +870,12 @@ HRESULT Renderer9::createVertexBuffer(UINT Length,
                                       DWORD Usage,
                                       IDirect3DVertexBuffer9 **ppVertexBuffer)
 {
+    // Force buffers to be limited to a fixed max size.
+    if (Length > kMaximumBufferSizeHardLimit)
+    {
+        return E_OUTOFMEMORY;
+    }
+
     D3DPOOL Pool = getBufferPool(Usage);
     return mDevice->CreateVertexBuffer(Length, Usage, 0, Pool, ppVertexBuffer, nullptr);
 }
@@ -879,6 +890,12 @@ HRESULT Renderer9::createIndexBuffer(UINT Length,
                                      D3DFORMAT Format,
                                      IDirect3DIndexBuffer9 **ppIndexBuffer)
 {
+    // Force buffers to be limited to a fixed max size.
+    if (Length > kMaximumBufferSizeHardLimit)
+    {
+        return E_OUTOFMEMORY;
+    }
+
     D3DPOOL Pool = getBufferPool(Usage);
     return mDevice->CreateIndexBuffer(Length, Usage, Format, Pool, ppIndexBuffer, nullptr);
 }

src/tests/gl_tests/LineLoopTest.cpp

@@ -6,6 +6,8 @@
 
 #include "test_utils/ANGLETest.h"
 
+#include "test_utils/gl_raii.h"
+
 using namespace angle;
 
 class LineLoopTest : public ANGLETest
@@ -186,6 +188,22 @@ TEST_P(LineLoopTest, LineLoopUIntIndexBuffer)
     glDeleteBuffers(1, &buf);
 }
 
+// Tests an edge case with a very large line loop element count.
+// Disabled because it is slow and triggers an internal error.
+TEST_P(LineLoopTest, DISABLED_DrawArraysWithLargeCount)
+{
+    constexpr char kVS[] = "void main() { gl_Position = vec4(0); }";
+    constexpr char kFS[] = "void main() { gl_FragColor = vec4(0, 1, 0, 1); }";
+
+    ANGLE_GL_PROGRAM(program, kVS, kFS);
+    glUseProgram(program);
+    glDrawArrays(GL_LINE_LOOP, 0, 0x3FFFFFFE);
+    EXPECT_GL_ERROR(GL_OUT_OF_MEMORY);
+
+    glDrawArrays(GL_LINE_LOOP, 0, 0x1FFFFFFE);
+    EXPECT_GL_NO_ERROR();
+}
+
 // Use this to select which configurations (e.g. which renderer, which GLES major version) these
 // tests should be run against.
 ANGLE_INSTANTIATE_TEST(LineLoopTest,