Metal: Fix undefined behavior of depth write

Writing to an unbound depth attachment is undefined behavior in Metal.
Fix this by emitting a function constant to guard depth buffer use
in fragment shaders.

Bug: angleproject:6865
Change-Id: Id7c10d0aeb349aacfe09c397bc292a71199ab50a
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3380304
Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>
Reviewed-by: Kenneth Russell <kbr@chromium.org>
Commit-Queue: Kenneth Russell <kbr@chromium.org>

include/GLSLANG/ShaderLang.h

@@ -895,6 +895,9 @@ extern const char kCoverageMaskEnabledConstName[];
 
 // Specialization constant to emulate rasterizer discard.
 extern const char kRasterizerDiscardEnabledConstName[];
+
+// Specialization constant to enable depth write in fragment shaders.
+extern const char kDepthWriteEnabledConstName[];
 }  // namespace mtl
 
 // For backends that use glslang (the Vulkan shader compiler), i.e. Vulkan and Metal, call these to

src/compiler.gni

@@ -382,6 +382,8 @@ angle_translator_lib_metal_sources = [
   "src/compiler/translator/TranslatorMetalDirect/EmitMetal.h",
   "src/compiler/translator/TranslatorMetalDirect/FixTypeConstructors.cpp",
   "src/compiler/translator/TranslatorMetalDirect/FixTypeConstructors.h",
+  "src/compiler/translator/TranslatorMetalDirect/GuardFragDepthWrite.cpp",
+  "src/compiler/translator/TranslatorMetalDirect/GuardFragDepthWrite.h",
   "src/compiler/translator/TranslatorMetalDirect/HoistConstants.cpp",
   "src/compiler/translator/TranslatorMetalDirect/HoistConstants.h",
   "src/compiler/translator/TranslatorMetalDirect/IdGen.cpp",

src/compiler/translator/TranslatorMetal.cpp

@@ -37,6 +37,7 @@ namespace mtl
 /** extern */
 const char kCoverageMaskEnabledConstName[]      = "ANGLECoverageMaskEnabled";
 const char kRasterizerDiscardEnabledConstName[] = "ANGLERasterizerDisabled";
+const char kDepthWriteEnabledConstName[]        = "ANGLEDepthWriteEnabled";
 }  // namespace mtl
 
 namespace

src/compiler/translator/TranslatorMetalDirect/EmitMetal.cpp

@@ -1058,7 +1058,8 @@ void GenMetalTraverser::emitFieldDeclaration(const TField &field,
             break;
 
         case TQualifier::EvqFragDepth:
-            mOut << " [[depth(any)]]";
+            mOut << " [[depth(any), function_constant(" << sh::mtl::kDepthWriteEnabledConstName
+                 << ")]]";
             break;
 
         case TQualifier::EvqSampleMask:

src/compiler/translator/TranslatorMetalDirect/GuardFragDepthWrite.cpp

@@ -0,0 +1,73 @@
+//
+// Copyright 2022 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// GuardFragDepthWrite: Guards use of frag depth behind the function constant
+// ANGLEDepthWriteEnabled to ensure it is only used when a valid depth buffer
+// is bound.
+
+#include "compiler/translator/TranslatorMetalDirect/GuardFragDepthWrite.h"
+#include "compiler/translator/TranslatorMetalDirect/AstHelpers.h"
+#include "compiler/translator/tree_util/BuiltIn.h"
+#include "compiler/translator/tree_util/IntermRebuild.h"
+
+using namespace sh;
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace
+{
+
+class Rewriter : public TIntermRebuild
+{
+  public:
+    Rewriter(TCompiler &compiler) : TIntermRebuild(compiler, false, true) {}
+
+    PostResult visitBinaryPost(TIntermBinary &node) override
+    {
+        if (TIntermSymbol *leftSymbolNode = node.getLeft()->getAsSymbolNode())
+        {
+            if (leftSymbolNode->getType().getQualifier() == TQualifier::EvqFragDepth)
+            {
+                // This transformation leaves the tree in an inconsistent state by using a variable
+                // that's defined in text, outside of the knowledge of the AST.
+                // FIXME(jcunningham): remove once function constants (specconst) are implemented
+                // with the metal translator.
+                mCompiler.disableValidateVariableReferences();
+
+                TSymbolTable *symbolTable = &mCompiler.getSymbolTable();
+
+                // Create kDepthWriteEnabled variable reference.
+                TType *boolType = new TType(EbtBool);
+                boolType->setQualifier(EvqConst);
+                TVariable *depthWriteEnabledVar = new TVariable(
+                    symbolTable, sh::ImmutableString(sh::mtl::kDepthWriteEnabledConstName),
+                    boolType, SymbolType::AngleInternal);
+
+                TIntermBlock *innerif = new TIntermBlock;
+                innerif->appendStatement(&node);
+
+                TIntermSymbol *depthWriteEnabled = new TIntermSymbol(depthWriteEnabledVar);
+                TIntermIfElse *ifCall = new TIntermIfElse(depthWriteEnabled, innerif, nullptr);
+                return ifCall;
+            }
+        }
+
+        return node;
+    }
+};
+
+}  // anonymous namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool sh::GuardFragDepthWrite(TCompiler &compiler, TIntermBlock &root)
+{
+    Rewriter rewriter(compiler);
+    if (!rewriter.rebuildRoot(root))
+    {
+        return false;
+    }
+    return true;
+}

src/compiler/translator/TranslatorMetalDirect/GuardFragDepthWrite.h

@@ -0,0 +1,21 @@
+//
+// Copyright 2022 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_TRANSLATOR_TRANSLATORMETALDIRECT_GUARDFRAGDEPTHWRITE_H_
+#define COMPILER_TRANSLATOR_TRANSLATORMETALDIRECT_GUARDFRAGDEPTHWRITE_H_
+
+#include "compiler/translator/Compiler.h"
+#include "compiler/translator/TranslatorMetalDirect/ProgramPrelude.h"
+#include "compiler/translator/TranslatorMetalDirect/SymbolEnv.h"
+
+namespace sh
+{
+
+ANGLE_NO_DISCARD bool GuardFragDepthWrite(TCompiler &compiler, TIntermBlock &root);
+
+}  // namespace sh
+
+#endif  // COMPILER_TRANSLATOR_TRANSLATORMETALDIRECT_GUARDFRAGDEPTHWRITE_H_

src/compiler/translator/TranslatorMetalDirect/ProgramPrelude.cpp

@@ -1507,11 +1507,13 @@ PROGRAM_PRELUDE_DECLARE(functionConstants,
 #define ANGLE_SAMPLE_COMPARE_LOD_INDEX      1
 #define ANGLE_RASTERIZATION_DISCARD_INDEX   2
 #define ANGLE_COVERAGE_MASK_ENABLED_INDEX   3
+#define ANGLE_DEPTH_WRITE_ENABLED_INDEX     4
 
 constant bool ANGLEUseSampleCompareGradient [[function_constant(ANGLE_SAMPLE_COMPARE_GRADIENT_INDEX)]];
 constant bool ANGLEUseSampleCompareLod      [[function_constant(ANGLE_SAMPLE_COMPARE_LOD_INDEX)]];
 constant bool ANGLERasterizerDisabled       [[function_constant(ANGLE_RASTERIZATION_DISCARD_INDEX)]];
 constant bool ANGLECoverageMaskEnabled      [[function_constant(ANGLE_COVERAGE_MASK_ENABLED_INDEX)]];
+constant bool ANGLEDepthWriteEnabled        [[function_constant(ANGLE_DEPTH_WRITE_ENABLED_INDEX)]];
 )")
 
 PROGRAM_PRELUDE_DECLARE(texelFetch,
@@ -3951,7 +3953,8 @@ void ProgramPrelude::visitVariable(const Name &name, const TType &type)
     }
     else
     {
-        if (name.rawName() == sh::mtl::kRasterizerDiscardEnabledConstName)
+        if (name.rawName() == sh::mtl::kRasterizerDiscardEnabledConstName ||
+            name.rawName() == sh::mtl::kDepthWriteEnabledConstName)
         {
             functionConstants();
         }

src/libANGLE/renderer/metal/ProgramMtl.mm

@@ -675,10 +675,20 @@ angle::Result ProgramMtl::getSpecializedShader(ContextMtl *context,
                     [NSString stringWithUTF8String:sh::mtl::kCoverageMaskEnabledConstName];
             }
 
+            NSString *depthWriteEnabledStr =
+                [NSString stringWithUTF8String:sh::mtl::kDepthWriteEnabledConstName];
+
             funcConstants = [[MTLFunctionConstantValues alloc] init];
             [funcConstants setConstantValue:&emulateCoverageMask
                                        type:MTLDataTypeBool
                                    withName:coverageMaskEnabledStr];
+
+            MTLPixelFormat depthPixelFormat =
+                (MTLPixelFormat)renderPipelineDesc.outputDescriptor.depthAttachmentPixelFormat;
+            BOOL fragDepthWriteEnabled = depthPixelFormat != MTLPixelFormatInvalid;
+            [funcConstants setConstantValue:&fragDepthWriteEnabled
+                                       type:MTLDataTypeBool
+                                   withName:depthWriteEnabledStr];
         }
 
     }  // gl::ShaderType::Fragment

src/tests/angle_end2end_tests.gni

@@ -71,6 +71,7 @@ angle_end2end_tests_sources = [
   "gl_tests/ExternalWrapTest.cpp",
   "gl_tests/FenceSyncTests.cpp",
   "gl_tests/FloatingPointSurfaceTest.cpp",
+  "gl_tests/FragDepthTest.cpp",
   "gl_tests/FramebufferFetchTest.cpp",
   "gl_tests/FramebufferMixedSamplesTest.cpp",
   "gl_tests/FramebufferMultiviewTest.cpp",

src/tests/gl_tests/FragDepthTest.cpp

@@ -0,0 +1,123 @@
+//
+// Copyright 2022 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// FragDepthTest:
+//   Tests the correctness of gl_FragDepth usage.
+//
+
+#include "test_utils/ANGLETest.h"
+#include "test_utils/gl_raii.h"
+
+using namespace angle;
+
+class FragDepthTest : public ANGLETest
+{
+  protected:
+    struct FragDepthTestResources
+    {
+        GLuint program;
+        GLint depthLocation;
+        GLTexture colorTexture;
+        GLTexture depthTexture;
+        GLFramebuffer framebuffer;
+    };
+
+    FragDepthTestResources setupResources()
+    {
+
+        FragDepthTestResources resources;
+        // Writes a fixed depth value and green.
+        constexpr char kFS[] =
+            R"(#version 300 es
+            precision highp float;
+            layout(location = 0) out vec4 fragColor;
+            uniform float u_depth;
+            void main(){
+                gl_FragDepth = u_depth;
+                fragColor = vec4(0.0, 1.0, 0.0, 1.0);
+            })";
+
+        resources.program       = CompileProgram(essl3_shaders::vs::Simple(), kFS);
+        resources.depthLocation = glGetUniformLocation(resources.program, "u_depth");
+
+        glBindTexture(GL_TEXTURE_2D, resources.colorTexture);
+        glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, 1, 1);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+
+        glBindTexture(GL_TEXTURE_2D, resources.depthTexture);
+        glTexStorage2D(GL_TEXTURE_2D, 1, GL_DEPTH_COMPONENT32F, 1, 1);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+
+        glBindFramebuffer(GL_FRAMEBUFFER, resources.framebuffer);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
+                               resources.colorTexture, 0);
+
+        return resources;
+    }
+
+    void cleanupResources(FragDepthTestResources &resources) { glDeleteProgram(resources.program); }
+
+    void checkDepthWritten(float expectedDepth, float fsDepth, bool bindDepthBuffer)
+    {
+        FragDepthTestResources resources = setupResources();
+        ASSERT_NE(0u, resources.program);
+        ASSERT_NE(-1, resources.depthLocation);
+        ASSERT_GLENUM_EQ(GL_FRAMEBUFFER_COMPLETE, glCheckFramebufferStatus(GL_FRAMEBUFFER));
+        ASSERT_GL_NO_ERROR();
+
+        glBindFramebuffer(GL_FRAMEBUFFER, resources.framebuffer);
+        if (bindDepthBuffer)
+        {
+            glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D,
+                                   resources.depthTexture, 0);
+            EXPECT_GL_FRAMEBUFFER_COMPLETE(GL_FRAMEBUFFER);
+
+            // Clear to the expected depth so it will be compared to the FS depth with
+            // DepthFunc(GL_EQUAL)
+            glClearDepthf(expectedDepth);
+            glDepthFunc(GL_EQUAL);
+            glEnable(GL_DEPTH_TEST);
+        }
+        else
+        {
+            glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, 0, 0);
+            EXPECT_GL_FRAMEBUFFER_COMPLETE(GL_FRAMEBUFFER);
+            glDisable(GL_DEPTH_TEST);
+        }
+        glUseProgram(resources.program);
+
+        // Clear to red, the FS will write green on success
+        glClearColor(1.0f, 0.0f, 0.0f, 0.0f);
+        // Clear to the expected depth so it will be compared to the FS depth with
+        // DepthFunc(GL_EQUAL)
+
+        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+        glUniform1f(resources.depthLocation, fsDepth);
+
+        drawQuad(resources.program, "a_position", 0.0f);
+        EXPECT_GL_NO_ERROR();
+
+        EXPECT_PIXEL_COLOR_EQ(0, 0, GLColor::green);
+        cleanupResources(resources);
+    }
+};
+
+// Test that writing to gl_FragDepth works
+TEST_P(FragDepthTest, DepthBufferBound)
+{
+    checkDepthWritten(0.5f, 0.5f, true);
+}
+
+// Test that writing to gl_FragDepth with no depth buffer works.
+TEST_P(FragDepthTest, DepthBufferUnbound)
+{
+    // Depth test is disabled, so the expected depth should not matter.
+    checkDepthWritten(0.f, 0.5f, false);
+}
+
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(FragDepthTest);
+ANGLE_INSTANTIATE_TEST_ES3(FragDepthTest);