[SPIRV] Always provide original source to debug (#6085)

This PR improves the experience for debugging tools that consume SPIR-V
produced by passing in-memory strings to dxcompiler.dll. When dxc.exe is
used, OpSource and OpDebugSource are populated by reading original
source from the filesystem. dxcompiler.dll users instead find their
input string is preprocessed, which is confusing behavior.
Contrary to the leading comment in the code this change modifies,
debugging tools do not want preprocessed source. Such tools show and
allow editing of include files. If everything is pasted into a single
source string, finding what to edit is difficult. Editing the
preprocessed source has the additional complication that a user would
have to edit also the #line markers, which is hard to get right
manually. In the preprocessed source, all comments are lost, which makes
it hard to read the code. Finally in preprocessed source, all macros and
code under conditional compilation are lost. That will make it
impossible to modify those parts in an edit.

Co-authored-by: Steve Urquhart <stevur01@ntd.nintendo.com>

include/dxc/Support/SPIRVOptions.h

@@ -99,6 +99,9 @@ struct SpirvCodeGenOptions {
   // String representation of all command line options and input file.
   std::string clOptions;
   std::string inputFile;
+
+  // String representation of original source
+  std::string origSource;
 };
 
 } // namespace spirv

tools/clang/lib/SPIRV/EmitVisitor.cpp

@@ -133,7 +133,9 @@ uint32_t getHeaderVersion(spv_target_env env) {
 
 // Read the file in |filePath| and returns its contents as a string.
 // This function will be used by DebugSource to get its source code.
-std::string ReadSourceCode(llvm::StringRef filePath) {
+std::string
+ReadSourceCode(llvm::StringRef filePath,
+               const clang::spirv::SpirvCodeGenOptions &spvOptions) {
   try {
     dxc::DxcDllSupport dllSupport;
     IFT(dllSupport.Initialize());
@@ -150,15 +152,21 @@ std::string ReadSourceCode(llvm::StringRef filePath) {
     return std::string(utf8Source->GetStringPointer(),
                        utf8Source->GetStringLength());
   } catch (...) {
-    // An exception has occured while reading the file
+    // An exception has occurred while reading the file
+    // return the original source (which may have been supplied directly)
+    if (!spvOptions.origSource.empty()) {
+      return spvOptions.origSource.c_str();
+    }
     return "";
   }
 }
 
 // Returns a vector of strings after chopping |inst| for the operand size
 // limitation of OpSource.
-llvm::SmallVector<std::string, 2> getChoppedSourceCode(SpirvSource *inst) {
+llvm::SmallVector<std::string, 2>
-  std::string text = ReadSourceCode(inst->getFile()->getString());
+getChoppedSourceCode(SpirvSource *inst,
+                     const clang::spirv::SpirvCodeGenOptions &spvOptions) {
+  std::string text = ReadSourceCode(inst->getFile()->getString(), spvOptions);
   if (text.empty()) {
     text = inst->getSource().str();
   }
@@ -654,7 +662,7 @@ bool EmitVisitor::visit(SpirvSource *inst) {
   // Chop up the source into multiple segments if it is too long.
   llvm::SmallVector<std::string, 2> choppedSrcCode;
   if (spvOptions.debugInfoSource && inst->hasFile()) {
-    choppedSrcCode = getChoppedSourceCode(inst);
+    choppedSrcCode = getChoppedSourceCode(inst, spvOptions);
     if (!choppedSrcCode.empty()) {
       // Note: in order to improve performance and avoid multiple copies, we
       // encode this (potentially large) string directly into the
@@ -1440,7 +1448,7 @@ void EmitVisitor::generateChoppedSource(uint32_t fileId,
   if (spvOptions.debugInfoSource) {
     std::string text = inst->getContent();
     if (text.empty())
-      text = ReadSourceCode(inst->getFile());
+      text = ReadSourceCode(inst->getFile(), spvOptions);
     if (!text.empty()) {
       // Maximum characters for DebugSource and DebugSourceContinued
       // OpString literal minus terminating null.
@@ -1481,7 +1489,7 @@ bool EmitVisitor::visit(SpirvDebugSource *inst) {
   // NonSemantic.Shader.DebugInfo.100 logic above can be used for both cases.
   uint32_t textId = 0;
   if (spvOptions.debugInfoSource) {
-    auto text = ReadSourceCode(inst->getFile());
+    auto text = ReadSourceCode(inst->getFile(), spvOptions);
     if (!text.empty())
       textId = getOrCreateOpStringId(text);
   }

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -613,13 +613,21 @@ public:
                               nullptr, &pSourceEncoding));
 
 #ifdef ENABLE_SPIRV_CODEGEN
-      // We want to embed the preprocessed source code in the final SPIR-V if
+      // We want to embed the original source code in the final SPIR-V if
-      // debug information is enabled. Therefore, we invoke Preprocess() here
+      // debug information is enabled. But the compiled source requires
+      // pre-seeding with #line directives. We invoke Preprocess() here
       // first for such case. Then we invoke the compilation process over the
-      // preprocessed source code, so that line numbers are consistent with the
+      // preprocessed source code.
-      // embedded source code.
       if (!isPreprocessing && opts.GenSPIRV && opts.DebugInfo &&
           !opts.SpirvOptions.debugInfoVulkan) {
+        // Convert source code encoding
+        CComPtr<IDxcBlobUtf8> pOrigUtf8Source;
+        IFC(hlsl::DxcGetBlobAsUtf8(pSourceEncoding, m_pMalloc,
+                                   &pOrigUtf8Source));
+        opts.SpirvOptions.origSource.assign(
+            static_cast<const char *>(pOrigUtf8Source->GetStringPointer()),
+            pOrigUtf8Source->GetStringLength());
+
         CComPtr<IDxcResult> pSrcCodeResult;
         std::vector<LPCWSTR> PreprocessArgs;
         PreprocessArgs.reserve(argCount + 1);

tools/clang/unittests/SPIRV/CodeGenSpirvTest.cpp

@@ -19,7 +19,7 @@ using ::testing::ContainsRegex;
 // This test is purely for demonstrating how to use `LibTest`, and does not
 // test anything specific in DXC itself.
 TEST_F(LibTest, SourceCodeWithoutFilePath) {
-  const std::string command(R"(// RUN: %dxc -T ps_6_0 -E PSMain -Zi)");
+  const std::string command(R"(// RUN: %dxc -T ps_6_0 -E PSMain)");
   const std::string code = command + R"(
 float4 PSMain(float4 color : COLOR) : SV_TARGET { return color; }
 )";
@@ -27,4 +27,18 @@ float4 PSMain(float4 color : COLOR) : SV_TARGET { return color; }
   EXPECT_THAT(spirv, ContainsRegex("%PSMain = OpFunction"));
 }
 
+// This test demonstrates that in-memory source is transmitted
+// to OpSource faithfully
+TEST_F(LibTest, InlinedCodeWithDebugTest) {
+  const std::string command(R"(// RUN: %dxc -T ps_6_0 -E PSMain -Zi)");
+  const std::string code = command + R"(
+float4 PSMain(float4 color : COLOR) : SV_TARGET { return color; }
+)";
+  std::string spirv = compileCodeAndGetSpirvAsm(code);
+  EXPECT_THAT(
+      spirv,
+      ContainsRegex(
+          "OpSource HLSL 600 %4 \"// RUN: %dxc -T ps_6_0 -E PSMain -Zi"));
+}
+
 } // namespace