Expand macros in root signature and semantic defines (#122)

This commit adds a standalone class for expanding macros and uses it
to for root signature and semantic define processing. Previously,
we were using the raw macro value instead of the expanded value
so something like:

    #define MYRS "DescriptorTable(SRV(t0))"
    #define RS   MYRS

would fail to compile the root signature with an entry point of
RS because it would get a root signature value of MYRS instead
of the expanded value. Similarly, for semantic defines we were
saving the non-expanded value into the bitcode.

This commit also adds a new flag that is used to specify that
the root signature should be read from a #define. So compiling
with `-rootsig-define RS` will read the root signature from
a `#define RS ...` in the source. The -roosig-define flag
takes precedence over a root signature annotation in the souce.

include/dxc/HLSL/HLSLExtensionsCodegenHelper.h

@@ -27,9 +27,10 @@ namespace hlsl {
 //  1. You can mark certain defines as "semantic" defines which
 //     will be preserved as metadata in the final DXIL.
 //  2. You can add new HLSL intrinsic functions.
+//  3. You can read a root signature from a custom define.
 //
 // This class provides an interface for generating the DXIL bitcode
-// needed for the two types of extensions above.
+// needed for the types of extensions above.
 //  
 class HLSLExtensionsCodegenHelper {
 public:
@@ -64,6 +65,16 @@ public:
   // Get the name to use for the dxil intrinsic function.
   virtual std::string GetIntrinsicName(unsigned opcode) = 0;
 
+  // Struct to hold a root signature that is read from a define.
+  struct CustomRootSignature {
+    std::string RootSignature;
+    unsigned  EncodedSourceLocation;
+    enum Status { NOT_FOUND = 0, FOUND };
+  };
+
+  // Get custom defined root signature.
+  virtual CustomRootSignature::Status GetCustomRootSignature(CustomRootSignature *out) = 0;
+
   // Virtual destructor.
   virtual ~HLSLExtensionsCodegenHelper() {};
 };

include/dxc/Support/ErrorCodes.h

@@ -91,4 +91,7 @@
 #define DXC_E_INCORRECT_ROOT_SIGNATURE                DXC_MAKE_HRESULT(DXC_SEVERITY_ERROR,FACILITY_DXC,(0x0014))
 
 // 0X80AA0015 - DXIL container is missing DebugInfo part.
-#define DXC_E_CONTAINER_MISSING_DEBUG                 DXC_MAKE_HRESULT(DXC_SEVERITY_ERROR,FACILITY_DXC,(0x0015))
+#define DXC_E_CONTAINER_MISSING_DEBUG                 DXC_MAKE_HRESULT(DXC_SEVERITY_ERROR,FACILITY_DXC,(0x0015))
+
+// 0X80AA0016 - Unexpected failure in macro expansion.
+#define DXC_E_MACRO_EXPANSION_FAILURE                 DXC_MAKE_HRESULT(DXC_SEVERITY_ERROR,FACILITY_DXC,(0x0016))

include/dxc/Support/HLSLOptions.h

@@ -105,6 +105,7 @@ public:
   llvm::StringRef PrivateSource; // OPT_setprivate
   llvm::StringRef RootSignatureSource; // OPT_setrootsignature
   llvm::StringRef VerifyRootSignatureSource; //OPT_verifyrootsignature
+  llvm::StringRef RootSignatureDefine; // OPT_rootsig_define
 
   bool AllResourcesBound; // OPT_all_resources_bound
   bool AstDump; // OPT_ast_dump

include/dxc/Support/HLSLOptions.td

@@ -225,6 +225,8 @@ def hlsl_version : Separate<["-", "/"], "HV">, Group<hlslcomp_Group>, Flags<[Cor
   HelpText<"HLSL version (Only supports 2016 for now)">;
 def no_warnings : Flag<["-", "/"], "no-warnings">, Group<hlslcomp_Group>, Flags<[CoreOption]>,
   HelpText<"Suppress warnings">;
+def rootsig_define : Separate<["-", "/"], "rootsig-define">, Group<hlslcomp_Group>, Flags<[CoreOption]>,
+  HelpText<"Read root signature from a #define">;
 
 //////////////////////////////////////////////////////////////////////////////
 // fxc-based flags that don't match those previously defined.

lib/DxcSupport/HLSLOptions.cpp

@@ -260,6 +260,7 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
   opts.PrivateSource = Args.getLastArgValue(OPT_setprivate);
   opts.RootSignatureSource = Args.getLastArgValue(OPT_setrootsignature);
   opts.VerifyRootSignatureSource = Args.getLastArgValue(OPT_verifyrootsignature);
+  opts.RootSignatureDefine = Args.getLastArgValue(OPT_rootsig_define);
 
   if (!opts.ForceRootSigVer.empty() && opts.ForceRootSigVer != "rootsig_1_0" &&
       opts.ForceRootSigVer != "rootsig_1_1") {

tools/clang/include/clang/Lex/HLSLMacroExpander.h

@@ -0,0 +1,63 @@
+//===--- HLSLMacroExpander.h - Standalone Macro expansion ------*- C++ -*-===//
+///////////////////////////////////////////////////////////////////////////////
+//                                                                           //
+// HLSLMacroExpander.h                                                       //
+// Copyright (C) Microsoft Corporation. All rights reserved.                 //
+// This file is distributed under the University of Illinois Open Source     //
+// License. See LICENSE.TXT for details.                                     //
+//                                                                           //
+//  This file defines utilites for expanding macros after lexing has         //
+//  completed. Normally, macros are expanded as part of the lexing           //
+//  phase and returned in an expanded form directly from the lexer.          //
+//  For hlsl we need to be able to expand macros after the fact to           //
+//  correctly capture semantic defines and root signature defines.           //
+//                                                                           //
+///////////////////////////////////////////////////////////////////////////////
+#ifndef LLVM_CLANG_LEX_HLSLMACROEXPANDER_H
+#define LLVM_CLANG_LEX_HLSLMACROEXPANDER_H
+
+#include "clang/Basic/SourceLocation.h"
+
+#include <string>
+#include <utility>
+
+namespace clang {
+  class Preprocessor;
+  class Token;
+  class MacroInfo;
+}
+
+namespace llvm {
+  class StringRef;
+}
+
+namespace hlsl {
+class MacroExpander {
+public:
+  // Options used during macro expansion.
+  enum Option : unsigned {
+    // Strip quotes from string literals. Enables concatenating adjacent
+    // string literals into a single value.
+    STRIP_QUOTES = 1 << 1,
+  };
+
+  // Constructor
+  MacroExpander(clang::Preprocessor &PP, unsigned options = 0);
+
+  // Expand the given macro into the output string.
+  // Returns true if macro was expanded successfully.
+  bool ExpandMacro(clang::MacroInfo *macro, std::string *out);
+
+
+  // Look in the preprocessor for a macro with the provided name.
+  // Return nullptr if the macro could not be found.
+  static clang::MacroInfo *FindMacroInfo(clang::Preprocessor &PP, llvm::StringRef macroName);
+
+private:
+  clang::Preprocessor &PP;
+  clang::FileID m_expansionFileId;
+  bool m_stripQuotes;
+};
+}
+
+#endif // header include guard

tools/clang/lib/CodeGen/CGHLSLMS.cpp

@@ -23,6 +23,7 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/HlslTypes.h"
 #include "clang/Frontend/CodeGenOptions.h"
+#include "clang/Lex/HLSLMacroExpander.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
@@ -3859,8 +3860,9 @@ void CGMSHLSLRuntime::FinishCodeGen() {
   // Do simple transform to make later lower pass easier.
   SimpleTransformForHLDXIR(m_pHLModule->GetModule());
 
-  // Add semantic defines for extensions if any are available.
+  // Handle lang extensions if provided.
   if (CGM.getCodeGenOpts().HLSLExtensionsCodegen) {
+    // Add semantic defines for extensions if any are available.
     HLSLExtensionsCodegenHelper::SemanticDefineErrorList errors =
       CGM.getCodeGenOpts().HLSLExtensionsCodegen->WriteSemanticDefines(m_pHLModule->GetModule());
 
@@ -3872,6 +3874,18 @@ void CGMSHLSLRuntime::FinishCodeGen() {
       unsigned DiagID = Diags.getCustomDiagID(level, "%0");
       Diags.Report(SourceLocation::getFromRawEncoding(error.Location()), DiagID) << error.Message();
     }
+
+    // Add root signature from a #define. Overrides root signature in function attribute.
+    {
+      using Status = HLSLExtensionsCodegenHelper::CustomRootSignature::Status;
+      HLSLExtensionsCodegenHelper::CustomRootSignature customRootSig;
+      Status status = CGM.getCodeGenOpts().HLSLExtensionsCodegen->GetCustomRootSignature(&customRootSig);
+      if (status == Status::FOUND) {
+          CompileRootSignature(customRootSig.RootSignature, Diags,
+                               SourceLocation::getFromRawEncoding(customRootSig.EncodedSourceLocation),
+                               rootSigVer, &m_pHLModule->GetRootSignature());
+      }
+    }
   }
 
 }

tools/clang/lib/Frontend/FrontendActions.cpp

@@ -17,6 +17,7 @@
 #include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HLSLMacroExpander.h"
 #include "clang/Lex/Pragma.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Parse/Parser.h"
@@ -733,21 +734,11 @@ void HLSLRootSignatureAction::ExecuteAction() {
 
   assert(rootSigMajor == 1 &&
            "else CGMSHLSLRuntime Constructor needs to be updated");
-  // Try to find HLSLRootSignatureMacro in macros.
-  MacroInfo *rootSigMI = nullptr;
-  for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();
-       I != E; ++I) {
-    auto *MD = I->second.getLatest();
-    if (MD && MD->isDefined()) {
-      if (I->first->getName() == HLSLRootSignatureMacro) {
-        rootSigMI = MD->getMacroInfo();
-        break;
-      }
-    }
-  }
 
+  // Try to find HLSLRootSignatureMacro in macros.
+  MacroInfo *rootSigMacro = hlsl::MacroExpander::FindMacroInfo(PP, HLSLRootSignatureMacro);
   DiagnosticsEngine &Diags = CI.getDiagnostics();
-  if (!rootSigMI) {
+  if (!rootSigMacro)  {
     std::string cannotFindMacro =
         "undeclared identifier " + HLSLRootSignatureMacro;
     SourceLocation SLoc = Tok.getLocation();
@@ -756,31 +747,18 @@ void HLSLRootSignatureAction::ExecuteAction() {
     return;
   }
 
-  SourceLocation SLoc = rootSigMI->getDefinitionLoc();
-
-  std::string rootSigStr;
-  llvm::raw_string_ostream rootSigOS(rootSigStr);
-
-  SmallString<128> SpellingBuffer;
-  for (const auto &T : rootSigMI->tokens()) {
-    if (T.hasLeadingSpace())
-      rootSigOS << ' ';
-
-    rootSigOS << PP.getSpelling(T, SpellingBuffer);
-  }
-  rootSigOS.flush();
-  StringRef StrRef = rootSigStr;
-
-  // Remove the "" on rootsig macro.
-  if (StrRef.size() >= 2) {
-    if (rootSigStr.front() == '"') {
-      if (rootSigStr.back() == '"') {
-        StrRef = StrRef.substr(1, StrRef.size()-2);
-      }
-    }
+  // Expand HLSLRootSignatureMacro.
+  SourceLocation SLoc = rootSigMacro->getDefinitionLoc();
+  std::string rootSigString;
+  hlsl::MacroExpander expander(PP, hlsl::MacroExpander::STRIP_QUOTES);
+  if (!expander.ExpandMacro(rootSigMacro, &rootSigString)) {
+    StringRef error("error expanding root signature macro");
+    ReportHLSLRootSigError(Diags, SLoc, error.data(), error.size());
+    return;
   }
 
-  clang::CompileRootSignature(StrRef, Diags, SLoc, rootSigVer, rootSigHandle.get());
+  // Compile the expanded root signature.
+  clang::CompileRootSignature(rootSigString, Diags, SLoc, rootSigVer, rootSigHandle.get());
 }
 
 std::unique_ptr<hlsl::RootSignatureHandle> HLSLRootSignatureAction::takeRootSigHandle() {

tools/clang/lib/Lex/CMakeLists.txt

@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS support)
 add_clang_library(clangLex
   HeaderMap.cpp
   HeaderSearch.cpp
+  HLSLMacroExpander.cpp
   Lexer.cpp
   LiteralSupport.cpp
   MacroArgs.cpp

tools/clang/lib/Lex/HLSLMacroExpander.cpp

@@ -0,0 +1,169 @@
+//===--- HLSLMacroExpander.cpp - Standalone Macro expansion -----*- C++ -*-===//
+//                                                                            //
+// HLSLMacroExpander.cpp                                                      //
+// Copyright (C) Microsoft Corporation. All rights reserved.                  //
+// This file is distributed under the University of Illinois Open Source      //
+// License. See LICENSE.TXT for details.                                      //
+//===----------------------------------------------------------------------===//
+//
+// This file implements the MacroExpander class.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Lex/HLSLMacroExpander.h"
+
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/ModuleMap.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PTHLexer.h"
+#include "clang/Lex/PTHManager.h"
+#include "clang/Lex/Token.h"
+#include "clang/Lex/TokenLexer.h"
+#include "llvm/ADT/StringRef.h"
+
+#include "dxc/Support/Global.h"
+using namespace clang;
+using namespace llvm;
+using namespace hlsl;
+
+MacroExpander::MacroExpander(Preprocessor &PP_, unsigned options)
+  : PP(PP_)
+  , m_expansionFileId()
+  , m_stripQuotes(false)
+{
+  if (options & STRIP_QUOTES)
+    m_stripQuotes = true;
+
+  // The preprocess requires a file to be on the lexing stack when we
+  // call ExpandMacro. We add an empty in-memory buffer that we use
+  // just for expanding macros.
+  std::unique_ptr<llvm::MemoryBuffer> SB = llvm::MemoryBuffer::getMemBuffer("", "<hlsl-semantic-defines>");
+  if (!SB) {
+    DXASSERT(false, "Cannot create macro expansion source buffer");
+    throw hlsl::Exception(DXC_E_MACRO_EXPANSION_FAILURE);
+  }
+
+  // Unfortunately, there is no api in the SourceManager to lookup a
+  // previously added file, so we have to add the empty file every time
+  // we expand macros. We could modify source manager to get/set the
+  // macro file id similar to the one we have for getPreambleFileID.
+  // Macros should only be expanded once (if needed for a root signature)
+  // or twice (for semantic defines) so adding an empty file every time
+  // is probably not a big deal.
+  m_expansionFileId = PP.getSourceManager().createFileID(std::move(SB));
+  if (m_expansionFileId.isInvalid()) {
+    DXASSERT(false, "Could not create FileID for macro expnasion?");
+    throw hlsl::Exception(DXC_E_MACRO_EXPANSION_FAILURE);
+  }
+}
+
+// Simple struct to hold a data/length pair.
+struct LiteralData {
+  const char *Data;
+  unsigned Length;
+};
+
+// Get the literal data from a literal token.
+// If stripQuotes flag is true the quotes (and string literal type) will
+// be removed from the data and only the raw string literal value will be
+// returned.
+static LiteralData GetLiteralData(const Token &Tok, bool stripQuotes) {
+  if (!tok::isStringLiteral(Tok.getKind()))
+    return LiteralData{ Tok.getLiteralData(), Tok.getLength() };
+
+  unsigned start_offset = 0;
+  unsigned end_offset = 0;
+  switch (Tok.getKind()) {
+  case tok::string_literal:       start_offset = 1; end_offset = 1;  break; // "foo"
+  case tok::wide_string_literal:  start_offset = 2; end_offset = 1;  break; // L"foo"
+  case tok::utf8_string_literal:  start_offset = 3; end_offset = 1;  break; // u8"foo"
+  case tok::utf16_string_literal: start_offset = 2; end_offset = 1;  break; // u"foo"
+  case tok::utf32_string_literal: start_offset = 2; end_offset = 1;  break; // U"foo"
+  default: break;
+  }
+
+  unsigned length = Tok.getLength() - (start_offset + end_offset);
+  if (length > Tok.getLength()) { // Check for unsigned underflow.
+    DXASSERT(false, "string literal quote count is wrong?");
+    start_offset = 0;
+    length = Tok.getLength();
+  }
+
+  return LiteralData {Tok.getLiteralData() + start_offset, length};
+}
+
+// Print leading spaces if needed by the token.
+// Take care when stripping string literal quoates that we do not add extra
+// spaces to the output.
+static bool ShouldPrintLeadingSpace(const Token &Tok, const Token &PrevTok, bool stripQuotes) {
+  if (!Tok.hasLeadingSpace())
+    return false;
+
+  // Token has leading spaces, but the previous token was a sting literal
+  // and we are stripping quotes to paste the strings together so do not
+  // add a space between the string literal values.
+  if (tok::isStringLiteral(PrevTok.getKind()) && stripQuotes)
+    return false;
+
+  return true;
+}
+
+// Macro expansion implementation.
+// We re-lex the macro using the preprocessors lexer.
+bool MacroExpander::ExpandMacro(MacroInfo *pMacro, std::string *out) {
+  if (!pMacro || !out)
+    return false;
+  MacroInfo &macro = *pMacro;
+
+  // Initialize the token from the macro definition location.
+  Token Tok;
+  bool failed = PP.getRawToken(macro.getDefinitionLoc(), Tok);
+  if (failed)
+    return false;
+
+  // Start the lexing process. Use an outer file to make the preprocessor happy.
+  PP.EnterSourceFile(m_expansionFileId, nullptr, PP.getSourceManager().getLocForStartOfFile(m_expansionFileId));
+  PP.EnterMacro(Tok, macro.getDefinitionEndLoc(), &macro, nullptr);
+  PP.Lex(Tok);
+  llvm::raw_string_ostream OS(*out);
+
+  // Keep track of previous token to print spaces correctly.
+  Token PrevTok;
+  PrevTok.startToken();
+
+  // Lex all the tokens from the macro and add them to the output.
+  while (!Tok.is(tok::eof)) {
+    if (ShouldPrintLeadingSpace(Tok, PrevTok, m_stripQuotes)) {
+      OS << ' ';
+    }
+    if (IdentifierInfo *II = Tok.getIdentifierInfo()) {
+      OS << II->getName();
+    }
+    else if (Tok.isLiteral() && !Tok.needsCleaning() &&
+      Tok.getLiteralData()) {
+      LiteralData literalData = GetLiteralData(Tok, m_stripQuotes);
+      OS.write(literalData.Data, literalData.Length);
+    }
+    else {
+      std::string S = PP.getSpelling(Tok);
+      OS.write(&S[0], S.size());
+    }
+    PrevTok = Tok;
+    PP.Lex(Tok);
+  }
+
+  return true;
+}
+
+// Search for the macro info by the given name.
+MacroInfo *MacroExpander::FindMacroInfo(clang::Preprocessor &PP, StringRef macroName) {
+  // Lookup macro identifier.
+  IdentifierInfo *ii = PP.getIdentifierInfo(macroName);
+  if (!ii)
+    return nullptr;
+
+  // Lookup macro info.
+  return PP.getMacroInfo(ii);
+}

tools/clang/test/CodeGenHLSL/rootSigDefine1.hlsl

@@ -0,0 +1,11 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define.
+
+#define RS DescriptorTable(SRV(t3))
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine10.hlsl

@@ -0,0 +1,13 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s | FileCheck %s
+// Test root signature define error: missing descriptor range.
+
+// CHECK: error: validation errors
+// CHECK: Root Signature in DXIL container is not compatible with shader
+#define RS DescriptorTable(SRV(t0))
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine11.hlsl

@@ -0,0 +1,14 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define overrides root signature attribute.
+// Put an invalid root signature in the attribute and a valid one in the
+// define. If compilation succeeds then the valid root signature was used.
+
+Texture1D<float> tex : register(t3);
+
+#define RS DescriptorTable(SRV(t3))
+[RootSignature("SRV(t0)")]
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine2.hlsl

@@ -0,0 +1,11 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define string
+
+#define RS "DescriptorTable(SRV(t3))"
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine3.hlsl

@@ -0,0 +1,10 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define empty
+
+#define RS 
+
+float main(float i : I) : SV_Target
+{
+  return i;
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine4.hlsl

@@ -0,0 +1,10 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define empty string
+
+#define RS ""
+
+float main(float i : I) : SV_Target
+{
+  return i;
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine5.hlsl

@@ -0,0 +1,12 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define concatenated string
+
+#define RS "DescriptorTable" \
+           "(SRV(t3))"
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine6.hlsl

@@ -0,0 +1,13 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define expanded macro.
+
+#define YYY "DescriptorTable(SRV(t3))"
+#define RS YYY
+           
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine7.hlsl

@@ -0,0 +1,13 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define expanded macro concatenated string.
+
+#define YYY "DescriptorTable" "(SRV(t3))"
+#define RS YYY
+           
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine8.hlsl

@@ -0,0 +1,13 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s
+// Test root signature define expanded macro call.
+
+#define YYY(x) DescriptorTable(SRV(x))
+#define RS YYY(t3)
+           
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigDefine9.hlsl

@@ -0,0 +1,12 @@
+// RUN: %dxc -E main -T ps_6_0 -rootsig-define RS %s | FileCheck %s
+// Test root signature define error: syntax error in root signature.
+
+// CHECK: root signature error - Unexpected token 'XescriptorTable' when parsing root signature
+#define RS XescriptorTable(SRV(x))
+
+Texture1D<float> tex : register(t3);
+float main(float i : I) : SV_Target
+{
+  return tex[i];
+}
+

tools/clang/test/CodeGenHLSL/rootSigProfile5.hlsl

@@ -0,0 +1,5 @@
+// RUN: %dxc -E RS -T rootsig_1_0 %s
+// Test root signature compilation from expanded macro.
+
+#define YYY "DescriptorTable" "(SRV(t3))"
+#define RS YYY

tools/clang/test/HLSL/rewriter/correct_rewrites/semantic-defines_gold.hlsl

@@ -0,0 +1,10 @@
+// Rewrite unchanged result:
+int func(int a) {
+  return a;
+}
+
+
+
+// Macros:
+#define SD_FOO 1
+#define SD_ZOO 2

tools/clang/test/HLSL/rewriter/semantic-defines.hlsl

@@ -0,0 +1,8 @@
+#define BAR 1
+#define SD_FOO BAR
+#define SD_ZOO 2
+
+int func(int a)
+{
+  return a;
+}

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -19,6 +19,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/HLSLMacroExpander.h"
 #include "clang/Parse/ParseAST.h"
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/Sema/SemaConsumer.h"
@@ -1779,6 +1780,7 @@ class HLSLExtensionsCodegenHelperImpl : public HLSLExtensionsCodegenHelper {
 private:
   CompilerInstance &m_CI;
   DxcLangExtensionsHelper &m_langExtensionsHelper;
+  std::string m_rootSigDefine;
 
   // The metadata format is a root node that has pointers to metadata
   // nodes for each define. The metatdata node for a define is a pair
@@ -1818,8 +1820,9 @@ private:
   }
 
 public:
-  HLSLExtensionsCodegenHelperImpl(CompilerInstance &CI, DxcLangExtensionsHelper &langExtensionsHelper)
+  HLSLExtensionsCodegenHelperImpl(CompilerInstance &CI, DxcLangExtensionsHelper &langExtensionsHelper, StringRef rootSigDefine)
   : m_CI(CI), m_langExtensionsHelper(langExtensionsHelper)
+  , m_rootSigDefine(rootSigDefine)
   {}
 
   // Write semantic defines as metadata in the module.
@@ -1842,6 +1845,24 @@ public:
   virtual std::string GetIntrinsicName(UINT opcode) override {
     return m_langExtensionsHelper.GetIntrinsicName(opcode);
   }
+  
+  virtual HLSLExtensionsCodegenHelper::CustomRootSignature::Status GetCustomRootSignature(CustomRootSignature *out) {
+    // Find macro definition in preprocessor.
+    Preprocessor &pp = m_CI.getPreprocessor();
+    MacroInfo *macro = MacroExpander::FindMacroInfo(pp, m_rootSigDefine);
+    if (!macro)
+      return CustomRootSignature::NOT_FOUND;
+
+    // Combine tokens into single string
+    MacroExpander expander(pp, MacroExpander::STRIP_QUOTES);
+    if (!expander.ExpandMacro(macro, &out->RootSignature))
+      return CustomRootSignature::NOT_FOUND;
+
+    // Record source location of root signature macro.
+    out->EncodedSourceLocation = macro->getDefinitionLoc().getRawEncoding();
+
+    return CustomRootSignature::FOUND;
+  }
 };
 
 // Class to manage lifetime of llvm module and provide some utility
@@ -2570,7 +2591,7 @@ public:
     compiler.getCodeGenOpts().setInlining(
         clang::CodeGenOptions::OnlyAlwaysInlining);
 
-    compiler.getCodeGenOpts().HLSLExtensionsCodegen = std::make_shared<HLSLExtensionsCodegenHelperImpl>(compiler, m_langExtensionsHelper);
+    compiler.getCodeGenOpts().HLSLExtensionsCodegen = std::make_shared<HLSLExtensionsCodegenHelperImpl>(compiler, m_langExtensionsHelper, Opts.RootSignatureDefine);
   }
 };
 

tools/clang/tools/libclang/dxcrewriteunused.cpp

@@ -19,6 +19,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/HLSLMacroExpander.h"
 #include "clang/Parse/ParseAST.h"
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/Sema/SemaConsumer.h"
@@ -180,14 +181,14 @@ ParsedSemanticDefineList hlsl::CollectSemanticDefinesParsedByCompiler(CompilerIn
   // This is very inefficient in general, but in practice we either have
   // no semantic defines, or we have a star define for a some reserved prefix. These will be
   // sorted so rewrites are stable.
-  std::vector<std::pair<const IdentifierInfo*, const MacroInfo*> > macros;
+  std::vector<std::pair<const IdentifierInfo*, MacroInfo*> > macros;
   Preprocessor& pp = compiler.getPreprocessor();
   Preprocessor::macro_iterator end = pp.macro_end();
   for (Preprocessor::macro_iterator i = pp.macro_begin(); i != end; ++i) {
     if (!i->second.getLatest()->isDefined()) {
       continue;
     }
-    const MacroInfo* mi = i->second.getLatest()->getMacroInfo();
+    MacroInfo* mi = i->second.getLatest()->getMacroInfo();
     if (mi->isFunctionLike()) {
       continue;
     }
@@ -211,37 +212,17 @@ ParsedSemanticDefineList hlsl::CollectSemanticDefinesParsedByCompiler(CompilerIn
         continue;
       }
 
-      macros.push_back(std::pair<const IdentifierInfo*, const MacroInfo*>(ii, mi));
+      macros.push_back(std::pair<const IdentifierInfo*, MacroInfo*>(ii, mi));
     }
   }
 
   if (!macros.empty()) {
     std::sort(macros.begin(), macros.end(), MacroPairCompareIsLessThan);
-    SmallVector<std::string, 8> tokens;
-    for (auto&& m : macros) {
-      std::string name = m.first->getName();
-      // Collect all macro token values into a vector.
-      // Put them in a vector to avoid repeated copying of data when appending strings.
-      tokens.clear();
-      tokens.reserve(m.second->getNumTokens());
-      for (MacroInfo::tokens_iterator ti = m.second->tokens_begin(), tiEnd = m.second->tokens_end(); ti != tiEnd; ++ti) {
-        if (ti->hasLeadingSpace())
-          tokens.push_back(" ");
-        tokens.push_back(pp.getSpelling(*ti));
-      }
-
-      // Compute total size of defined string value.
-      size_t size = 0;
-      for (const std::string &s : tokens)
-        size += s.size();
-
-      // Concatenate all values into a single string.
-      std::string value;
-      value.reserve(size);
-      for (const std::string &s : tokens)
-        value.append(s);
-
-      parsedDefines.emplace_back(ParsedSemanticDefine{ name, value, m.second->getDefinitionLoc().getRawEncoding() });
+    MacroExpander expander(pp);
+    for (std::pair<const IdentifierInfo *, MacroInfo *> m : macros) {
+      std::string expandedValue;
+      expander.ExpandMacro(m.second, &expandedValue);
+      parsedDefines.emplace_back(ParsedSemanticDefine{ m.first->getName(), expandedValue, m.second->getDefinitionLoc().getRawEncoding() });
     }
   }
 

tools/clang/unittests/HLSL/CompilerTest.cpp

@@ -597,6 +597,18 @@ public:
   TEST_METHOD(CodeGenRootSigProfile2)
   TEST_METHOD(CodeGenRootSigProfile3)
   TEST_METHOD(CodeGenRootSigProfile4)
+  TEST_METHOD(CodeGenRootSigProfile5)
+  TEST_METHOD(CodeGenRootSigDefine1)
+  TEST_METHOD(CodeGenRootSigDefine2)
+  TEST_METHOD(CodeGenRootSigDefine3)
+  TEST_METHOD(CodeGenRootSigDefine4)
+  TEST_METHOD(CodeGenRootSigDefine5)
+  TEST_METHOD(CodeGenRootSigDefine6)
+  TEST_METHOD(CodeGenRootSigDefine7)
+  TEST_METHOD(CodeGenRootSigDefine8)
+  TEST_METHOD(CodeGenRootSigDefine9)
+  TEST_METHOD(CodeGenRootSigDefine10)
+  TEST_METHOD(CodeGenRootSigDefine11)
   TEST_METHOD(CodeGenCBufferStructArray)
   TEST_METHOD(PreprocessWhenValidThenOK)
   TEST_METHOD(WhenSigMismatchPCFunctionThenFail)
@@ -3054,6 +3066,54 @@ TEST_F(CompilerTest, CodeGenRootSigProfile4) {
   CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigProfile4.hlsl");
 }
 
+TEST_F(CompilerTest, CodeGenRootSigProfile5) {
+  CodeGenTest(L"..\\CodeGenHLSL\\rootSigProfile5.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine1) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine1.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine2) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine2.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine3) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine3.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine4) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine4.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine5) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine5.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine6) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine6.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine7) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine7.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine8) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine8.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine9) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine9.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine10) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine10.hlsl");
+}
+
+TEST_F(CompilerTest, CodeGenRootSigDefine11) {
+  CodeGenTestCheck(L"..\\CodeGenHLSL\\rootSigDefine11.hlsl");
+}
+
 TEST_F(CompilerTest, CodeGenCBufferStructArray) {
   CodeGenTestCheck(L"..\\CodeGenHLSL\\cbuffer-structarray.hlsl");
 }

tools/clang/unittests/HLSL/ExtensionTest.cpp

@@ -395,6 +395,7 @@ public:
   TEST_METHOD(DefineValidationError);
   TEST_METHOD(DefineValidationWarning);
   TEST_METHOD(DefineNoValidatorOk);
+  TEST_METHOD(DefineFromMacro);
   TEST_METHOD(IntrinsicWhenAvailableThenUsed);
   TEST_METHOD(CustomIntrinsicName);
   TEST_METHOD(NoLowering);
@@ -523,6 +524,26 @@ TEST_F(ExtensionTest, DefineNoValidatorOk) {
     disassembly.find("!{!\"FOO\", !\"1\"}"));
 }
 
+TEST_F(ExtensionTest, DefineFromMacro) {
+  Compiler c(m_dllSupport);
+  c.RegisterSemanticDefine(L"FOO*");
+  c.Compile(
+    "#define BAR 1\n"
+    "#define FOO BAR\n"
+    "float4 main() : SV_Target {\n"
+    "  return 0;\n"
+    "}\n",
+    { L"/Vd" }, {}
+  );
+
+  std::string disassembly = c.Disassemble();
+  // Check the define is emitted.
+  // #define FOO 1
+  VERIFY_IS_TRUE(
+    disassembly.npos !=
+    disassembly.find("!{!\"FOO\", !\"1\"}"));
+}
+
 
 TEST_F(ExtensionTest, IntrinsicWhenAvailableThenUsed) {
   Compiler c(m_dllSupport);

tools/clang/unittests/HLSL/RewriterTest.cpp

@@ -31,6 +31,7 @@
 #include "HLSLTestData.h"
 #include "WexTestClass.h"
 #include "HlslTestUtils.h"
+#include "DxcTestUtils.h"
 
 #include "dxc/Support/Global.h"
 #include "dxc/dxctools.h"
@@ -70,6 +71,7 @@ public:
   TEST_METHOD(RunUTF16ThreeByte);
   TEST_METHOD(RunNonUnicode);
   TEST_METHOD(RunEffect);
+  TEST_METHOD(RunSemanticDefines);
 
   dxc::DxcDllSupport m_dllSupport;
 
@@ -101,7 +103,10 @@ public:
   VerifyResult CheckVerifies(LPCWSTR path, LPCWSTR goldPath) {   
     CComPtr<IDxcRewriter> pRewriter;
     VERIFY_SUCCEEDED(CreateRewriter(&pRewriter));
-    
+    return CheckVerifies(pRewriter, path, goldPath);
+  }
+
+  VerifyResult CheckVerifies(IDxcRewriter *pRewriter, LPCWSTR path, LPCWSTR goldPath) {
     CComPtr<IDxcOperationResult> pRewriteResult;
     RewriteCompareGold(path, goldPath, &pRewriteResult, pRewriter);
 
@@ -125,6 +130,16 @@ public:
     return m_dllSupport.CreateInstance(CLSID_DxcRewriter, pRewriter);
   }
 
+  HRESULT CreateRewriterWithSemanticDefines(IDxcRewriter** pRewriter, std::vector<LPCWSTR> defines) {
+    VERIFY_SUCCEEDED(CreateRewriter(pRewriter));
+    CComPtr<IDxcLangExtensions> pLangExtensions;
+    VERIFY_SUCCEEDED((*pRewriter)->QueryInterface(&pLangExtensions));
+    for (LPCWSTR define : defines)
+      VERIFY_SUCCEEDED(pLangExtensions->RegisterSemanticDefine(define));
+
+    return S_OK;
+  }
+
   VerifyResult CheckVerifiesHLSL(LPCWSTR name, LPCWSTR goldName) {
     return CheckVerifies(GetPathToHlslDataFile(name).c_str(),
                          GetPathToHlslDataFile(goldName).c_str());
@@ -396,3 +411,9 @@ TEST_F(RewriterTest, RunEffect) {
   CheckVerifiesHLSL(L"rewriter\\effects-syntax.hlsl", L"rewriter\\correct_rewrites\\effects-syntax_gold.hlsl");
 }
 
+TEST_F(RewriterTest, RunSemanticDefines) {
+  CComPtr<IDxcRewriter> pRewriter;
+  VERIFY_SUCCEEDED(CreateRewriterWithSemanticDefines(&pRewriter, {L"SD_*"}));
+  CheckVerifies(pRewriter, hlsl_test::GetPathToHlslDataFile(L"rewriter\\semantic-defines.hlsl").c_str(),
+                           hlsl_test::GetPathToHlslDataFile(L"rewriter\\correct_rewrites\\semantic-defines_gold.hlsl").c_str());
+}