Add seperate reflection part (STAT), add -Qstrip_reflect_from_dxil

- Put separate reflection in STAT part for now.
- Separate reflection is the module with deleted function bodies.
- Use new -Qstrip_reflect_from_dxil to drive stipping of reflection
  metadata from DXIL part, since now -Qstrip_reflect means strip
  the STAT reflection part, or don't include it in the first place.
- Update disassembler to use STAT part if available for reflecting
  resource bindings, buffer descriptions, and ViewID state.
- Put some Qstrip_* flags under DriverOption as well as CoreOption.
This commit is contained in:
Tex Riddell 2019-08-12 18:15:19 -07:00
Родитель 5894e7ab66
Коммит 5aec592757
10 изменённых файлов: 174 добавлений и 64 удалений

Просмотреть файл

@ -404,11 +404,12 @@ inline bool GetDxilShaderDebugName(const DxilPartHeader *pDebugNamePart,
}
enum class SerializeDxilFlags : uint32_t {
None = 0, // No flags defined.
IncludeDebugInfoPart = 1, // Include the debug info part in the container.
IncludeDebugNamePart = 2, // Include the debug name part in the container.
DebugNameDependOnSource = 4, // Make the debug name depend on source (and not just final module).
StripReflectionFromDxilPart = 8, // Strip Reflection info from DXIL part.
None = 0, // No flags defined.
IncludeDebugInfoPart = 1 << 0, // Include the debug info part in the container.
IncludeDebugNamePart = 1 << 1, // Include the debug name part in the container.
DebugNameDependOnSource = 1 << 2, // Make the debug name depend on source (and not just final module).
StripReflectionFromDxilPart = 1 << 3, // Strip Reflection info from DXIL part.
IncludeReflectionPart = 1 << 4, // Include reflection in STAT part.
};
inline SerializeDxilFlags& operator |=(SerializeDxilFlags& l, const SerializeDxilFlags& r) {
l = static_cast<SerializeDxilFlags>(static_cast<int>(l) | static_cast<int>(r));

Просмотреть файл

@ -51,7 +51,8 @@ void SerializeDxilContainerForModule(hlsl::DxilModule *pModule,
AbstractMemoryStream *pStream,
llvm::StringRef DebugName,
SerializeDxilFlags Flags,
DxilShaderHash *pShaderHashOut = nullptr);
DxilShaderHash *pShaderHashOut = nullptr,
AbstractMemoryStream *pReflectionStreamOut = nullptr);
void SerializeDxilContainerForRootSignature(hlsl::RootSignatureHandle *pRootSigHandle,
AbstractMemoryStream *pStream);

Просмотреть файл

@ -154,6 +154,7 @@ public:
bool StripRootSignature = false; // OPT_Qstrip_rootsignature
bool StripPrivate = false; // OPT_Qstrip_priv
bool StripReflection = false; // OPT_Qstrip_reflect
bool StripReflectionFromDxil = false; // OPT_Qstrip_reflect_from_dxil
bool ExtractRootSignature = false; // OPT_extractrootsignature
bool DisassembleColorCoded = false; // OPT_Cc
bool DisassembleInstNumbers = false; //OPT_Ni

Просмотреть файл

@ -351,9 +351,11 @@ def P : Separate<["-", "/"], "P">, Flags<[CoreOption, DriverOption]>, Group<hlsl
def dumpbin : Flag<["-", "/"], "dumpbin">, Flags<[DriverOption]>, Group<hlslutil_Group>,
HelpText<"Load a binary file rather than compiling">;
def Qstrip_reflect : Flag<["-", "/"], "Qstrip_reflect">, Flags<[CoreOption]>, Group<hlslutil_Group>,
def Qstrip_reflect : Flag<["-", "/"], "Qstrip_reflect">, Flags<[CoreOption, DriverOption]>, Group<hlslutil_Group>,
HelpText<"Strip reflection data from shader bytecode (must be used with /Fo <file>)">;
def Qstrip_debug : Flag<["-", "/"], "Qstrip_debug">, Flags<[CoreOption]>, Group<hlslutil_Group>,
def Qstrip_reflect_from_dxil : Flag<["-", "/"], "Qstrip_reflect_from_dxil">, Flags<[CoreOption]>, Group<hlslutil_Group>,
HelpText<"Strip reflection data from shader bytecode (must be used with /Fo <file>)">;
def Qstrip_debug : Flag<["-", "/"], "Qstrip_debug">, Flags<[CoreOption, DriverOption]>, Group<hlslutil_Group>,
HelpText<"Strip debug information from 4_0+ shader bytecode (must be used with /Fo <file>)">;
def Qembed_debug : Flag<["-", "/"], "Qembed_debug">, Flags<[CoreOption]>, Group<hlslutil_Group>,
HelpText<"Embed PDB in shader container (must be used with /Zi)">;

Просмотреть файл

@ -553,6 +553,7 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
opts.StripRootSignature = Args.hasFlag(OPT_Qstrip_rootsignature, OPT_INVALID, false);
opts.StripPrivate = Args.hasFlag(OPT_Qstrip_priv, OPT_INVALID, false);
opts.StripReflection = Args.hasFlag(OPT_Qstrip_reflect, OPT_INVALID, false);
opts.StripReflectionFromDxil = Args.hasFlag(OPT_Qstrip_reflect_from_dxil, OPT_INVALID, false);
opts.ExtractRootSignature = Args.hasFlag(OPT_extractrootsignature, OPT_INVALID, false);
opts.DisassembleColorCoded = Args.hasFlag(OPT_Cc, OPT_INVALID, false);
opts.DisassembleInstNumbers = Args.hasFlag(OPT_Ni, OPT_INVALID, false);

Просмотреть файл

@ -17,6 +17,7 @@
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/Support/MD5.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "dxc/DxilContainer/DxilContainer.h"
#include "dxc/DXIL/DxilModule.h"
#include "dxc/DXIL/DxilShaderModel.h"
@ -1543,7 +1544,8 @@ void hlsl::SerializeDxilContainerForModule(DxilModule *pModule,
AbstractMemoryStream *pFinalStream,
llvm::StringRef DebugName,
SerializeDxilFlags Flags,
DxilShaderHash *pShaderHashOut) {
DxilShaderHash *pShaderHashOut,
AbstractMemoryStream *pReflectionStreamOut) {
// TODO: add a flag to update the module and remove information that is not part
// of DXIL proper and is used only to assemble the container.
@ -1557,6 +1559,10 @@ void hlsl::SerializeDxilContainerForModule(DxilModule *pModule,
Flags &= ~SerializeDxilFlags::IncludeDebugNamePart;
bool bSupportsShaderHash = DXIL::CompareVersions(ValMajor, ValMinor, 1, 5) >= 0;
bool bCompat_1_4 = DXIL::CompareVersions(ValMajor, ValMinor, 1, 5) < 0;
if (DXIL::CompareVersions(ValMajor, ValMinor, 1, 5) < 0)
Flags &= ~SerializeDxilFlags::IncludeReflectionPart;
bool bEmitReflection = Flags & SerializeDxilFlags::IncludeReflectionPart ||
pReflectionStreamOut;
DxilContainerWriter_impl writer;
@ -1670,9 +1676,43 @@ void hlsl::SerializeDxilContainerForModule(DxilModule *pModule,
Flags &= ~SerializeDxilFlags::DebugNameDependOnSource;
}
// Clone module for reflection, strip function defs
std::unique_ptr<Module> reflectionModule;
if (bEmitReflection) {
reflectionModule.reset(llvm::CloneModule(pModule->GetModule()));
for (Function &F : reflectionModule->functions()) {
if (!F.isDeclaration()) {
F.deleteBody();
}
}
// Just make sure this doesn't crash/assert on debug build:
DXASSERT_NOMSG(&reflectionModule->GetOrCreateDxilModule());
}
CComPtr<AbstractMemoryStream> pReflectionBitcodeStream;
if (bEmitReflection)
{
IFT(CreateMemoryStream(DxcGetThreadMallocNoRef(), &pReflectionBitcodeStream));
raw_stream_ostream outStream(pReflectionBitcodeStream.p);
WriteBitcodeToFile(reflectionModule.get(), outStream, false);
outStream.flush();
}
if (pReflectionStreamOut) {
WriteProgramPart(pModule->GetShaderModel(), pReflectionBitcodeStream, pReflectionStreamOut);
}
if (Flags & SerializeDxilFlags::IncludeReflectionPart) {
uint32_t reflectInUInt32, reflectPaddingBytes;
GetPaddedProgramPartSize(pReflectionBitcodeStream, reflectInUInt32, reflectPaddingBytes);
writer.AddPart(DFCC_ShaderStatistics, reflectInUInt32 * sizeof(uint32_t) + sizeof(DxilProgramHeader),
[pModule, pReflectionBitcodeStream](AbstractMemoryStream *pStream) {
WriteProgramPart(pModule->GetShaderModel(), pReflectionBitcodeStream, pStream);
});
}
if (Flags & SerializeDxilFlags::StripReflectionFromDxilPart) {
pModule->StripReflection();
bModuleStripped = true;
bModuleStripped |= pModule->StripReflection();
}
// If debug info or reflection was stripped, re-serialize the module.

Просмотреть файл

@ -84,7 +84,7 @@ enum class PublicAPI { D3D12 = 0, D3D11_47 = 1, D3D11_43 = 2 };
class DxilModuleReflection {
public:
CComPtr<IDxcBlob> m_pContainer;
hlsl::RDAT::DxilRuntimeData m_RDAT;
LLVMContext Context;
std::unique_ptr<Module> m_pModule; // Must come after LLVMContext, otherwise unique_ptr will over-delete.
DxilModule *m_pDxilModule = nullptr;
@ -95,7 +95,8 @@ public:
void CreateReflectionObjects();
void CreateReflectionObjectForResource(DxilResourceBase *R);
HRESULT LoadModule(IDxcBlob *pBlob, const DxilPartHeader *pPart);
HRESULT LoadRDAT(const DxilPartHeader *pPart);
HRESULT LoadModule(const DxilPartHeader *pPart);
// Common code
ID3D12ShaderReflectionConstantBuffer* _GetConstantBufferByIndex(UINT Index);
@ -153,7 +154,7 @@ public:
return hr;
}
HRESULT Load(IDxcBlob *pBlob, const DxilPartHeader *pPart);
HRESULT Load(const DxilPartHeader *pModulePart, const DxilPartHeader *pRDATPart);
// ID3D12ShaderReflection
STDMETHODIMP GetDesc(THIS_ _Out_ D3D12_SHADER_DESC *pDesc);
@ -218,7 +219,7 @@ public:
return DoBasicQueryInterface<ID3D12LibraryReflection>(this, iid, ppvObject);
}
HRESULT Load(IDxcBlob *pBlob, const DxilPartHeader *pPart);
HRESULT Load(const DxilPartHeader *pModulePart, const DxilPartHeader *pDXILPart);
// ID3D12LibraryReflection
STDMETHOD(GetDesc)(THIS_ _Out_ D3D12_LIBRARY_DESC * pDesc);
@ -226,6 +227,31 @@ public:
STDMETHOD_(ID3D12FunctionReflection *, GetFunctionByIndex)(THIS_ _In_ INT FunctionIndex);
};
namespace hlsl {
HRESULT CreateDxilShaderReflection(const DxilPartHeader *pModulePart, const DxilPartHeader *pRDATPart, REFIID iid, void **ppvObject) {
if (!ppvObject)
return E_INVALIDARG;
CComPtr<DxilShaderReflection> pReflection = DxilShaderReflection::Alloc(DxcGetThreadMallocNoRef());
IFROOM(pReflection.p);
PublicAPI api = DxilShaderReflection::IIDToAPI(iid);
pReflection->SetPublicAPI(api);
// pRDATPart to be used for transition.
IFR(pReflection->Load(pModulePart, pRDATPart));
IFR(pReflection.p->QueryInterface(iid, ppvObject));
return S_OK;
}
HRESULT CreateDxilLibraryReflection(const DxilPartHeader *pModulePart, const DxilPartHeader *pRDATPart, REFIID iid, void **ppvObject) {
if (!ppvObject)
return E_INVALIDARG;
CComPtr<DxilLibraryReflection> pReflection = DxilLibraryReflection::Alloc(DxcGetThreadMallocNoRef());
IFROOM(pReflection.p);
// pRDATPart used for resource usage per-function.
IFR(pReflection->Load(pModulePart, pRDATPart));
IFR(pReflection.p->QueryInterface(iid, ppvObject));
return S_OK;
}
}
_Use_decl_annotations_
HRESULT DxilContainerReflection::Load(IDxcBlob *pContainer) {
@ -313,32 +339,45 @@ HRESULT DxilContainerReflection::GetPartReflection(UINT32 idx, REFIID iid, void
if (!IsLoaded()) return E_NOT_VALID_STATE;
if (idx >= m_pHeader->PartCount) return E_BOUNDS;
const DxilPartHeader *pPart = GetDxilContainerPart(m_pHeader, idx);
if (pPart->PartFourCC != DFCC_DXIL && pPart->PartFourCC != DFCC_ShaderDebugInfoDXIL) {
if (pPart->PartFourCC != DFCC_DXIL && pPart->PartFourCC != DFCC_ShaderDebugInfoDXIL &&
pPart->PartFourCC != DFCC_ShaderStatistics) {
return E_NOTIMPL;
}
// Use DFCC_ShaderStatistics for reflection instead of DXIL part, until switch
// to using RDAT for reflection instead of module.
const DxilPartHeader *pRDATPart = nullptr;
for (idx = 0; idx < m_pHeader->PartCount; ++idx) {
const DxilPartHeader *pPartTest = GetDxilContainerPart(m_pHeader, idx);
if (pPartTest->PartFourCC == DFCC_RuntimeData) {
pRDATPart = pPartTest;
}
if (pPart->PartFourCC != DFCC_ShaderStatistics) {
if (pPartTest->PartFourCC == DFCC_ShaderStatistics) {
const DxilProgramHeader *pProgramHeaderTest =
reinterpret_cast<const DxilProgramHeader*>(GetDxilPartData(pPartTest));
if (IsValidDxilProgramHeader(pProgramHeaderTest, pPartTest->PartSize)) {
pPart = pPartTest;
continue;
}
}
}
}
DxcThreadMalloc TM(m_pMalloc);
HRESULT hr = S_OK;
const DxilProgramHeader *pProgramHeader =
reinterpret_cast<const DxilProgramHeader*>(GetDxilPartData(pPart));
if (!IsValidDxilProgramHeader(pProgramHeader, pPart->PartSize)) {
return E_INVALIDARG;
}
DxcThreadMalloc TM(m_pMalloc);
HRESULT hr = S_OK;
DXIL::ShaderKind SK = GetVersionShaderType(pProgramHeader->ProgramVersion);
if (SK == DXIL::ShaderKind::Library) {
CComPtr<DxilLibraryReflection> pReflection = DxilLibraryReflection::Alloc(m_pMalloc);
IFCOOM(pReflection.p);
IFC(pReflection->Load(m_container, pPart));
IFC(pReflection.p->QueryInterface(iid, ppvObject));
IFC(hlsl::CreateDxilLibraryReflection(pPart, pRDATPart, iid, ppvObject));
} else {
CComPtr<DxilShaderReflection> pReflection = DxilShaderReflection::Alloc(m_pMalloc);
IFCOOM(pReflection.p);
PublicAPI api = DxilShaderReflection::IIDToAPI(iid);
pReflection->SetPublicAPI(api);
IFC(pReflection->Load(m_container, pPart));
IFC(pReflection.p->QueryInterface(iid, ppvObject));
IFC(hlsl::CreateDxilShaderReflection(pPart, pRDATPart, iid, ppvObject));
}
Cleanup:
@ -1842,12 +1881,16 @@ LPCSTR DxilShaderReflection::CreateUpperCase(LPCSTR pValue) {
return m_UpperCaseNames.back().get();
}
HRESULT DxilModuleReflection::LoadModule(IDxcBlob *pBlob,
const DxilPartHeader *pPart) {
DXASSERT_NOMSG(pBlob != nullptr);
DXASSERT_NOMSG(pPart != nullptr);
m_pContainer = pBlob;
const char *pData = GetDxilPartData(pPart);
HRESULT DxilModuleReflection::LoadRDAT(const DxilPartHeader *pPart) {
if (pPart) {
IFRBOOL(m_RDAT.InitFromRDAT(GetDxilPartData(pPart), pPart->PartSize), DXC_E_CONTAINER_INVALID);
}
return S_OK;
}
HRESULT DxilModuleReflection::LoadModule(const DxilPartHeader *pShaderPart) {
DXASSERT_NOMSG(pShaderPart != nullptr);
const char *pData = GetDxilPartData(pShaderPart);
try {
const char *pBitcode;
uint32_t bitcodeLength;
@ -1877,9 +1920,10 @@ HRESULT DxilModuleReflection::LoadModule(IDxcBlob *pBlob,
CATCH_CPP_RETURN_HRESULT();
};
HRESULT DxilShaderReflection::Load(IDxcBlob *pBlob,
const DxilPartHeader *pPart) {
IFR(LoadModule(pBlob, pPart));
HRESULT DxilShaderReflection::Load(const DxilPartHeader *pModulePart,
const DxilPartHeader *pRDATPart) {
IFR(LoadRDAT(pRDATPart));
IFR(LoadModule(pModulePart));
try {
// Set cbuf usage.
@ -2360,32 +2404,17 @@ HRESULT CFunctionReflection::GetResourceBindingDescByName(LPCSTR Name,
// DxilLibraryReflection
void DxilLibraryReflection::AddResourceDependencies() {
const DxilContainerHeader *pHeader =
IsDxilContainerLike(m_pContainer->GetBufferPointer(),
m_pContainer->GetBufferSize());
IFTBOOL(pHeader, DXC_E_MALFORMED_CONTAINER);
const DxilPartHeader *pPart = nullptr;
for (uint32_t idx = 0; idx < pHeader->PartCount; ++idx) {
const DxilPartHeader *pPartTest = GetDxilContainerPart(pHeader, idx);
if (pPartTest->PartFourCC == DFCC_RuntimeData) {
pPart = pPartTest;
break;
}
}
IFTBOOL(pPart, DXC_E_MISSING_PART);
RDAT::DxilRuntimeData RDAT(GetDxilPartData(pPart), pPart->PartSize);
RDAT::FunctionTableReader *functionTable = RDAT.GetFunctionTableReader();
RDAT::FunctionTableReader *functionTable = m_RDAT.GetFunctionTableReader();
m_FunctionVector.clear();
m_FunctionVector.reserve(functionTable->GetNumFunctions());
std::map<StringRef, CFunctionReflection*> orderedMap;
RDAT::ResourceTableReader *resourceTable = RDAT.GetResourceTableReader();
RDAT::ResourceTableReader *resourceTable = m_RDAT.GetResourceTableReader();
unsigned SamplersStart = resourceTable->GetNumCBuffers();
unsigned SRVsStart = SamplersStart + resourceTable->GetNumSamplers();
unsigned UAVsStart = SRVsStart + resourceTable->GetNumSRVs();
IFTBOOL(UAVsStart + resourceTable->GetNumUAVs() == m_Resources.size(), DXC_E_INCORRECT_DXIL_METADATA);
IFTBOOL(resourceTable->GetNumResources() == m_Resources.size(),
DXC_E_INCORRECT_DXIL_METADATA);
for (unsigned iFunc = 0; iFunc < functionTable->GetNumFunctions(); ++iFunc) {
RDAT::FunctionReader FR = functionTable->GetItem(iFunc);
@ -2452,9 +2481,10 @@ void DxilLibraryReflection::SetCBufferUsage() {
// ID3D12LibraryReflection
HRESULT DxilLibraryReflection::Load(IDxcBlob *pBlob,
const DxilPartHeader *pPart) {
IFR(LoadModule(pBlob, pPart));
HRESULT DxilLibraryReflection::Load(const DxilPartHeader *pModulePart,
const DxilPartHeader *pRDATPart) {
IFR(LoadRDAT(pRDATPart));
IFR(LoadModule(pModulePart));
try {
AddResourceDependencies();

Просмотреть файл

@ -1549,6 +1549,8 @@ namespace dxcutil {
HRESULT Disassemble(IDxcBlob *pProgram, raw_string_ostream &Stream) {
const char *pIL = (const char *)pProgram->GetBufferPointer();
uint32_t pILLength = pProgram->GetBufferSize();
const char *pReflectionIL = nullptr;
uint32_t pReflectionILLength = 0;
const DxilPartHeader *pRDATPart = nullptr;
if (const DxilContainerHeader *pContainer =
IsDxilContainerLike(pIL, pILLength)) {
@ -1649,6 +1651,18 @@ HRESULT Disassemble(IDxcBlob *pProgram, raw_string_ostream &Stream) {
}
GetDxilProgramBitcode(pProgramHeader, &pIL, &pILLength);
it = std::find_if(begin(pContainer), end(pContainer),
DxilPartIsType(DFCC_ShaderStatistics));
if (it != end(pContainer)) {
// If this part exists, use it for reflection data, probably stripped from DXIL part.
const DxilProgramHeader *pReflectionProgramHeader =
reinterpret_cast<const DxilProgramHeader *>(GetDxilPartData(*it));
if (IsValidDxilProgramHeader(pReflectionProgramHeader, (*it)->PartSize)) {
GetDxilProgramBitcode(pReflectionProgramHeader, &pReflectionIL, &pReflectionILLength);
}
}
} else {
const DxilProgramHeader *pProgramHeader =
reinterpret_cast<const DxilProgramHeader *>(pIL);
@ -1665,8 +1679,20 @@ HRESULT Disassemble(IDxcBlob *pProgram, raw_string_ostream &Stream) {
return DXC_E_IR_VERIFICATION_FAILED;
}
std::unique_ptr<llvm::Module> pReflectionModule;
if (pReflectionIL && pReflectionILLength) {
pReflectionModule = dxilutil::LoadModuleFromBitcode(
llvm::StringRef(pReflectionIL, pReflectionILLength), llvmContext, DiagStr);
if (pReflectionModule.get() == nullptr) {
return DXC_E_IR_VERIFICATION_FAILED;
}
}
if (pModule->getNamedMetadata("dx.version")) {
DxilModule &dxilModule = pModule->GetOrCreateDxilModule();
DxilModule &dxilReflectionModule = pReflectionModule.get()
? pReflectionModule->GetOrCreateDxilModule()
: dxilModule;
if (!dxilModule.GetShaderModel()->IsLib()) {
PrintDxilSignature("Input", dxilModule.GetInputSignature(), Stream,
@ -1685,9 +1711,9 @@ HRESULT Disassemble(IDxcBlob *pProgram, raw_string_ostream &Stream) {
/*comment*/ ";");
}
}
PrintBufferDefinitions(dxilModule, Stream, /*comment*/ ";");
PrintResourceBindings(dxilModule, Stream, /*comment*/ ";");
PrintViewIdState(dxilModule, Stream, /*comment*/ ";");
PrintBufferDefinitions(dxilReflectionModule, Stream, /*comment*/ ";");
PrintResourceBindings(dxilReflectionModule, Stream, /*comment*/ ";");
PrintViewIdState(dxilReflectionModule, Stream, /*comment*/ ";");
if (pRDATPart) {
RDAT::DxilRuntimeData runtimeData(GetDxilPartData(pRDATPart), pRDATPart->PartSize);
@ -1706,6 +1732,10 @@ HRESULT Disassemble(IDxcBlob *pProgram, raw_string_ostream &Stream) {
}
DxcAssemblyAnnotationWriter w;
pModule->print(Stream, &w);
//if (pReflectionModule) {
// Stream << "\n========== Reflection Module from STAT part ==========\n";
// pReflectionModule->print(Stream, &w);
//}
Stream.flush();
return S_OK;
}

Просмотреть файл

@ -729,9 +729,12 @@ public:
// Implies name part
SerializeFlags |= SerializeDxilFlags::IncludeDebugNamePart;
}
if (opts.StripReflection) {
if (opts.StripReflectionFromDxil) {
SerializeFlags |= SerializeDxilFlags::StripReflectionFromDxilPart;
}
if (!opts.StripReflection) {
SerializeFlags |= SerializeDxilFlags::IncludeReflectionPart;
}
// Don't do work to put in a container if an error has occurred
// Do not create a container when there is only a a high-level representation in the module.

Просмотреть файл

@ -20,6 +20,7 @@ set( LLVM_LINK_COMPONENTS
analysis
ipa
irreader
transformutils # for CloneModule
)
if(WIN32)