From 45deef90b02f8eefb5dee11816ef1b1fc5d99b8d Mon Sep 17 00:00:00 2001 From: Tex Riddell Date: Thu, 6 Feb 2020 18:23:31 -0800 Subject: [PATCH] Add missing UTF8 for source and output filenames (#2674) * Add missing UTF8 for source and output filenames - These need to be converted back to UTF-16 after argument processing for a couple APIs and CA2W constructors were missing CP_UTF8 for the input encoding from the utf-8 argument strings. * Add CP_UTF8 to more CW2A and CA2W uses --- lib/DxilDia/DxilDiaSession.cpp | 2 +- lib/DxilDia/DxilDiaSymbolManager.cpp | 22 +++++++++---------- tools/clang/lib/Sema/SemaHLSL.cpp | 2 +- tools/clang/tools/d3dcomp/d3dcomp.cpp | 16 +++++++------- tools/clang/tools/dxa/dxa.cpp | 2 +- .../clang/tools/dxcompiler/dxcfilesystem.cpp | 4 ++-- .../clang/tools/dxcompiler/dxcompilerobj.cpp | 5 +++-- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/lib/DxilDia/DxilDiaSession.cpp b/lib/DxilDia/DxilDiaSession.cpp index 268f5d7ff..6cb95313a 100644 --- a/lib/DxilDia/DxilDiaSession.cpp +++ b/lib/DxilDia/DxilDiaSession.cpp @@ -353,7 +353,7 @@ STDMETHODIMP dxil_dia::Session::findInjectedSource( /* [in] */ LPCOLESTR srcFile, /* [out] */ IDiaEnumInjectedSources **ppResult) { if (Contents() != nullptr) { - CW2A pUtf8FileName(srcFile); + CW2A pUtf8FileName(srcFile, CP_UTF8); DxcThreadMalloc TM(m_pMalloc); IDiaTable *pTable; IFT(Table::Create(this, Table::Kind::InjectedSource, &pTable)); diff --git a/lib/DxilDia/DxilDiaSymbolManager.cpp b/lib/DxilDia/DxilDiaSymbolManager.cpp index ee8a66ee5..77b757015 100644 --- a/lib/DxilDia/DxilDiaSymbolManager.cpp +++ b/lib/DxilDia/DxilDiaSymbolManager.cpp @@ -286,7 +286,7 @@ public: IMalloc *pMalloc = pSession->GetMallocNoRef(); IFR(FunctionSymbol::Create(pMalloc, pSession, m_ID, m_Node, m_TypeID, m_Node->getType(), ppRet)); (*ppRet)->SetLexicalParent(m_ParentID); - (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str())); + (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8)); return S_OK; } @@ -378,7 +378,7 @@ public: IMalloc *pMalloc = pSession->GetMallocNoRef(); IFR(TypedefTypeSymbol::Create(pMalloc, pSession, m_ParentID, m_ID, m_Node, m_BaseTypeID, ppRet)); (*ppRet)->SetLexicalParent(m_ParentID); - (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str())); + (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8)); return S_OK; } @@ -413,7 +413,7 @@ public: IMalloc *pMalloc = pSession->GetMallocNoRef(); IFR(VectorTypeSymbol::Create(pMalloc, pSession, m_ParentID, m_ID, m_Node, m_ElemTyID, m_NumElts, ppRet)); (*ppRet)->SetLexicalParent(m_ParentID); - (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str())); + (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8)); return S_OK; } @@ -468,7 +468,7 @@ public: IMalloc *pMalloc = pSession->GetMallocNoRef(); IFR(GlobalVariableSymbol::Create(pMalloc, pSession, m_ID, m_GV, m_TypeID, m_Type, ppRet)); (*ppRet)->SetLexicalParent(m_ParentID); - (*ppRet)->SetName(CA2W(m_GV->getName().str().c_str())); + (*ppRet)->SetName(CA2W(m_GV->getName().str().c_str(), CP_UTF8)); (*ppRet)->SetIsHLSLData(true); return S_OK; } @@ -538,7 +538,7 @@ public: IMalloc *pMalloc = pSession->GetMallocNoRef(); IFR(LocalVariableSymbol::Create(pMalloc, pSession, m_ID, m_Node, m_TypeID, m_Type, m_VI->GetOffsetInUDT(), m_VI->GetDxilRegister(), ppRet)); (*ppRet)->SetLexicalParent(m_ParentID); - (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str())); + (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8)); (*ppRet)->SetDataKind(m_Node->getTag() == llvm::dwarf::DW_TAG_arg_variable ? DataIsParam : DataIsLocal); return S_OK; } @@ -572,7 +572,7 @@ public: IMalloc *pMalloc = pSession->GetMallocNoRef(); IFR(UDTFieldSymbol::Create(pMalloc, pSession, m_ID, m_Node, m_TypeID, m_Type, ppRet)); (*ppRet)->SetLexicalParent(m_ParentID); - (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str())); + (*ppRet)->SetName(CA2W(m_Node->getName().str().c_str(), CP_UTF8)); (*ppRet)->SetDataKind(m_Node->isStaticMember() ? DataIsStaticLocal : DataIsMember); return S_OK; } @@ -841,7 +841,7 @@ HRESULT dxil_dia::hlsl_symbols::CompilandEnvSymbol::CreateDefines(IMalloc *pMall for (llvm::MDNode::op_iterator it = definesNode->op_begin(); it != definesNode->op_end(); ++it) { llvm::StringRef strRef = llvm::dyn_cast(*it)->getString(); std::string str(strRef.begin(), strRef.size()); - CA2W cv(str.c_str()); + CA2W cv(str.c_str(), CP_UTF8); pBSTR.Append(cv); pBSTR.Append(L"\0", 1); } @@ -904,7 +904,7 @@ STDMETHODIMP dxil_dia::hlsl_symbols::TypeSymbol::get_name( DXASSERT(!this->HasName(), "Setting type name multiple times."); std::string Name; IFR(m_lazySymbolName(m_pSession, &Name)); - this->SetName(CA2W(Name.c_str())); + this->SetName(CA2W(Name.c_str(), CP_UTF8)); m_lazySymbolName = nullptr; } return Symbol::get_name(pRetVal); @@ -1349,7 +1349,7 @@ HRESULT dxil_dia::hlsl_symbols::SymbolManagerInit::CreateSubroutineType(DWORD dw if (!name) { OS << "???"; } else { - OS << CW2A((BSTR)name); + OS << CW2A((BSTR)name, CP_UTF8); } } if (first) { @@ -1416,7 +1416,7 @@ HRESULT dxil_dia::hlsl_symbols::SymbolManagerInit::CreateCompositeType(DWORD dwP if (!name) { OS << "???"; } else { - OS << CW2A((BSTR)name); + OS << CW2A((BSTR)name, CP_UTF8); } OS << "["; @@ -1613,7 +1613,7 @@ HRESULT dxil_dia::hlsl_symbols::SymbolManagerInit::HandleDerivedType(DWORD dwPar if (!name) { OS << "???"; } else { - OS << CW2A((BSTR)name); + OS << CW2A((BSTR)name, CP_UTF8); } OS << Qualifier; return S_OK; diff --git a/tools/clang/lib/Sema/SemaHLSL.cpp b/tools/clang/lib/Sema/SemaHLSL.cpp index 0a63b00fd..e23281ce2 100644 --- a/tools/clang/lib/Sema/SemaHLSL.cpp +++ b/tools/clang/lib/Sema/SemaHLSL.cpp @@ -3886,7 +3886,7 @@ public: const HLSL_INTRINSIC *pIntrinsic = nullptr; const HLSL_INTRINSIC *pPrior = nullptr; UINT64 lookupCookie = 0; - CA2W wideTypeName(typeName); + CA2W wideTypeName(typeName, CP_UTF8); HRESULT found = table->LookupIntrinsic(wideTypeName, L"*", &pIntrinsic, &lookupCookie); while (pIntrinsic != nullptr && SUCCEEDED(found)) { if (!AreIntrinsicTemplatesEquivalent(pIntrinsic, pPrior)) { diff --git a/tools/clang/tools/d3dcomp/d3dcomp.cpp b/tools/clang/tools/d3dcomp/d3dcomp.cpp index 90e9b82e1..47b8307ba 100644 --- a/tools/clang/tools/d3dcomp/d3dcomp.cpp +++ b/tools/clang/tools/d3dcomp/d3dcomp.cpp @@ -52,8 +52,8 @@ HRESULT CompileFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName, } try { - CA2W pEntrypointW(pEntrypoint); - CA2W pTargetProfileW(pTarget); + CA2W pEntrypointW(pEntrypoint, CP_UTF8); + CA2W pTargetProfileW(pTarget, CP_UTF8); std::vector defineValues; std::vector defines; if (pDefines) { @@ -61,9 +61,9 @@ HRESULT CompileFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName, // Convert to UTF-16. while (pCursor->Name) { - defineValues.push_back(std::wstring(CA2W(pCursor->Name))); + defineValues.push_back(std::wstring(CA2W(pCursor->Name, CP_UTF8))); if (pCursor->Definition) - defineValues.push_back(std::wstring(CA2W(pCursor->Definition))); + defineValues.push_back(std::wstring(CA2W(pCursor->Definition, CP_UTF8))); else defineValues.push_back(std::wstring()); ++pCursor; @@ -154,7 +154,7 @@ HRESULT WINAPI BridgeD3DCompile(LPCVOID pSrcData, SIZE_T SrcDataSize, } try { - CA2W pFileName(pSourceName); + CA2W pFileName(pSourceName, CP_UTF8); return CompileFromBlob(source, pFileName, pDefines, includeHandler, pEntrypoint, pTarget, Flags1, Flags2, ppCode, ppErrorMsgs); } catch (const std::bad_alloc &) { @@ -289,9 +289,9 @@ HRESULT PreprocessFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName, // Convert to UTF-16. while (pCursor->Name) { - defineValues.push_back(std::wstring(CA2W(pCursor->Name))); + defineValues.push_back(std::wstring(CA2W(pCursor->Name, CP_UTF8))); if (pCursor->Definition) - defineValues.push_back(std::wstring(CA2W(pCursor->Definition))); + defineValues.push_back(std::wstring(CA2W(pCursor->Definition, CP_UTF8))); else defineValues.push_back(std::wstring()); ++pCursor; @@ -357,7 +357,7 @@ HRESULT WINAPI BridgeD3DPreprocess(_In_reads_bytes_(SrcDataSize) LPCVOID pSrcDat } try { - CA2W pFileName(pSourceName); + CA2W pFileName(pSourceName, CP_UTF8); return PreprocessFromBlob(source, pFileName, pDefines, includeHandler, ppCodeText, ppErrorMsgs); } catch (const std::bad_alloc &) { diff --git a/tools/clang/tools/dxa/dxa.cpp b/tools/clang/tools/dxa/dxa.cpp index 71296859c..c091da530 100644 --- a/tools/clang/tools/dxa/dxa.cpp +++ b/tools/clang/tools/dxa/dxa.cpp @@ -207,7 +207,7 @@ bool DxaContext::ExtractFile(const char *pName) { return printedAny; } - CA2W WideName(pName); + CA2W WideName(pName, CP_UTF8); for (;;) { CComPtr pInjectedSourceUnk; ULONG fetched; diff --git a/tools/clang/tools/dxcompiler/dxcfilesystem.cpp b/tools/clang/tools/dxcompiler/dxcfilesystem.cpp index 674d226cd..8065ff556 100644 --- a/tools/clang/tools/dxcompiler/dxcfilesystem.cpp +++ b/tools/clang/tools/dxcompiler/dxcfilesystem.cpp @@ -775,7 +775,7 @@ public: } virtual int Open(const char *lpFileName, int flags, mode_t mode) throw() override { - HANDLE H = CreateFileW(CA2W(lpFileName), GENERIC_READ | GENERIC_WRITE, + HANDLE H = CreateFileW(CA2W(lpFileName, CP_UTF8), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL); if (H == INVALID_HANDLE_VALUE) @@ -788,7 +788,7 @@ public: // fake my way toward as linux-y a file_status as I can get virtual int Stat(const char *lpFileName, struct stat *Status) throw() override { - CA2W fileName_utf16(lpFileName); + CA2W fileName_utf16(lpFileName, CP_UTF8); DWORD attr = GetFileAttributesW(fileName_utf16); if (attr == INVALID_FILE_ATTRIBUTES) diff --git a/tools/clang/tools/dxcompiler/dxcompilerobj.cpp b/tools/clang/tools/dxcompiler/dxcompilerobj.cpp index 7058f2a11..8e517dab5 100644 --- a/tools/clang/tools/dxcompiler/dxcompilerobj.cpp +++ b/tools/clang/tools/dxcompiler/dxcompilerobj.cpp @@ -509,14 +509,15 @@ public: // Formerly API values. const char *pUtf8SourceName = opts.InputFile.empty() ? "hlsl.hlsl" : opts.InputFile.data(); - CA2W pUtf16SourceName(pUtf8SourceName); + CA2W pUtf16SourceName(pUtf8SourceName, CP_UTF8); const char *pUtf8EntryPoint = opts.EntryPoint.empty() ? "main" : opts.EntryPoint.data(); const char *pUtf8OutputName = isPreprocessing ? opts.Preprocess.data() : opts.OutputObject.empty() ? "" : opts.OutputObject.data(); CA2W pUtf16OutputName(isPreprocessing ? - opts.Preprocess.data() : pUtf8OutputName); + opts.Preprocess.data() : pUtf8OutputName, + CP_UTF8); LPCWSTR pObjectName = (!isPreprocessing && opts.OutputObject.empty()) ? nullptr : pUtf16OutputName.m_psz; IFT(primaryOutput.SetName(pObjectName));