<rdar://problem/13363214> Eliminate race condition between module rebuild and the global module index.

The global module index was querying the file manager for each of the
module files it knows about at load time, to prune out any out-of-date
information. The file manager would then cache the results of the
stat() falls used to find that module file.

Later, the same translation unit could end up trying to import one of the
module files that had previously been ignored by the module cache, but
after some other Clang instance rebuilt the module file to bring it
up-to-date. The stale stat() results in the file manager would
trigger a second rebuild of the already-up-to-date module, causing
failures down the line.

The global module index now lazily resolves its module file references
to actual AST reader module files only after the module file has been
loaded, eliminating the stat-caching race. Moreover, the AST reader
can communicate to its caller that a module file is missing (rather
than simply being out-of-date), allowing us to simplify the
module-loading logic and allowing the compiler to recover if a
dependent module file ends up getting deleted.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@177367 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Douglas Gregor 2013-03-19 00:28:20 +00:00
Родитель aa624954c5
Коммит 677e15ffee
13 изменённых файлов: 490 добавлений и 320 удалений

Просмотреть файл

@ -48,6 +48,7 @@
#include <string>
#include <utility>
#include <vector>
#include <sys/stat.h>
namespace llvm {
class MemoryBuffer;
@ -236,6 +237,8 @@ public:
Success,
/// \brief The AST file itself appears corrupted.
Failure,
/// \brief The AST file was missing.
Missing,
/// \brief The AST file is out-of-date relative to its input files,
/// and needs to be regenerated.
OutOfDate,
@ -949,6 +952,7 @@ private:
ASTReadResult ReadASTCore(StringRef FileName, ModuleKind Type,
SourceLocation ImportLoc, ModuleFile *ImportedBy,
SmallVectorImpl<ImportedModule> &Loaded,
off_t ExpectedSize, time_t ExpectedModTime,
unsigned ClientLoadCapabilities);
ASTReadResult ReadControlBlock(ModuleFile &F,
SmallVectorImpl<ImportedModule> &Loaded,
@ -1146,15 +1150,18 @@ public:
/// \brief The client can't handle any AST loading failures.
ARR_None = 0,
/// \brief The client can handle an AST file that cannot load because it
/// is missing.
ARR_Missing = 0x1,
/// \brief The client can handle an AST file that cannot load because it
/// is out-of-date relative to its input files.
ARR_OutOfDate = 0x1,
ARR_OutOfDate = 0x2,
/// \brief The client can handle an AST file that cannot load because it
/// was built with a different version of Clang.
ARR_VersionMismatch = 0x2,
ARR_VersionMismatch = 0x4,
/// \brief The client can handle an AST file that cannot load because it's
/// compiled configuration doesn't match that of the context it was
/// loaded into.
ARR_ConfigurationMismatch = 0x4
ARR_ConfigurationMismatch = 0x8
};
/// \brief Load the AST file designated by the given file name.

Просмотреть файл

@ -34,9 +34,44 @@ class DirectoryEntry;
class FileEntry;
class FileManager;
namespace serialization {
class ModuleFile;
};
using llvm::SmallVector;
using llvm::SmallVectorImpl;
using llvm::StringRef;
using serialization::ModuleFile;
/// \brief Abstract class that resolves a module file name to a ModuleFile
/// pointer, which is used to uniquely describe a module file.
class ModuleFileNameResolver {
public:
virtual ~ModuleFileNameResolver();
/// \brief Attempt to resolve the given module file name to a specific,
/// already-loaded module.
///
/// \param FileName The name of the module file.
///
/// \param ExpectedSize The size that the module file is expected to have.
/// If the actual size differs, the resolver should return \c true.
///
/// \param ExpectedModTime The modification time that the module file is
/// expected to have. If the actual modification time differs, the resolver
/// should return \c true.
///
/// \param File Will be set to the module file if there is one, or null
/// otherwise.
///
/// \returns True if a module file exists but does not meet the size/
/// modification time criteria, false if the module file is available or has
/// not yet been loaded.
virtual bool resolveModuleFileName(StringRef FileName,
off_t ExpectedSize,
time_t ExpectedModTime,
ModuleFile *&File) = 0;
};
/// \brief A global index for a set of module files, providing information about
/// the identifiers within those module files.
@ -54,6 +89,9 @@ class GlobalModuleIndex {
/// as the global module index is live.
llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
/// \brief The module file name resolver.
ModuleFileNameResolver *Resolver;
/// \brief The hash table.
///
/// This pointer actually points to a IdentifierIndexTable object,
@ -63,25 +101,39 @@ class GlobalModuleIndex {
/// \brief Information about a given module file.
struct ModuleInfo {
ModuleInfo() : File() { }
ModuleInfo() : File(), Size(), ModTime() { }
/// \brief The module file entry.
const FileEntry *File;
/// \brief The module file, if it is known.
ModuleFile *File;
/// \brief The module files on which this module directly depends.
llvm::SmallVector<const FileEntry *, 4> Dependencies;
/// \brief The module file name.
std::string FileName;
/// \brief Size of the module file at the time the global index was built.
off_t Size;
/// \brief Modification time of the module file at the time the global
/// index was built.
time_t ModTime;
/// \brief The module IDs on which this module directly depends.
/// FIXME: We don't really need a vector here.
llvm::SmallVector<unsigned, 4> Dependencies;
/// \brief The module IDs that directly depend on this module.
llvm::SmallVector<unsigned, 4> ImportedBy;
};
/// \brief A mapping from module IDs to information about each module.
///
/// This vector may have gaps, if module files have been removed or have
/// been updated since the index was built. A gap is indicated by an empty
/// \c File pointer.
/// file name.
llvm::SmallVector<ModuleInfo, 16> Modules;
/// \brief Lazily-populated mapping from module file entries to their
/// \brief Lazily-populated mapping from module files to their
/// corresponding index into the \c Modules vector.
llvm::DenseMap<const FileEntry *, unsigned> ModulesByFile;
llvm::DenseMap<ModuleFile *, unsigned> ModulesByFile;
/// \brief The number of identifier lookups we performed.
unsigned NumIdentifierLookups;
@ -91,7 +143,7 @@ class GlobalModuleIndex {
unsigned NumIdentifierLookupHits;
/// \brief Internal constructor. Use \c readIndex() to read an index.
explicit GlobalModuleIndex(FileManager &FileMgr, llvm::MemoryBuffer *Buffer,
explicit GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
llvm::BitstreamCursor Cursor);
GlobalModuleIndex(const GlobalModuleIndex &) LLVM_DELETED_FUNCTION;
@ -115,29 +167,27 @@ public:
/// \brief Read a global index file for the given directory.
///
/// \param FileMgr The file manager to use for reading files.
///
/// \param Path The path to the specific module cache where the module files
/// for the intended configuration reside.
///
/// \returns A pair containing the global module index (if it exists) and
/// the error code.
static std::pair<GlobalModuleIndex *, ErrorCode>
readIndex(FileManager &FileMgr, StringRef Path);
readIndex(StringRef Path);
/// \brief Retrieve the set of modules that have up-to-date indexes.
///
/// \param ModuleFiles Will be populated with the set of module files that
/// have been indexed.
void getKnownModules(SmallVectorImpl<const FileEntry *> &ModuleFiles);
void getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles);
/// \brief Retrieve the set of module files on which the given module file
/// directly depends.
void getModuleDependencies(const FileEntry *ModuleFile,
SmallVectorImpl<const FileEntry *> &Dependencies);
void getModuleDependencies(ModuleFile *File,
SmallVectorImpl<ModuleFile *> &Dependencies);
/// \brief A set of module files in which we found a result.
typedef llvm::SmallPtrSet<const FileEntry *, 4> HitSet;
typedef llvm::SmallPtrSet<ModuleFile *, 4> HitSet;
/// \brief Look for all of the module files with information about the given
/// identifier, e.g., a global function, variable, or type with that name.
@ -150,6 +200,20 @@ public:
/// \returns true if the identifier is known to the index, false otherwise.
bool lookupIdentifier(StringRef Name, HitSet &Hits);
/// \brief Set the module file name resolver.
void setResolver(ModuleFileNameResolver *Resolver) {
this->Resolver = Resolver;
}
/// \brief Note that additional modules have been loaded, which invalidates
/// the module file -> module cache.
void noteAdditionalModulesLoaded() {
ModulesByFile.clear();
}
/// \brief Resolve the module file for the module with the given ID.
ModuleFile *resolveModuleFile(unsigned ID);
/// \brief Print statistics to standard error.
void printStats();

Просмотреть файл

@ -16,17 +16,18 @@
#define LLVM_CLANG_SERIALIZATION_MODULE_MANAGER_H
#include "clang/Basic/FileManager.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/Module.h"
#include "llvm/ADT/DenseMap.h"
namespace clang {
class GlobalModuleIndex;
class ModuleMap;
namespace serialization {
/// \brief Manages the set of modules loaded by an AST reader.
class ModuleManager {
class ModuleManager : public ModuleFileNameResolver {
/// \brief The chain of AST files. The first entry is the one named by the
/// user, the last one is the one that doesn't depend on anything further.
SmallVector<ModuleFile *, 2> Chain;
@ -143,6 +144,19 @@ public:
/// \brief Number of modules loaded
unsigned size() const { return Chain.size(); }
/// \brief The result of attempting to add a new module.
enum AddModuleResult {
/// \brief The module file had already been loaded.
AlreadyLoaded,
/// \brief The module file was just loaded in response to this call.
NewlyLoaded,
/// \brief The module file is missing.
Missing,
/// \brief The module file is out-of-date.
OutOfDate
};
/// \brief Attempts to create a new module and add it to the list of known
/// modules.
///
@ -157,18 +171,30 @@ public:
///
/// \param Generation The generation in which this module was loaded.
///
/// \param ExpectedSize The expected size of the module file, used for
/// validation. This will be zero if unknown.
///
/// \param ExpectedModTime The expected modification time of the module
/// file, used for validation. This will be zero if unknown.
///
/// \param Module A pointer to the module file if the module was successfully
/// loaded.
///
/// \param ErrorStr Will be set to a non-empty string if any errors occurred
/// while trying to load the module.
///
/// \return A pointer to the module that corresponds to this file name,
/// and a boolean indicating whether the module was newly added.
std::pair<ModuleFile *, bool>
addModule(StringRef FileName, ModuleKind Type, SourceLocation ImportLoc,
ModuleFile *ImportedBy, unsigned Generation,
std::string &ErrorStr);
/// and a value indicating whether the module was loaded.
AddModuleResult addModule(StringRef FileName, ModuleKind Type,
SourceLocation ImportLoc,
ModuleFile *ImportedBy, unsigned Generation,
off_t ExpectedSize, time_t ExpectedModTime,
ModuleFile *&Module,
std::string &ErrorStr);
/// \brief Remove the given set of modules.
void removeModules(ModuleIterator first, ModuleIterator last);
void removeModules(ModuleIterator first, ModuleIterator last,
ModuleMap *modMap);
/// \brief Add an in-memory buffer the list of known buffers
void addInMemoryBuffer(StringRef FileName, llvm::MemoryBuffer *Buffer);
@ -200,7 +226,7 @@ public:
/// Any module that is known to both the global module index and the module
/// manager that is *not* in this set can be skipped.
void visit(bool (*Visitor)(ModuleFile &M, void *UserData), void *UserData,
llvm::SmallPtrSet<const FileEntry *, 4> *ModuleFilesHit = 0);
llvm::SmallPtrSet<ModuleFile *, 4> *ModuleFilesHit = 0);
/// \brief Visit each of the modules with a depth-first traversal.
///
@ -221,7 +247,34 @@ public:
void visitDepthFirst(bool (*Visitor)(ModuleFile &M, bool Preorder,
void *UserData),
void *UserData);
/// \brief Attempt to resolve the given module file name to a file entry.
///
/// \param FileName The name of the module file.
///
/// \param ExpectedSize The size that the module file is expected to have.
/// If the actual size differs, the resolver should return \c true.
///
/// \param ExpectedModTime The modification time that the module file is
/// expected to have. If the actual modification time differs, the resolver
/// should return \c true.
///
/// \param File Will be set to the file if there is one, or null
/// otherwise.
///
/// \returns True if a file exists but does not meet the size/
/// modification time criteria, false if the file is either available and
/// suitable, or is missing.
bool lookupModuleFile(StringRef FileName,
off_t ExpectedSize,
time_t ExpectedModTime,
const FileEntry *&File);
virtual bool resolveModuleFileName(StringRef FileName,
off_t ExpectedSize,
time_t ExpectedModTime,
ModuleFile *&File);
/// \brief View the graphviz representation of the module graph.
void viewGraph();
};

Просмотреть файл

@ -801,6 +801,7 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
break;
case ASTReader::Failure:
case ASTReader::Missing:
case ASTReader::OutOfDate:
case ASTReader::VersionMismatch:
case ASTReader::ConfigurationMismatch:

Просмотреть файл

@ -47,6 +47,7 @@ static ASTReader *createASTReader(CompilerInstance &CI,
return Reader.take();
case ASTReader::Failure:
case ASTReader::Missing:
case ASTReader::OutOfDate:
case ASTReader::VersionMismatch:
case ASTReader::ConfigurationMismatch:

Просмотреть файл

@ -338,6 +338,7 @@ CompilerInstance::createPCHExternalASTSource(StringRef Path,
// Unrecoverable failure: don't even try to process the input file.
break;
case ASTReader::Missing:
case ASTReader::OutOfDate:
case ASTReader::VersionMismatch:
case ASTReader::ConfigurationMismatch:
@ -945,88 +946,6 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
} else
ModuleFileName = PP->getHeaderSearchInfo().getModuleFileName(ModuleName);
if (ModuleFileName.empty()) {
getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_found)
<< ModuleName
<< SourceRange(ImportLoc, ModuleNameLoc);
LastModuleImportLoc = ImportLoc;
LastModuleImportResult = ModuleLoadResult();
return LastModuleImportResult;
}
const FileEntry *ModuleFile
= getFileManager().getFile(ModuleFileName, /*OpenFile=*/false,
/*CacheFailure=*/false);
bool BuildingModule = false;
if (!ModuleFile && Module) {
// The module is not cached, but we have a module map from which we can
// build the module.
// Check whether there is a cycle in the module graph.
ModuleBuildStack Path = getSourceManager().getModuleBuildStack();
ModuleBuildStack::iterator Pos = Path.begin(), PosEnd = Path.end();
for (; Pos != PosEnd; ++Pos) {
if (Pos->first == ModuleName)
break;
}
if (Pos != PosEnd) {
SmallString<256> CyclePath;
for (; Pos != PosEnd; ++Pos) {
CyclePath += Pos->first;
CyclePath += " -> ";
}
CyclePath += ModuleName;
getDiagnostics().Report(ModuleNameLoc, diag::err_module_cycle)
<< ModuleName << CyclePath;
return ModuleLoadResult();
}
// Check whether we have already attempted to build this module (but
// failed).
if (getPreprocessorOpts().FailedModules &&
getPreprocessorOpts().FailedModules->hasAlreadyFailed(ModuleName)) {
getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_built)
<< ModuleName
<< SourceRange(ImportLoc, ModuleNameLoc);
ModuleBuildFailed = true;
return ModuleLoadResult();
}
BuildingModule = true;
compileModule(*this, ModuleNameLoc, Module, ModuleFileName);
ModuleFile = FileMgr->getFile(ModuleFileName, /*OpenFile=*/false,
/*CacheFailure=*/false);
if (!ModuleFile && getPreprocessorOpts().FailedModules)
getPreprocessorOpts().FailedModules->addFailed(ModuleName);
}
if (!ModuleFile) {
getDiagnostics().Report(ModuleNameLoc,
BuildingModule? diag::err_module_not_built
: diag::err_module_not_found)
<< ModuleName
<< SourceRange(ImportLoc, ModuleNameLoc);
ModuleBuildFailed = true;
return ModuleLoadResult();
}
// If there is already a module file associated with this module, make sure
// it is the same as the module file we're looking for. Otherwise, we
// have two module files for the same module.
if (const FileEntry *CurModuleFile = Module? Module->getASTFile() : 0) {
if (CurModuleFile != ModuleFile) {
getDiagnostics().Report(ModuleNameLoc, diag::err_module_file_conflict)
<< ModuleName
<< CurModuleFile->getName()
<< ModuleFile->getName();
ModuleBuildFailed = true;
return ModuleLoadResult();
}
}
// If we don't already have an ASTReader, create one now.
if (!ModuleManager) {
if (!hasASTContext())
@ -1056,21 +975,53 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
ModuleManager->StartTranslationUnit(&getASTConsumer());
}
// Try to load the module we found.
unsigned ARRFlags = ASTReader::ARR_None;
if (Module)
ARRFlags |= ASTReader::ARR_OutOfDate;
switch (ModuleManager->ReadAST(ModuleFile->getName(),
serialization::MK_Module, ImportLoc,
ARRFlags)) {
// Try to load the module file.
unsigned ARRFlags = ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing;
switch (ModuleManager->ReadAST(ModuleFileName, serialization::MK_Module,
ImportLoc, ARRFlags)) {
case ASTReader::Success:
break;
case ASTReader::OutOfDate: {
// The module file is out-of-date. Rebuild it.
getFileManager().invalidateCache(ModuleFile);
// The module file is out-of-date. Remove it, then rebuild it.
bool Existed;
llvm::sys::fs::remove(ModuleFileName, Existed);
}
// Fall through to build the module again.
case ASTReader::Missing: {
// The module file is (now) missing. Build it.
// If we don't have a module, we don't know how to build the module file.
// Complain and return.
if (!Module) {
getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_found)
<< ModuleName
<< SourceRange(ImportLoc, ModuleNameLoc);
ModuleBuildFailed = true;
return ModuleLoadResult();
}
// Check whether there is a cycle in the module graph.
ModuleBuildStack ModPath = getSourceManager().getModuleBuildStack();
ModuleBuildStack::iterator Pos = ModPath.begin(), PosEnd = ModPath.end();
for (; Pos != PosEnd; ++Pos) {
if (Pos->first == ModuleName)
break;
}
if (Pos != PosEnd) {
SmallString<256> CyclePath;
for (; Pos != PosEnd; ++Pos) {
CyclePath += Pos->first;
CyclePath += " -> ";
}
CyclePath += ModuleName;
getDiagnostics().Report(ModuleNameLoc, diag::err_module_cycle)
<< ModuleName << CyclePath;
return ModuleLoadResult();
}
// Check whether we have already attempted to build this module (but
// failed).
@ -1083,15 +1034,23 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
return ModuleLoadResult();
}
// Try to compile the module.
compileModule(*this, ModuleNameLoc, Module, ModuleFileName);
// Try loading the module again.
ModuleFile = FileMgr->getFile(ModuleFileName, /*OpenFile=*/false,
/*CacheFailure=*/false);
if (!ModuleFile ||
ModuleManager->ReadAST(ModuleFileName,
// Try to read the module file, now that we've compiled it.
ASTReader::ASTReadResult ReadResult
= ModuleManager->ReadAST(ModuleFileName,
serialization::MK_Module, ImportLoc,
ASTReader::ARR_None) != ASTReader::Success) {
ASTReader::ARR_Missing);
if (ReadResult != ASTReader::Success) {
if (ReadResult == ASTReader::Missing) {
getDiagnostics().Report(ModuleNameLoc,
Module? diag::err_module_not_built
: diag::err_module_not_found)
<< ModuleName
<< SourceRange(ImportLoc, ModuleNameLoc);
}
if (getPreprocessorOpts().FailedModules)
getPreprocessorOpts().FailedModules->addFailed(ModuleName);
KnownModules[Path[0].first] = 0;
@ -1125,10 +1084,6 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
.findModule((Path[0].first->getName()));
}
if (Module) {
Module->setASTFile(ModuleFile);
}
// Cache the result of this top-level module lookup for later.
Known = KnownModules.insert(std::make_pair(Path[0].first, Module)).first;
}

Просмотреть файл

@ -1616,8 +1616,10 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
|| StoredTime != File->getModificationTime()
#endif
)) {
if (Complain)
if (Complain) {
Error(diag::err_fe_pch_file_modified, Filename, F.FileName);
}
IsOutOfDate = true;
}
@ -1774,6 +1776,8 @@ ASTReader::ReadControlBlock(ModuleFile &F,
// location info are setup.
SourceLocation ImportLoc =
SourceLocation::getFromRawEncoding(Record[Idx++]);
off_t StoredSize = (off_t)Record[Idx++];
time_t StoredModTime = (time_t)Record[Idx++];
unsigned Length = Record[Idx++];
SmallString<128> ImportedFile(Record.begin() + Idx,
Record.begin() + Idx + Length);
@ -1781,9 +1785,11 @@ ASTReader::ReadControlBlock(ModuleFile &F,
// Load the AST file.
switch(ReadASTCore(ImportedFile, ImportedKind, ImportLoc, &F, Loaded,
StoredSize, StoredModTime,
ClientLoadCapabilities)) {
case Failure: return Failure;
// If we have to ignore the dependency, we'll have to ignore this too.
case Missing: return Missing;
case OutOfDate: return OutOfDate;
case VersionMismatch: return VersionMismatch;
case ConfigurationMismatch: return ConfigurationMismatch;
@ -2774,7 +2780,7 @@ bool ASTReader::loadGlobalIndex() {
StringRef ModuleCachePath
= getPreprocessor().getHeaderSearchInfo().getModuleCachePath();
std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode> Result
= GlobalModuleIndex::readIndex(FileMgr, ModuleCachePath);
= GlobalModuleIndex::readIndex(ModuleCachePath);
if (!Result.first)
return true;
@ -2799,13 +2805,18 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
SmallVector<ImportedModule, 4> Loaded;
switch(ASTReadResult ReadResult = ReadASTCore(FileName, Type, ImportLoc,
/*ImportedBy=*/0, Loaded,
0, 0,
ClientLoadCapabilities)) {
case Failure:
case Missing:
case OutOfDate:
case VersionMismatch:
case ConfigurationMismatch:
case HadErrors:
ModuleMgr.removeModules(ModuleMgr.begin() + NumModules, ModuleMgr.end());
ModuleMgr.removeModules(ModuleMgr.begin() + NumModules, ModuleMgr.end(),
Context.getLangOpts().Modules
? &PP.getHeaderSearchInfo().getModuleMap()
: 0);
// If we find that any modules are unusable, the global index is going
// to be out-of-date. Just remove it.
@ -2923,26 +2934,53 @@ ASTReader::ReadASTCore(StringRef FileName,
SourceLocation ImportLoc,
ModuleFile *ImportedBy,
SmallVectorImpl<ImportedModule> &Loaded,
off_t ExpectedSize, time_t ExpectedModTime,
unsigned ClientLoadCapabilities) {
ModuleFile *M;
bool NewModule;
std::string ErrorStr;
llvm::tie(M, NewModule) = ModuleMgr.addModule(FileName, Type, ImportLoc,
ImportedBy, CurrentGeneration,
ErrorStr);
ModuleManager::AddModuleResult AddResult
= ModuleMgr.addModule(FileName, Type, ImportLoc, ImportedBy,
CurrentGeneration, ExpectedSize, ExpectedModTime,
M, ErrorStr);
if (!M) {
// We couldn't load the module.
std::string Msg = "Unable to load module \"" + FileName.str() + "\": "
+ ErrorStr;
Error(Msg);
switch (AddResult) {
case ModuleManager::AlreadyLoaded:
return Success;
case ModuleManager::NewlyLoaded:
// Load module file below.
break;
case ModuleManager::Missing:
// The module file was missing; if the client handle handle, that, return
// it.
if (ClientLoadCapabilities & ARR_Missing)
return Missing;
// Otherwise, return an error.
{
std::string Msg = "Unable to load module \"" + FileName.str() + "\": "
+ ErrorStr;
Error(Msg);
}
return Failure;
case ModuleManager::OutOfDate:
// We couldn't load the module file because it is out-of-date. If the
// client can handle out-of-date, return it.
if (ClientLoadCapabilities & ARR_OutOfDate)
return OutOfDate;
// Otherwise, return an error.
{
std::string Msg = "Unable to load module \"" + FileName.str() + "\": "
+ ErrorStr;
Error(Msg);
}
return Failure;
}
if (!NewModule) {
// We've already loaded this module.
return Success;
}
assert(M && "Missing module file");
// FIXME: This seems rather a hack. Should CurrentDir be part of the
// module?
@ -2997,6 +3035,7 @@ ASTReader::ReadASTCore(StringRef FileName,
break;
case Failure: return Failure;
case Missing: return Missing;
case OutOfDate: return OutOfDate;
case VersionMismatch: return VersionMismatch;
case ConfigurationMismatch: return ConfigurationMismatch;
@ -3467,18 +3506,22 @@ bool ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
return true;
}
if (const FileEntry *CurFile = CurrentModule->getASTFile()) {
if (CurFile != F.File) {
if (!Diags.isDiagnosticInFlight()) {
Diag(diag::err_module_file_conflict)
<< CurrentModule->getTopLevelModuleName()
<< CurFile->getName()
<< F.File->getName();
if (!ParentModule) {
if (const FileEntry *CurFile = CurrentModule->getASTFile()) {
if (CurFile != F.File) {
if (!Diags.isDiagnosticInFlight()) {
Diag(diag::err_module_file_conflict)
<< CurrentModule->getTopLevelModuleName()
<< CurFile->getName()
<< F.File->getName();
}
return true;
}
return true;
}
CurrentModule->setASTFile(F.File);
}
CurrentModule->setASTFile(F.File);
CurrentModule->IsFromModuleFile = true;
CurrentModule->IsSystem = IsSystem || CurrentModule->IsSystem;
CurrentModule->InferSubmodules = InferSubmodules;

Просмотреть файл

@ -1034,6 +1034,8 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
Record.push_back((unsigned)(*M)->Kind); // FIXME: Stable encoding
AddSourceLocation((*M)->ImportLoc, Record);
Record.push_back((*M)->File->getSize());
Record.push_back((*M)->File->getModificationTime());
// FIXME: This writes the absolute path for AST files we depend on.
const std::string &FileName = (*M)->FileName;
Record.push_back(FileName.size());

Просмотреть файл

@ -57,6 +57,8 @@ static const char * const IndexFileName = "modules.idx";
/// \brief The global index file version.
static const unsigned CurrentVersion = 1;
ModuleFileNameResolver::~ModuleFileNameResolver() { }
//----------------------------------------------------------------------------//
// Global module index reader.
//----------------------------------------------------------------------------//
@ -115,29 +117,16 @@ public:
typedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable;
/// \brief Module information as it was loaded from the index file.
struct LoadedModuleInfo {
const FileEntry *File;
SmallVector<unsigned, 2> Dependencies;
SmallVector<unsigned, 2> ImportedBy;
};
}
GlobalModuleIndex::GlobalModuleIndex(FileManager &FileMgr,
llvm::MemoryBuffer *Buffer,
GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
llvm::BitstreamCursor Cursor)
: Buffer(Buffer), IdentifierIndex(),
: Buffer(Buffer), Resolver(), IdentifierIndex(),
NumIdentifierLookups(), NumIdentifierLookupHits()
{
typedef llvm::DenseMap<unsigned, LoadedModuleInfo> LoadedModulesMap;
LoadedModulesMap LoadedModules;
// Read the global index.
unsigned LargestID = 0;
bool InGlobalIndexBlock = false;
bool Done = false;
bool AnyOutOfDate = false;
while (!Done) {
llvm::BitstreamEntry Entry = Cursor.advance();
@ -185,41 +174,33 @@ GlobalModuleIndex::GlobalModuleIndex(FileManager &FileMgr,
case MODULE: {
unsigned Idx = 0;
unsigned ID = Record[Idx++];
if (ID > LargestID)
LargestID = ID;
off_t Size = Record[Idx++];
time_t ModTime = Record[Idx++];
// Make room for this module's information.
if (ID == Modules.size())
Modules.push_back(ModuleInfo());
else
Modules.resize(ID + 1);
// Size/modification time for this module file at the time the
// global index was built.
Modules[ID].Size = Record[Idx++];
Modules[ID].ModTime = Record[Idx++];
// File name.
unsigned NameLen = Record[Idx++];
llvm::SmallString<64> FileName(Record.begin() + Idx,
Record.begin() + Idx + NameLen);
Modules[ID].FileName.assign(Record.begin() + Idx,
Record.begin() + Idx + NameLen);
Idx += NameLen;
// Dependencies
unsigned NumDeps = Record[Idx++];
llvm::SmallVector<unsigned, 2>
Dependencies(Record.begin() + Idx, Record.begin() + Idx + NumDeps);
Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
Record.begin() + Idx,
Record.begin() + Idx + NumDeps);
Idx += NumDeps;
// Find the file. If we can't find it, ignore it.
const FileEntry *File = FileMgr.getFile(FileName, /*openFile=*/false,
/*cacheFailure=*/false);
if (!File) {
AnyOutOfDate = true;
break;
}
// If the module file is newer than the index, ignore it.
if (File->getSize() != Size || File->getModificationTime() != ModTime) {
AnyOutOfDate = true;
break;
}
// Record this module. The dependencies will be resolved later.
LoadedModuleInfo &Info = LoadedModules[ID];
Info.File = File;
Info.Dependencies.swap(Dependencies);
// Make sure we're at the end of the record.
assert(Idx == Record.size() && "More module info?");
break;
}
@ -235,74 +216,11 @@ GlobalModuleIndex::GlobalModuleIndex(FileManager &FileMgr,
}
}
// If there are any modules that have gone out-of-date, prune out any modules
// that depend on them.
if (AnyOutOfDate) {
// First, build back links in the module dependency graph.
SmallVector<unsigned, 4> Stack;
for (LoadedModulesMap::iterator LM = LoadedModules.begin(),
LMEnd = LoadedModules.end();
LM != LMEnd; ++LM) {
unsigned ID = LM->first;
// If this module is out-of-date, push it onto the stack.
if (LM->second.File == 0)
Stack.push_back(ID);
for (unsigned I = 0, N = LM->second.Dependencies.size(); I != N; ++I) {
unsigned DepID = LM->second.Dependencies[I];
LoadedModulesMap::iterator Known = LoadedModules.find(DepID);
if (Known == LoadedModules.end() || !Known->second.File) {
// The dependency was out-of-date, so mark us as out of date.
// This is just an optimization.
if (LM->second.File)
Stack.push_back(ID);
LM->second.File = 0;
continue;
}
// Record this reverse dependency.
Known->second.ImportedBy.push_back(ID);
}
}
// Second, walk the back links from out-of-date modules to those modules
// that depend on them, making those modules out-of-date as well.
while (!Stack.empty()) {
unsigned ID = Stack.back();
Stack.pop_back();
LoadedModuleInfo &Info = LoadedModules[ID];
for (unsigned I = 0, N = Info.ImportedBy.size(); I != N; ++I) {
unsigned FromID = Info.ImportedBy[I];
if (LoadedModules[FromID].File) {
LoadedModules[FromID].File = 0;
Stack.push_back(FromID);
}
}
}
}
// Allocate the vector containing information about all of the modules.
Modules.resize(LargestID + 1);
for (LoadedModulesMap::iterator LM = LoadedModules.begin(),
LMEnd = LoadedModules.end();
LM != LMEnd; ++LM) {
if (!LM->second.File)
continue;
Modules[LM->first].File = LM->second.File;
// Resolve dependencies. Drop any we can't resolve due to out-of-date
// module files.
for (unsigned I = 0, N = LM->second.Dependencies.size(); I != N; ++I) {
unsigned DepID = LM->second.Dependencies[I];
LoadedModulesMap::iterator Known = LoadedModules.find(DepID);
if (Known == LoadedModules.end() || !Known->second.File)
continue;
Modules[LM->first].Dependencies.push_back(Known->second.File);
// Compute imported-by relation.
for (unsigned ID = 0, IDEnd = Modules.size(); ID != IDEnd; ++ID) {
for (unsigned D = 0, DEnd = Modules[ID].Dependencies.size();
D != DEnd; ++D) {
Modules[Modules[ID].Dependencies[D]].ImportedBy.push_back(ID);
}
}
}
@ -310,15 +228,14 @@ GlobalModuleIndex::GlobalModuleIndex(FileManager &FileMgr,
GlobalModuleIndex::~GlobalModuleIndex() { }
std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
GlobalModuleIndex::readIndex(FileManager &FileMgr, StringRef Path) {
GlobalModuleIndex::readIndex(StringRef Path) {
// Load the index file, if it's there.
llvm::SmallString<128> IndexPath;
IndexPath += Path;
llvm::sys::path::append(IndexPath, IndexFileName);
llvm::OwningPtr<llvm::MemoryBuffer> Buffer(
FileMgr.getBufferForFile(IndexPath));
if (!Buffer)
llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
if (llvm::MemoryBuffer::getFile(IndexPath, Buffer) != llvm::errc::success)
return std::make_pair((GlobalModuleIndex *)0, EC_NotFound);
/// \brief The bitstream reader from which we'll read the AST file.
@ -336,38 +253,41 @@ GlobalModuleIndex::readIndex(FileManager &FileMgr, StringRef Path) {
return std::make_pair((GlobalModuleIndex *)0, EC_IOError);
}
return std::make_pair(new GlobalModuleIndex(FileMgr, Buffer.take(), Cursor),
EC_None);
return std::make_pair(new GlobalModuleIndex(Buffer.take(), Cursor), EC_None);
}
void GlobalModuleIndex::getKnownModules(
SmallVectorImpl<const FileEntry *> &ModuleFiles) {
void
GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
ModuleFiles.clear();
for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
if (Modules[I].File)
ModuleFiles.push_back(Modules[I].File);
if (ModuleFile *File = resolveModuleFile(I))
ModuleFiles.push_back(File);
}
}
void GlobalModuleIndex::getModuleDependencies(
const clang::FileEntry *ModuleFile,
SmallVectorImpl<const clang::FileEntry *> &Dependencies) {
ModuleFile *File,
SmallVectorImpl<ModuleFile *> &Dependencies) {
// If the file -> index mapping is empty, populate it now.
if (ModulesByFile.empty()) {
for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
if (Modules[I].File)
ModulesByFile[Modules[I].File] = I;
resolveModuleFile(I);
}
}
// Look for information about this module file.
llvm::DenseMap<const FileEntry *, unsigned>::iterator Known
= ModulesByFile.find(ModuleFile);
llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
= ModulesByFile.find(File);
if (Known == ModulesByFile.end())
return;
// Record dependencies.
Dependencies = Modules[Known->second].Dependencies;
Dependencies.clear();
ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
if (ModuleFile *MF = resolveModuleFile(I))
Dependencies.push_back(MF);
}
}
bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
@ -388,17 +308,62 @@ bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
SmallVector<unsigned, 2> ModuleIDs = *Known;
for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
unsigned ID = ModuleIDs[I];
if (ID >= Modules.size() || !Modules[ID].File)
continue;
Hits.insert(Modules[ID].File);
if (ModuleFile *File = resolveModuleFile(ModuleIDs[I]))
Hits.insert(File);
}
++NumIdentifierLookupHits;
return true;
}
ModuleFile *GlobalModuleIndex::resolveModuleFile(unsigned ID) {
assert(ID < Modules.size() && "Out-of-bounds module index");
// If we already have a module file, return it.
if (Modules[ID].File)
return Modules[ID].File;
// If we don't have a file name, or if there is no resolver, we can't
// resolve this.
if (Modules[ID].FileName.empty() || !Resolver)
return 0;
// Try to resolve this module file.
ModuleFile *File;
if (Resolver->resolveModuleFileName(Modules[ID].FileName, Modules[ID].Size,
Modules[ID].ModTime, File)) {
// Clear out the module files for anything that depends on this module.
llvm::SmallVector<unsigned, 8> Stack;
Stack.push_back(ID);
while (!Stack.empty()) {
unsigned Victim = Stack.back();
Stack.pop_back();
// Mark this file as ignored.
Modules[Victim].File = 0;
Modules[Victim].FileName.clear();
// Push any not-yet-ignored imported modules onto the stack.
for (unsigned I = 0, N = Modules[Victim].ImportedBy.size(); I != N; ++I) {
unsigned NextVictim = Modules[Victim].ImportedBy[I];
if (!Modules[NextVictim].FileName.empty())
Stack.push_back(NextVictim);
}
}
return 0;
}
// If we have a module file, record it.
if (File) {
Modules[ID].File = File;
ModulesByFile[File] = ID;
}
return File;
}
void GlobalModuleIndex::printStats() {
std::fprintf(stderr, "*** Global Module Index Statistics:\n");
if (NumIdentifierLookups) {
@ -629,6 +594,10 @@ bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
// Skip the import location
++Idx;
// Load stored size/modification time.
off_t StoredSize = (off_t)Record[Idx++];
time_t StoredModTime = (time_t)Record[Idx++];
// Retrieve the imported file name.
unsigned Length = Record[Idx++];
SmallString<128> ImportedFile(Record.begin() + Idx,
@ -639,7 +608,9 @@ bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
const FileEntry *DependsOnFile
= FileMgr.getFile(ImportedFile, /*openFile=*/false,
/*cacheFailure=*/false);
if (!DependsOnFile)
if (!DependsOnFile ||
(StoredSize != DependsOnFile->getSize()) ||
(StoredModTime != DependsOnFile->getModificationTime()))
return true;
// Record the dependency.

Просмотреть файл

@ -11,9 +11,11 @@
// modules for the ASTReader.
//
//===----------------------------------------------------------------------===//
#include "clang/Lex/ModuleMap.h"
#include "clang/Serialization/ModuleManager.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/PathV2.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/system_error.h"
@ -36,18 +38,28 @@ llvm::MemoryBuffer *ModuleManager::lookupBuffer(StringRef Name) {
return InMemoryBuffers[Entry];
}
std::pair<ModuleFile *, bool>
ModuleManager::AddModuleResult
ModuleManager::addModule(StringRef FileName, ModuleKind Type,
SourceLocation ImportLoc, ModuleFile *ImportedBy,
unsigned Generation, std::string &ErrorStr) {
const FileEntry *Entry = FileMgr.getFile(FileName, /*openFile=*/false,
/*cacheFailure=*/false);
unsigned Generation,
off_t ExpectedSize, time_t ExpectedModTime,
ModuleFile *&Module,
std::string &ErrorStr) {
Module = 0;
// Look for the file entry. This only fails if the expected size or
// modification time differ.
const FileEntry *Entry;
if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, Entry))
return OutOfDate;
if (!Entry && FileName != "-") {
ErrorStr = "file not found";
return std::make_pair(static_cast<ModuleFile*>(0), false);
return Missing;
}
// Check whether we already loaded this module, before
// Check whether we already loaded this module, before
AddModuleResult Result = AlreadyLoaded;
ModuleFile *&ModuleEntry = Modules[Entry];
bool NewModule = false;
if (!ModuleEntry) {
@ -77,12 +89,15 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
New->Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrorStr));
if (!New->Buffer)
return std::make_pair(static_cast<ModuleFile*>(0), false);
return Missing;
}
// Initialize the stream
New->StreamFile.init((const unsigned char *)New->Buffer->getBufferStart(),
(const unsigned char *)New->Buffer->getBufferEnd()); }
(const unsigned char *)New->Buffer->getBufferEnd());
Result = NewlyLoaded;
}
if (ImportedBy) {
ModuleEntry->ImportedBy.insert(ImportedBy);
@ -93,8 +108,9 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
ModuleEntry->DirectlyImported = true;
}
return std::make_pair(ModuleEntry, NewModule);
Module = ModuleEntry;
return NewModule? NewlyLoaded : AlreadyLoaded;
}
namespace {
@ -113,7 +129,8 @@ namespace {
};
}
void ModuleManager::removeModules(ModuleIterator first, ModuleIterator last) {
void ModuleManager::removeModules(ModuleIterator first, ModuleIterator last,
ModuleMap *modMap) {
if (first == last)
return;
@ -129,6 +146,14 @@ void ModuleManager::removeModules(ModuleIterator first, ModuleIterator last) {
// Delete the modules and erase them from the various structures.
for (ModuleIterator victim = first; victim != last; ++victim) {
Modules.erase((*victim)->File);
FileMgr.invalidateCache((*victim)->File);
if (modMap) {
StringRef ModuleName = llvm::sys::path::stem((*victim)->FileName);
if (Module *mod = modMap->findModule(ModuleName)) {
mod->setASTFile(0);
}
}
delete *victim;
}
@ -151,18 +176,8 @@ void ModuleManager::updateModulesInCommonWithGlobalIndex() {
return;
// Collect the set of modules known to the global index.
SmallVector<const FileEntry *, 16> KnownModules;
GlobalIndex->getKnownModules(KnownModules);
// Map those modules to AST files known to the module manager.
for (unsigned I = 0, N = KnownModules.size(); I != N; ++I) {
llvm::DenseMap<const FileEntry *, ModuleFile *>::iterator Known
= Modules.find(KnownModules[I]);
if (Known == Modules.end())
continue;
ModulesInCommonWithGlobalIndex.push_back(Known->second);
}
GlobalIndex->noteAdditionalModulesLoaded();
GlobalIndex->getKnownModules(ModulesInCommonWithGlobalIndex);
}
ModuleManager::VisitState *ModuleManager::allocateVisitState() {
@ -186,6 +201,9 @@ void ModuleManager::returnVisitState(VisitState *State) {
void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {
GlobalIndex = Index;
if (GlobalIndex) {
GlobalIndex->setResolver(this);
}
updateModulesInCommonWithGlobalIndex();
}
@ -201,7 +219,7 @@ ModuleManager::~ModuleManager() {
void
ModuleManager::visit(bool (*Visitor)(ModuleFile &M, void *UserData),
void *UserData,
llvm::SmallPtrSet<const FileEntry *, 4> *ModuleFilesHit) {
llvm::SmallPtrSet<ModuleFile *, 4> *ModuleFilesHit) {
// If the visitation order vector is the wrong size, recompute the order.
if (VisitOrder.size() != Chain.size()) {
unsigned N = size();
@ -268,7 +286,7 @@ ModuleManager::visit(bool (*Visitor)(ModuleFile &M, void *UserData),
for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
{
ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
if (!ModuleFilesHit->count(M->File))
if (!ModuleFilesHit->count(M))
State->VisitNumber[M->Index] = VisitNumber;
}
}
@ -354,6 +372,52 @@ void ModuleManager::visitDepthFirst(bool (*Visitor)(ModuleFile &M, bool Preorder
}
}
bool ModuleManager::lookupModuleFile(StringRef FileName,
off_t ExpectedSize,
time_t ExpectedModTime,
const FileEntry *&File) {
File = FileMgr.getFile(FileName, /*openFile=*/false, /*cacheFailure=*/false);
if (!File && FileName != "-") {
return false;
}
if ((ExpectedSize && ExpectedSize != File->getSize()) ||
(ExpectedModTime && ExpectedModTime != File->getModificationTime())) {
return true;
}
return false;
}
bool ModuleManager::resolveModuleFileName(StringRef FileName,
off_t ExpectedSize,
time_t ExpectedModTime,
ModuleFile *&File) {
File = 0;
// Look for the file entry corresponding to this name.
const FileEntry *F;
if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, F))
return true;
// If there is no file, we've succeeded (trivially).
if (!F)
return false;
// Determine whether we have a module file associated with this file entry.
llvm::DenseMap<const FileEntry *, ModuleFile *>::iterator Known
= Modules.find(F);
if (Known == Modules.end()) {
// We don't know about this module file; invalidate the cache.
FileMgr.invalidateCache(F);
return false;
}
File = Known->second;
return false;
}
#ifndef NDEBUG
namespace llvm {
template<>

Просмотреть файл

@ -1,2 +1,3 @@
#include "A.h"
@import A;
int getB();

Просмотреть файл

@ -1,2 +1,5 @@
module A { header "A.h" }
module B { header "B.h" }
module B {
header "B.h"
export *
}

Просмотреть файл

@ -6,12 +6,17 @@
// RUN: cp %S/Inputs/Modified/A.h %t/include
// RUN: cp %S/Inputs/Modified/B.h %t/include
// RUN: cp %S/Inputs/Modified/module.map %t/include
// RUN: %clang_cc1 -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// expected-no-diagnostics
// RUN: %clang_cc1 -fdisable-module-hash -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// RUN: echo '' >> %t/include/B.h
// RUN: %clang_cc1 -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// RUN: %clang_cc1 -fdisable-module-hash -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// RUN: echo 'int getA(); int getA2();' > %t/include/A.h
// RUN: %clang_cc1 -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// RUN: %clang_cc1 -fdisable-module-hash -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// RUN: rm %t/cache/A.pcm
// RUN: %clang_cc1 -fdisable-module-hash -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// RUN: touch %t/cache/A.pcm
// RUN: %clang_cc1 -fdisable-module-hash -fmodules-cache-path=%t/cache -fmodules -I %t/include %s -verify
// expected-no-diagnostics
// FIXME: It is intended to suppress this on win32.
// REQUIRES: ansi-escape-sequences