[modules] Add -cc1 flag -fmodules-embed-all-files.

This flag causes all files that were read by the compilation to be embedded
into a produced module file. This is useful for distributed build systems that
use an include scanning system to determine which files are "needed" by a
compilation, and only provide those files to remote compilation workers. Since
using a module can require any file that is part of that module (or anything it
transitively includes), files that are not found by an include scanner can be
required in a regular build using explicit modules. With this flag, only files
that are actually referenced by transitively-#included files are required to be
present on the build machine.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@253950 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Basic/SourceManager.h

@@ -566,6 +566,11 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// (likely to change while trying to use them). Defaults to false.
   bool UserFilesAreVolatile;
 
+  /// \brief True if all files read during this compilation should be treated
+  /// as transient (may not be present in later compilations using a module
+  /// file created from this compilation). Defaults to false.
+  bool FilesAreTransient;
+
   struct OverriddenFilesInfoTy {
     /// \brief Files that have been overridden with the contents from another
     /// file.
@@ -864,6 +869,12 @@ public:
   /// the module is used).
   void embedFileContentsInModule(const FileEntry *SourceFile);
 
+  /// \brief Request that all files that are read during this compilation be
+  /// written to any created module file.
+  void setEmbedAllFileContentsInModule(bool Embed) {
+    FilesAreTransient = Embed;
+  }
+
   //===--------------------------------------------------------------------===//
   // FileID manipulation methods.
   //===--------------------------------------------------------------------===//

include/clang/Driver/CC1Options.td

@@ -381,6 +381,9 @@ def fmodules_embed_file_EQ : Joined<["-"], "fmodules-embed-file=">,
   MetaVarName<"<file>">,
   HelpText<"Embed the contents of the specified file into the module file "
            "being compiled.">;
+def fmodules_embed_all_files : Joined<["-"], "fmodules-embed-all-files">,
+  HelpText<"Embed the contents of all files read by this compilation into "
+           "the produced module file.">;
 def fmodules_local_submodule_visibility :
   Flag<["-"], "fmodules-local-submodule-visibility">,
   HelpText<"Enforce name visibility rules across submodules of the same "

include/clang/Frontend/FrontendOptions.h

@@ -150,6 +150,8 @@ public:
                                            ///< dumps in AST dumps.
   unsigned BuildingImplicitModule : 1;     ///< Whether we are performing an
                                            ///< implicit module build.
+  unsigned ModulesEmbedAllFiles : 1;       ///< Whether we should embed all used
+                                           ///< files into the PCM file.
 
   CodeCompleteOptions CodeCompleteOpts;
 
@@ -272,7 +274,7 @@ public:
     FixToTemporaries(false), ARCMTMigrateEmitARCErrors(false),
     SkipFunctionBodies(false), UseGlobalModuleIndex(true),
     GenerateGlobalModuleIndex(true), ASTDumpDecls(false), ASTDumpLookups(false),
-    BuildingImplicitModule(false),
+    BuildingImplicitModule(false), ModulesEmbedAllFiles(false),
     ARCMTAction(ARCMT_None), ObjCMTAction(ObjCMT_None),
     ProgramAction(frontend::ParseSyntaxOnly)
   {}

lib/Basic/SourceManager.cpp

@@ -361,7 +361,7 @@ LineTableInfo &SourceManager::getLineTable() {
 SourceManager::SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
                              bool UserFilesAreVolatile)
   : Diag(Diag), FileMgr(FileMgr), OverridenFilesKeepOriginalName(true),
-    UserFilesAreVolatile(UserFilesAreVolatile),
+    UserFilesAreVolatile(UserFilesAreVolatile), FilesAreTransient(false),
     ExternalSLocEntries(nullptr), LineTable(nullptr), NumLinearScans(0),
     NumBinaryProbes(0) {
   clearIDTables();
@@ -439,6 +439,7 @@ SourceManager::getOrCreateContentCache(const FileEntry *FileEnt,
   }
 
   Entry->IsSystemFile = isSystemFile;
+  Entry->BufferOverridden = FilesAreTransient;
 
   return Entry;
 }

lib/Frontend/CompilerInvocation.cpp

@@ -1016,6 +1016,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.ModuleMapFiles = Args.getAllArgValues(OPT_fmodule_map_file);
   Opts.ModuleFiles = Args.getAllArgValues(OPT_fmodule_file);
   Opts.ModulesEmbedFiles = Args.getAllArgValues(OPT_fmodules_embed_file_EQ);
+  Opts.ModulesEmbedAllFiles = Args.hasArg(OPT_fmodules_embed_all_files);
 
   Opts.CodeCompleteOpts.IncludeMacros
     = Args.hasArg(OPT_code_completion_macros);

lib/Frontend/FrontendActions.cpp

@@ -299,6 +299,8 @@ bool GenerateModuleAction::BeginSourceFileAction(CompilerInstance &CI,
     else
       CI.getDiagnostics().Report(diag::err_modules_embed_file_not_found) << F;
   }
+  if (CI.getFrontendOpts().ModulesEmbedAllFiles)
+    CI.getSourceManager().setEmbedAllFileContentsInModule(true);
 
   // If we're being run from the command-line, the module build stack will not
   // have been filled in yet, so complete it now in order to allow us to detect

test/Modules/explicit-build-missing-files.cpp

@@ -13,6 +13,9 @@
 // RUN: %clang_cc1 -fmodules -I %t -emit-module -fmodule-name=a -x c++ %t/modulemap -o %t/a.pcm \
 // RUN:            -fmodule-map-file=%t/other.modulemap \
 // RUN:            -fmodules-embed-file=%t/modulemap -fmodules-embed-file=%t/other.modulemap
+// RUN: %clang_cc1 -fmodules -I %t -emit-module -fmodule-name=a -x c++ %t/modulemap -o %t/b.pcm \
+// RUN:            -fmodule-map-file=%t/other.modulemap \
+// RUN:            -fmodules-embed-all-files
 // RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s
 // RUN: not %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -DERRORS 2>&1 | FileCheck %s
 // RUN: rm %t/modulemap
@@ -27,9 +30,11 @@
 // RUN: not %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -DERRORS 2>&1 | FileCheck %s
 // RUN: rm %t/b.h
 // RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s
+// RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/b.pcm %s
 // RUN: not %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -DERRORS 2>&1 | FileCheck %s --check-prefix=MISSING-B
 // RUN: rm %t/a.h
 // RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -verify
+// RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/b.pcm %s -verify
 
 // Oftentimes on Windows there are open handles, and deletion will fail.
 // REQUIRES: can-remove-opened-file
@@ -46,6 +51,6 @@ int y = a2<int>;
 // MISSING-B-NOT: please delete the module cache
 #endif
 
-// RUN: not %clang_cc1 -fmodules -I %t -emit-module -fmodule-name=a -x c++ /dev/null -o %t/a.pcm \
+// RUN: not %clang_cc1 -fmodules -I %t -emit-module -fmodule-name=a -x c++ /dev/null -o %t/c.pcm \
 // RUN:                -fmodules-embed-file=%t/does-not-exist 2>&1 | FileCheck %s --check-prefix=MISSING-EMBED
 // MISSING-EMBED: fatal error: file '{{.*}}does-not-exist' specified by '-fmodules-embed-file=' not found