Introduce a new libclang API, clang_reparseTranslationUnit(), which

reparses an already-parsed translation unit. At the moment it's just a
convenience function, but we hope to use it for performance
optimizations.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@108756 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang-c/Index.h

@@ -639,6 +639,44 @@ CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnit(CXIndex,
  */
 CINDEX_LINKAGE void clang_disposeTranslationUnit(CXTranslationUnit);
 
+/**
+ * \brief Reparse the source files that produced this translation unit.
+ *
+ * This routine can be used to re-parse the source files that originally
+ * created the given translation unit, for example because those source files
+ * have changed (either on disk or as passed via \p unsaved_files). The
+ * source code will be reparsed with the same command-line options as it
+ * was originally parsed. 
+ *
+ * Reparsing a translation unit invalidates all cursors and source locations
+ * that refer into that translation unit. This makes reparsing a translation
+ * unit semantically equivalent to destroying the translation unit and then
+ * creating a new translation unit with the same command-line arguments.
+ * However, it may be more efficient to reparse a translation 
+ * unit using this routine.
+ *
+ * \param TU The translation unit whose contents will be re-parsed. The
+ * translation unit must originally have been built with 
+ * \c clang_createTranslationUnitFromSourceFile().
+ *
+ * \param num_unsaved_files The number of unsaved file entries in \p
+ * unsaved_files.
+ *
+ * \param unsaved_files The files that have not yet been saved to disk
+ * but may be required for parsing, including the contents of
+ * those files.  The contents and name of these files (as specified by
+ * CXUnsavedFile) are copied when necessary, so the client only needs to
+ * guarantee their validity until the call to this function returns.
+ * 
+ * \returns 0 if the sources could be reparsed. A non-zero value will be
+ * returned if reparsing was impossible, such that the translation unit is
+ * invalid. In such cases, the only valid call for \p TU is 
+ * \c clang_disposeTranslationUnit(TU).
+ */
+CINDEX_LINKAGE int clang_reparseTranslationUnit(CXTranslationUnit TU,
+                                                unsigned num_unsaved_files,
+                                          struct CXUnsavedFile *unsaved_files);
+  
 /**
  * @}
  */

include/clang/Basic/Diagnostic.h

@@ -404,6 +404,10 @@ public:
     ArgToStringCookie = Cookie;
   }
 
+  /// \brief Reset the state of the diagnostic object to its initial 
+  /// configuration.
+  void Reset();
+  
   //===--------------------------------------------------------------------===//
   // Diagnostic classification and reporting interfaces.
   //

include/clang/Frontend/ASTUnit.h

@@ -70,6 +70,9 @@ private:
   // FIXME: This is temporary; eventually, CIndex will always do this.
   bool                              OnlyLocalDecls;
 
+  /// \brief Whether to capture any diagnostics produced.
+  bool CaptureDiagnostics;
+  
   /// Track whether the main file was loaded from an AST or not.
   bool MainFileIsAST;
 
@@ -120,6 +123,9 @@ private:
   
   explicit ASTUnit(bool MainFileIsAST);
 
+  void CleanTemporaryFiles();
+  bool Parse();
+  
 public:
   class ConcurrencyCheck {
     volatile ASTUnit &Self;
@@ -259,6 +265,14 @@ public:
                                       RemappedFile *RemappedFiles = 0,
                                       unsigned NumRemappedFiles = 0,
                                       bool CaptureDiagnostics = false);
+  
+  /// \brief Reparse the source files using the same command-line options that
+  /// were originally used to produce this translation unit.
+  ///
+  /// \returns True if a failure occurred that causes the ASTUnit not to
+  /// contain any translation-unit information, false otherwise.  
+  bool Reparse(RemappedFile *RemappedFiles = 0,
+               unsigned NumRemappedFiles = 0);
 };
 
 } // namespace clang

include/clang/Frontend/PreprocessorOptions.h

@@ -95,6 +95,10 @@ public:
   void addRemappedFile(llvm::StringRef From, const llvm::MemoryBuffer * To) {
     RemappedFileBuffers.push_back(std::make_pair(From, To));
   }
+  void clearRemappedFiles() {
+    RemappedFiles.clear();
+    RemappedFileBuffers.clear();
+  }
 };
 
 } // end namespace clang

lib/Basic/Diagnostic.cpp

@@ -244,35 +244,10 @@ static void DummyArgToStringFn(Diagnostic::ArgumentKind AK, intptr_t QT,
 
 
 Diagnostic::Diagnostic(DiagnosticClient *client) : Client(client) {
-  AllExtensionsSilenced = 0;
-  IgnoreAllWarnings = false;
-  WarningsAsErrors = false;
-  ErrorsAsFatal = false;
-  SuppressSystemWarnings = false;
-  SuppressAllDiagnostics = false;
-  ShowOverloads = Ovl_All;
-  ExtBehavior = Ext_Ignore;
-
-  ErrorOccurred = false;
-  FatalErrorOccurred = false;
-  ErrorLimit = 0;
-  TemplateBacktraceLimit = 0;
-
-  NumWarnings = 0;
-  NumErrors = 0;
-  NumErrorsSuppressed = 0;
-  CustomDiagInfo = 0;
-  CurDiagID = ~0U;
-  LastDiagLevel = Ignored;
-
   ArgToStringFn = DummyArgToStringFn;
   ArgToStringCookie = 0;
 
-  DelayedDiagID = 0;
-
-  // Set all mappings to 'unset'.
-  DiagMappings BlankDiags(diag::DIAG_UPPER_LIMIT/2, 0);
-  DiagMappingsStack.push_back(BlankDiags);
+  Reset();
 }
 
 Diagnostic::~Diagnostic() {
@@ -335,6 +310,36 @@ bool Diagnostic::isBuiltinExtensionDiag(unsigned DiagID,
   return true;
 }
 
+void Diagnostic::Reset() {
+  AllExtensionsSilenced = 0;
+  IgnoreAllWarnings = false;
+  WarningsAsErrors = false;
+  ErrorsAsFatal = false;
+  SuppressSystemWarnings = false;
+  SuppressAllDiagnostics = false;
+  ShowOverloads = Ovl_All;
+  ExtBehavior = Ext_Ignore;
+  
+  ErrorOccurred = false;
+  FatalErrorOccurred = false;
+  ErrorLimit = 0;
+  TemplateBacktraceLimit = 0;
+  
+  NumWarnings = 0;
+  NumErrors = 0;
+  NumErrorsSuppressed = 0;
+  CustomDiagInfo = 0;
+  CurDiagID = ~0U;
+  LastDiagLevel = Ignored;
+  DelayedDiagID = 0;
+
+  // Set all mappings to 'unset'.
+  while (!DiagMappingsStack.empty())
+    DiagMappingsStack.pop_back();
+  
+  DiagMappings BlankDiags(diag::DIAG_UPPER_LIMIT/2, 0);
+  DiagMappingsStack.push_back(BlankDiags);
+}
 
 /// getDescription - Given a diagnostic ID, return a description of the
 /// issue.

lib/Frontend/ASTUnit.cpp

@@ -36,12 +36,18 @@
 using namespace clang;
 
 ASTUnit::ASTUnit(bool _MainFileIsAST)
-  : MainFileIsAST(_MainFileIsAST), ConcurrencyCheckValue(CheckUnlocked) { }
+  : CaptureDiagnostics(false), MainFileIsAST(_MainFileIsAST), 
+    ConcurrencyCheckValue(CheckUnlocked) { }
 
 ASTUnit::~ASTUnit() {
   ConcurrencyCheckValue = CheckLocked;
+  CleanTemporaryFiles();
+}
+
+void ASTUnit::CleanTemporaryFiles() {
   for (unsigned I = 0, N = TemporaryFiles.size(); I != N; ++I)
     TemporaryFiles[I].eraseFromDisk();
+  TemporaryFiles.clear();
 }
 
 namespace {
@@ -156,7 +162,8 @@ ASTUnit *ASTUnit::LoadFromPCHFile(const std::string &Filename,
     DiagnosticOptions DiagOpts;
     Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
   }
-  
+
+  AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->OnlyLocalDecls = OnlyLocalDecls;
   AST->Diagnostics = Diags;
   AST->FileMgr.reset(new FileManager);
@@ -298,41 +305,38 @@ public:
 
 }
 
-ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
-                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
-                                             bool OnlyLocalDecls,
-                                             bool CaptureDiagnostics) {
+/// Parse the source file into a translation unit using the given compiler
+/// invocation, replacing the current translation unit.
+///
+/// \returns True if a failure occurred that causes the ASTUnit not to
+/// contain any translation-unit information, false otherwise.
+bool ASTUnit::Parse() {
+  if (!Invocation.get())
+    return true;
+  
   // Create the compiler instance to use for building the AST.
   CompilerInstance Clang;
-  llvm::OwningPtr<ASTUnit> AST;
-  llvm::OwningPtr<TopLevelDeclTrackerAction> Act;
-
-  if (!Diags.getPtr()) {
-    // No diagnostics engine was provided, so create our own diagnostics object
-    // with the default options.
-    DiagnosticOptions DiagOpts;
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
-  }
+  Clang.setInvocation(Invocation.take());
+  OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second;
+    
+  // Set up diagnostics.
+  Clang.setDiagnostics(&getDiagnostics());
+  Clang.setDiagnosticClient(getDiagnostics().getClient());
   
-  Clang.setInvocation(CI);
-
-  Clang.setDiagnostics(Diags.getPtr());
-  Clang.setDiagnosticClient(Diags->getClient());
-
   // Create the target instance.
   Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(),
                                                Clang.getTargetOpts()));
   if (!Clang.hasTarget()) {
     Clang.takeDiagnosticClient();
-    return 0;
+    return true;
   }
-
+  
   // Inform the target of the language options.
   //
   // FIXME: We shouldn't need to do this, the target should be immutable once
   // created. This complexity should be lifted elsewhere.
   Clang.getTarget().setForcedLangOptions(Clang.getLangOpts());
-
+  
   assert(Clang.getFrontendOpts().Inputs.size() == 1 &&
          "Invocation must have exactly one source file!");
   assert(Clang.getFrontendOpts().Inputs[0].first != IK_AST &&
@@ -340,52 +344,84 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
   assert(Clang.getFrontendOpts().Inputs[0].first != IK_LLVM_IR &&
          "IR inputs not support here!");
 
-  // Create the AST unit.
-  AST.reset(new ASTUnit(false));
-  AST->Diagnostics = Diags;
-  AST->FileMgr.reset(new FileManager);
-  AST->SourceMgr.reset(new SourceManager(AST->getDiagnostics()));
-  AST->OnlyLocalDecls = OnlyLocalDecls;
-  AST->OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second;
-
+  // Configure the various subsystems.
+  // FIXME: Should we retain the previous file manager?
+  FileMgr.reset(new FileManager);
+  SourceMgr.reset(new SourceManager(getDiagnostics()));
+  Ctx.reset();
+  PP.reset();
+  
+  // Clear out old caches and data.
+  TopLevelDecls.clear();
+  StoredDiagnostics.clear();
+  CleanTemporaryFiles();
+  PreprocessedEntitiesByFile.clear();
+  
   // Capture any diagnostics that would otherwise be dropped.
   CaptureDroppedDiagnostics Capture(CaptureDiagnostics, 
                                     Clang.getDiagnostics(),
-                                    AST->StoredDiagnostics);
-
+                                    StoredDiagnostics);
+  
   // Create a file manager object to provide access to and cache the filesystem.
-  Clang.setFileManager(&AST->getFileManager());
-
+  Clang.setFileManager(&getFileManager());
+  
   // Create the source manager.
-  Clang.setSourceManager(&AST->getSourceManager());
-
-  Act.reset(new TopLevelDeclTrackerAction(*AST));
+  Clang.setSourceManager(&getSourceManager());
+  
+  llvm::OwningPtr<TopLevelDeclTrackerAction> Act;
+  Act.reset(new TopLevelDeclTrackerAction(*this));
   if (!Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second,
                             Clang.getFrontendOpts().Inputs[0].first))
     goto error;
-
+  
   Act->Execute();
-
+  
   // Steal the created target, context, and preprocessor, and take back the
   // source and file managers.
-  AST->Ctx.reset(Clang.takeASTContext());
-  AST->PP.reset(Clang.takePreprocessor());
+  Ctx.reset(Clang.takeASTContext());
+  PP.reset(Clang.takePreprocessor());
   Clang.takeSourceManager();
   Clang.takeFileManager();
-  AST->Target.reset(Clang.takeTarget());
-
+  Target.reset(Clang.takeTarget());
+  
   Act->EndSourceFile();
-
+  
   Clang.takeDiagnosticClient();
-  Clang.takeInvocation();
-
-  AST->Invocation.reset(Clang.takeInvocation());
-  return AST.take();
-
+  
+  Invocation.reset(Clang.takeInvocation());
+  return false;
+  
 error:
   Clang.takeSourceManager();
   Clang.takeFileManager();
   Clang.takeDiagnosticClient();
+  Invocation.reset(Clang.takeInvocation());
+  return true;
+}
+
+
+ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
+                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
+                                             bool OnlyLocalDecls,
+                                             bool CaptureDiagnostics) {
+  if (!Diags.getPtr()) {
+    // No diagnostics engine was provided, so create our own diagnostics object
+    // with the default options.
+    DiagnosticOptions DiagOpts;
+    Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
+  }
+  
+  // Create the AST unit.
+  llvm::OwningPtr<ASTUnit> AST;
+  AST.reset(new ASTUnit(false));
+  AST->Diagnostics = Diags;
+  AST->CaptureDiagnostics = CaptureDiagnostics;
+  AST->OnlyLocalDecls = OnlyLocalDecls;
+  AST->Invocation.reset(CI);
+  
+  if (!AST->Parse())
+    return AST.take();
+  
   return 0;
 }
 
@@ -459,3 +495,19 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   return LoadFromCompilerInvocation(CI.take(), Diags, OnlyLocalDecls,
                                     CaptureDiagnostics);
 }
+
+bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) {
+  if (!Invocation.get())
+    return true;
+  
+  // Clear out the diagnostics state.
+  getDiagnostics().Reset();
+  
+  // Remap files.
+  Invocation->getPreprocessorOpts().clearRemappedFiles();
+  for (unsigned I = 0; I != NumRemappedFiles; ++I)
+    Invocation->getPreprocessorOpts().addRemappedFile(RemappedFiles[I].first,
+                                                       RemappedFiles[I].second);
+  
+  return Parse();
+}

test/Index/cindex-from-source.m

@@ -7,3 +7,6 @@
 // CHECK: cindex-from-source.m:9:1: TypeRef=t0:1:13 Extent=[9:1 - 9:3]
 struct s0 {};
 t0 g0;
+
+// RUN: c-index-test -test-load-source-reparse 5 local %s -include %t.pfx.h > %t
+// RUN: FileCheck %s < %t

tools/c-index-test/c-index-test.c

@@ -558,7 +558,7 @@ int perform_test_load_source(int argc, const char **argv,
   struct CXUnsavedFile *unsaved_files = 0;
   int num_unsaved_files = 0;
   int result;
-
+  
   Idx = clang_createIndex(/* excludeDeclsFromPCH */
                           !strcmp(filter, "local") ? 1 : 0,
                           /* displayDiagnosics=*/1);
@@ -578,6 +578,7 @@ int perform_test_load_source(int argc, const char **argv,
                                                  unsaved_files);
   if (!TU) {
     fprintf(stderr, "Unable to load translation unit!\n");
+    free_remapped_files(unsaved_files, num_unsaved_files);
     clang_disposeIndex(Idx);
     return 1;
   }
@@ -588,6 +589,57 @@ int perform_test_load_source(int argc, const char **argv,
   return result;
 }
 
+int perform_test_reparse_source(int argc, const char **argv, int trials,
+                                const char *filter, CXCursorVisitor Visitor,
+                                PostVisitTU PV) {
+  const char *UseExternalASTs =
+  getenv("CINDEXTEST_USE_EXTERNAL_AST_GENERATION");
+  CXIndex Idx;
+  CXTranslationUnit TU;
+  struct CXUnsavedFile *unsaved_files = 0;
+  int num_unsaved_files = 0;
+  int result;
+  int trial;
+  
+  Idx = clang_createIndex(/* excludeDeclsFromPCH */
+                          !strcmp(filter, "local") ? 1 : 0,
+                          /* displayDiagnosics=*/1);
+  
+  if (UseExternalASTs && strlen(UseExternalASTs))
+    clang_setUseExternalASTGeneration(Idx, 1);
+  
+  if (parse_remapped_files(argc, argv, 0, &unsaved_files, &num_unsaved_files)) {
+    clang_disposeIndex(Idx);
+    return -1;
+  }
+  
+  TU = clang_createTranslationUnitFromSourceFile(Idx, 0,
+                                                 argc - num_unsaved_files,
+                                                 argv + num_unsaved_files,
+                                                 num_unsaved_files,
+                                                 unsaved_files);
+  if (!TU) {
+    fprintf(stderr, "Unable to load translation unit!\n");
+    free_remapped_files(unsaved_files, num_unsaved_files);
+    clang_disposeIndex(Idx);
+    return 1;
+  }
+  
+  for (trial = 0; trial < trials; ++trial) {
+    if (clang_reparseTranslationUnit(TU, num_unsaved_files, unsaved_files)) {
+      clang_disposeTranslationUnit(TU);
+      free_remapped_files(unsaved_files, num_unsaved_files);
+      clang_disposeIndex(Idx);
+      return -1;      
+    }
+  }
+  
+  result = perform_test_load(Idx, TU, filter, NULL, Visitor, PV);
+  free_remapped_files(unsaved_files, num_unsaved_files);
+  clang_disposeIndex(Idx);
+  return result;
+}
+
 /******************************************************************************/
 /* Logic for testing clang_getCursor().                                       */
 /******************************************************************************/
@@ -1219,6 +1271,8 @@ static void print_usage(void) {
            "[FileCheck prefix]\n"
     "       c-index-test -test-load-source <symbol filter> {<args>}*\n");
   fprintf(stderr,
+    "       c-index-test -test-load-source-reparse <trials> <symbol filter> "
+    "          {<args>}*\n"
     "       c-index-test -test-load-source-usrs <symbol filter> {<args>}*\n"
     "       c-index-test -test-annotate-tokens=<range> {<args>}*\n"
     "       c-index-test -test-inclusion-stack-source {<args>}*\n"
@@ -1252,6 +1306,14 @@ int main(int argc, const char **argv) {
       return perform_test_load_tu(argv[2], argv[3], argc >= 5 ? argv[4] : 0, I,
                                   NULL);
   }
+  else if (argc >= 5 && strncmp(argv[1], "-test-load-source-reparse", 25) == 0){
+    CXCursorVisitor I = GetVisitor(argv[1] + 25);
+    if (I) {
+      int trials = atoi(argv[2]);
+      return perform_test_reparse_source(argc - 4, argv + 4, trials, argv[3], I, 
+                                         NULL);
+    }
+  }
   else if (argc >= 4 && strncmp(argv[1], "-test-load-source", 17) == 0) {
     CXCursorVisitor I = GetVisitor(argv[1] + 17);
     if (I)

tools/libclang/CIndex.cpp

@@ -1390,6 +1390,25 @@ void clang_disposeTranslationUnit(CXTranslationUnit CTUnit) {
     delete static_cast<ASTUnit *>(CTUnit);
 }
 
+int clang_reparseTranslationUnit(CXTranslationUnit TU,
+                                 unsigned num_unsaved_files,
+                                 struct CXUnsavedFile *unsaved_files) {
+  if (!TU)
+    return 1;
+  
+  llvm::SmallVector<ASTUnit::RemappedFile, 4> RemappedFiles;
+  for (unsigned I = 0; I != num_unsaved_files; ++I) {
+    llvm::StringRef Data(unsaved_files[I].Contents, unsaved_files[I].Length);
+    const llvm::MemoryBuffer *Buffer
+    = llvm::MemoryBuffer::getMemBufferCopy(Data, unsaved_files[I].Filename);
+    RemappedFiles.push_back(std::make_pair(unsaved_files[I].Filename,
+                                           Buffer));
+  }
+  
+  return static_cast<ASTUnit *>(TU)->Reparse(RemappedFiles.data(),
+                                             RemappedFiles.size())? 1 : 0;
+}
+  
 CXString clang_getTranslationUnitSpelling(CXTranslationUnit CTUnit) {
   if (!CTUnit)
     return createCXString("");

tools/libclang/libclang.darwin.exports

@@ -86,6 +86,7 @@ _clang_isReference
 _clang_isStatement
 _clang_isTranslationUnit
 _clang_isUnexposed
+_clang_reparseTranslationUnit
 _clang_setUseExternalASTGeneration
 _clang_tokenize
 _clang_visitChildren

tools/libclang/libclang.exports

@@ -86,6 +86,7 @@ clang_isReference
 clang_isStatement
 clang_isTranslationUnit
 clang_isUnexposed
+clang_reparseTranslationUnit
 clang_setUseExternalASTGeneration
 clang_tokenize
 clang_visitChildren