allow the HandlerComment callback to push tokens into the

preprocessor.  This could be used by an OpenMP implementation
or something.  Patch by Abramo Bagnara!


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@93795 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Lex/Preprocessor.h

@@ -884,7 +884,9 @@ public:
   void HandlePragmaSystemHeader(Token &SysHeaderTok);
   void HandlePragmaDependency(Token &DependencyTok);
   void HandlePragmaComment(Token &CommentTok);
-  void HandleComment(SourceRange Comment);
+  // Return true and store the first token only if any CommentHandler
+  // has inserted some tokens and getCommentRetentionState() is false.
+  bool HandleComment(Token &Token, SourceRange Comment);
 };
 
 /// \brief Abstract base class that describes a handler that will receive
@@ -893,7 +895,9 @@ class CommentHandler {
 public:
   virtual ~CommentHandler();
 
-  virtual void HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
+  // The handler shall return true if it has pushed any tokens
+  // to be read using e.g. EnterToken or EnterTokenStream.
+  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
 };
 
 }  // end namespace clang

lib/Lex/Lexer.cpp

@@ -902,8 +902,10 @@ bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
 
 // SkipBCPLComment - We have just read the // characters from input.  Skip until
 // we find the newline character thats terminate the comment.  Then update
-/// BufferPtr and return.  If we're in KeepCommentMode, this will form the token
+/// BufferPtr and return.
-/// and return true.
+///
+/// If we're in KeepCommentMode or any CommentHandler has inserted
+/// some tokens, this will store the first token and return true.
 bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
   // If BCPL comments aren't explicitly enabled for this language, emit an
   // extension warning.
@@ -980,9 +982,12 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
   } while (C != '\n' && C != '\r');
 
   // Found but did not consume the newline.
-  if (PP)
+  if (PP && PP->HandleComment(Result,
-    PP->HandleComment(SourceRange(getSourceLocation(BufferPtr),
+                              SourceRange(getSourceLocation(BufferPtr),
-                                  getSourceLocation(CurPtr)));
+                                          getSourceLocation(CurPtr)))) {
+    BufferPtr = CurPtr;
+    return true; // A token has to be returned.
+  }
 
   // If we are returning comments as tokens, return this comment as a token.
   if (inKeepCommentMode())
@@ -1108,8 +1113,8 @@ static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
 /// happen is the comment could end with an escaped newline between the */ end
 /// of comment.
 ///
-/// If KeepCommentMode is enabled, this forms a token from the comment and
+/// If we're in KeepCommentMode or any CommentHandler has inserted
-/// returns true.
+/// some tokens, this will store the first token and return true.
 bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
   // Scan one character past where we should, looking for a '/' character.  Once
   // we find it, check to see if it was preceeded by a *.  This common
@@ -1226,9 +1231,12 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
     C = *CurPtr++;
   }
 
-  if (PP)
+  if (PP && PP->HandleComment(Result,
-    PP->HandleComment(SourceRange(getSourceLocation(BufferPtr),
+                              SourceRange(getSourceLocation(BufferPtr),
-                                  getSourceLocation(CurPtr)));
+                                          getSourceLocation(CurPtr)))) {
+    BufferPtr = CurPtr;
+    return true; // A token has to be returned.
+  }
 
   // If we are returning comments as tokens, return this comment as a token.
   if (inKeepCommentMode()) {
@@ -1606,10 +1614,12 @@ LexNextToken:
     // too (without going through the big switch stmt).
     if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
         Features.BCPLComment) {
-      SkipBCPLComment(Result, CurPtr+2);
+      if (SkipBCPLComment(Result, CurPtr+2))
+        return; // There is a token to return.
       goto SkipIgnoredUnits;
     } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
-      SkipBlockComment(Result, CurPtr+2);
+      if (SkipBlockComment(Result, CurPtr+2))
+        return; // There is a token to return.
       goto SkipIgnoredUnits;
     } else if (isHorizontalWhitespace(*CurPtr)) {
       goto SkipHorizontalWhitespace;
@@ -1795,7 +1805,7 @@ LexNextToken:
       if (Features.BCPLComment ||
           getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') {
         if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
-          return; // KeepCommentMode
+          return; // There is a token to return.
 
         // It is common for the tokens immediately after a // comment to be
         // whitespace (indentation for the next line).  Instead of going through
@@ -1806,7 +1816,7 @@ LexNextToken:
 
     if (Char == '*') {  // /**/ comment.
       if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
-        return; // KeepCommentMode
+        return; // There is a token to return.
       goto LexNextToken;   // GCC isn't tail call eliminating.
     }
 

lib/Lex/Preprocessor.cpp

@@ -583,11 +583,18 @@ void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
   CommentHandlers.erase(Pos);
 }
 
-void Preprocessor::HandleComment(SourceRange Comment) {
+bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
+  bool AnyPendingTokens = false;
   for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
        HEnd = CommentHandlers.end();
-       H != HEnd; ++H)
+       H != HEnd; ++H) {
-    (*H)->HandleComment(*this, Comment);
+    if ((*H)->HandleComment(*this, Comment))
+      AnyPendingTokens = true;
+  }
+  if (!AnyPendingTokens || getCommentRetentionState())
+    return false;
+  Lex(result);
+  return true;
 }
 
 CommentHandler::~CommentHandler() { }

lib/Parse/Parser.cpp

@@ -29,8 +29,9 @@ class ActionCommentHandler : public CommentHandler {
 public:
   explicit ActionCommentHandler(Action &Actions) : Actions(Actions) { }
 
-  virtual void HandleComment(Preprocessor &PP, SourceRange Comment) {
+  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) {
     Actions.ActOnComment(Comment);
+    return false;
   }
 };