Comment parsing: allow newlines between \param, direction specification (e.g.,

[in]), parameter name and description paragraph.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@160682 91177308-0d34-0410-b5e6-96231b3b80d8

lib/AST/CommentParser.cpp

@@ -20,8 +20,14 @@ namespace comments {
 class TextTokenRetokenizer {
   llvm::BumpPtrAllocator &Allocator;
   Parser &P;
+
+  /// This flag is set when there are no more tokens we can fetch from lexer.
+  bool NoMoreInterestingTokens;
+
+  /// Token buffer: tokens we have processed and lookahead.
   SmallVector<Token, 16> Toks;
 
+  /// A position in \c Toks.
   struct Position {
     unsigned CurToken;
     const char *BufferStart;
@@ -65,10 +71,11 @@ class TextTokenRetokenizer {
     Pos.BufferPtr++;
     if (Pos.BufferPtr == Pos.BufferEnd) {
       Pos.CurToken++;
-      if (isEnd() && addToken()) {
-        assert(!isEnd());
-        setupBuffer();
-      }
+      if (isEnd() && !addToken())
+        return;
+
+      assert(!isEnd());
+      setupBuffer();
     }
   }
 
@@ -76,9 +83,24 @@ class TextTokenRetokenizer {
   /// Returns true on success, false if there are no interesting tokens to
   /// fetch from lexer.
   bool addToken() {
-    if (P.Tok.isNot(tok::text))
+    if (NoMoreInterestingTokens)
       return false;
 
+    if (P.Tok.is(tok::newline)) {
+      // If we see a single newline token between text tokens, skip it.
+      Token Newline = P.Tok;
+      P.consumeToken();
+      if (P.Tok.isNot(tok::text)) {
+        P.putBack(Newline);
+        NoMoreInterestingTokens = true;
+        return false;
+      }
+    }
+    if (P.Tok.isNot(tok::text)) {
+      NoMoreInterestingTokens = true;
+      return false;
+    }
+
     Toks.push_back(P.Tok);
     P.consumeToken();
     if (Toks.size() == 1)
@@ -117,7 +139,7 @@ class TextTokenRetokenizer {
 
 public:
   TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
-      Allocator(Allocator), P(P) {
+      Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
     Pos.CurToken = 0;
     addToken();
   }

unittests/AST/CommentParser.cpp

@@ -205,10 +205,15 @@ template <typename T>
         << " direction, "
            "expected " << (IsDirectionExplicit ? "explicit" : "implicit");
 
+  if (!PCC->hasParamName())
+    return ::testing::AssertionFailure()
+        << "ParamCommandComment has no parameter name";
+
   StringRef ActualParamName = PCC->getParamName();
   if (ActualParamName != ParamName)
     return ::testing::AssertionFailure()
-        << "ParamCommandComment has name \"" << ActualParamName.str() << "\", "
+        << "ParamCommandComment has parameter name \"" << ActualParamName.str()
+        << "\", "
            "expected \"" << ParamName.str() << "\"";
 
   Paragraph = PCC->getParagraph();
@@ -672,69 +677,102 @@ TEST_F(CommentParserTest, Paragraph4) {
 }
 
 TEST_F(CommentParserTest, ParamCommand1) {
-  const char *Source =
-    "// \\param aaa Bbb\n";
+  const char *Sources[] = {
+    "// \\param aaa Bbb\n",
+    "// \\param\n"
+    "//     aaa Bbb\n",
+    "// \\param \n"
+    "//     aaa Bbb\n",
+    "// \\param aaa\n"
+    "// Bbb\n"
+  };
 
-  FullComment *FC = parseString(Source);
-  ASSERT_TRUE(HasChildCount(FC, 2));
+  for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
+    FullComment *FC = parseString(Sources[i]);
+    ASSERT_TRUE(HasChildCount(FC, 2));
 
-  ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
-  {
-    ParamCommandComment *PCC;
-    ParagraphComment *PC;
-    ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
-                                  ParamCommandComment::In,
-                                  /* IsDirectionExplicit = */ false,
-                                  "aaa", PC));
-    ASSERT_TRUE(HasChildCount(PCC, 1));
-    ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+    {
+      ParamCommandComment *PCC;
+      ParagraphComment *PC;
+      ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
+                                    ParamCommandComment::In,
+                                    /* IsDirectionExplicit = */ false,
+                                    "aaa", PC));
+      ASSERT_TRUE(HasChildCount(PCC, 1));
+      ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    }
   }
 }
 
 TEST_F(CommentParserTest, ParamCommand2) {
-  const char *Source =
-    "// \\param [in] aaa Bbb\n";
+  const char *Sources[] = {
+    "// \\param [in] aaa Bbb\n",
+    "// \\param\n"
+    "//     [in] aaa Bbb\n",
+    "// \\param [in]\n"
+    "//     aaa Bbb\n",
+    "// \\param [in] aaa\n"
+    "// Bbb\n",
+  };
 
-  FullComment *FC = parseString(Source);
-  ASSERT_TRUE(HasChildCount(FC, 2));
+  for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
+    FullComment *FC = parseString(Sources[i]);
+    ASSERT_TRUE(HasChildCount(FC, 2));
 
-  ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
-  {
-    ParamCommandComment *PCC;
-    ParagraphComment *PC;
-    ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
-                                  ParamCommandComment::In,
-                                  /* IsDirectionExplicit = */ true,
-                                  "aaa", PC));
-    ASSERT_TRUE(HasChildCount(PCC, 1));
-    ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+    {
+      ParamCommandComment *PCC;
+      ParagraphComment *PC;
+      ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
+                                    ParamCommandComment::In,
+                                    /* IsDirectionExplicit = */ true,
+                                    "aaa", PC));
+      ASSERT_TRUE(HasChildCount(PCC, 1));
+      ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    }
   }
 }
 
 TEST_F(CommentParserTest, ParamCommand3) {
-  const char *Source =
-    "// \\param [out] aaa Bbb\n";
+  const char *Sources[] = {
+    "// \\param [out] aaa Bbb\n",
+    "// \\param\n"
+    "//     [out] aaa Bbb\n",
+    "// \\param [out]\n"
+    "//     aaa Bbb\n",
+    "// \\param [out] aaa\n"
+    "// Bbb\n",
+  };
 
-  FullComment *FC = parseString(Source);
-  ASSERT_TRUE(HasChildCount(FC, 2));
+  for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
+    FullComment *FC = parseString(Sources[i]);
+    ASSERT_TRUE(HasChildCount(FC, 2));
 
-  ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
-  {
-    ParamCommandComment *PCC;
-    ParagraphComment *PC;
-    ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
-                                  ParamCommandComment::Out,
-                                  /* IsDirectionExplicit = */ true,
-                                  "aaa", PC));
-    ASSERT_TRUE(HasChildCount(PCC, 1));
-    ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " "));
+    {
+      ParamCommandComment *PCC;
+      ParagraphComment *PC;
+      ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param",
+                                    ParamCommandComment::Out,
+                                    /* IsDirectionExplicit = */ true,
+                                    "aaa", PC));
+      ASSERT_TRUE(HasChildCount(PCC, 1));
+      ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb"));
+    }
   }
 }
 
 TEST_F(CommentParserTest, ParamCommand4) {
   const char *Sources[] = {
     "// \\param [in,out] aaa Bbb\n",
-    "// \\param [in, out] aaa Bbb\n"
+    "// \\param [in, out] aaa Bbb\n",
+    "// \\param [in,\n"
+    "//     out] aaa Bbb\n",
+    "// \\param [in,out]\n"
+    "//     aaa Bbb\n",
+    "// \\param [in,out] aaa\n"
+    "// Bbb\n"
   };
 
   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {