diff --git a/lib/AST/CommentParser.cpp b/lib/AST/CommentParser.cpp index 3393349eaf..607ace3546 100644 --- a/lib/AST/CommentParser.cpp +++ b/lib/AST/CommentParser.cpp @@ -20,8 +20,14 @@ namespace comments { class TextTokenRetokenizer { llvm::BumpPtrAllocator &Allocator; Parser &P; + + /// This flag is set when there are no more tokens we can fetch from lexer. + bool NoMoreInterestingTokens; + + /// Token buffer: tokens we have processed and lookahead. SmallVector Toks; + /// A position in \c Toks. struct Position { unsigned CurToken; const char *BufferStart; @@ -65,10 +71,11 @@ class TextTokenRetokenizer { Pos.BufferPtr++; if (Pos.BufferPtr == Pos.BufferEnd) { Pos.CurToken++; - if (isEnd() && addToken()) { - assert(!isEnd()); - setupBuffer(); - } + if (isEnd() && !addToken()) + return; + + assert(!isEnd()); + setupBuffer(); } } @@ -76,9 +83,24 @@ class TextTokenRetokenizer { /// Returns true on success, false if there are no interesting tokens to /// fetch from lexer. bool addToken() { - if (P.Tok.isNot(tok::text)) + if (NoMoreInterestingTokens) return false; + if (P.Tok.is(tok::newline)) { + // If we see a single newline token between text tokens, skip it. + Token Newline = P.Tok; + P.consumeToken(); + if (P.Tok.isNot(tok::text)) { + P.putBack(Newline); + NoMoreInterestingTokens = true; + return false; + } + } + if (P.Tok.isNot(tok::text)) { + NoMoreInterestingTokens = true; + return false; + } + Toks.push_back(P.Tok); P.consumeToken(); if (Toks.size() == 1) @@ -117,7 +139,7 @@ class TextTokenRetokenizer { public: TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P): - Allocator(Allocator), P(P) { + Allocator(Allocator), P(P), NoMoreInterestingTokens(false) { Pos.CurToken = 0; addToken(); } diff --git a/unittests/AST/CommentParser.cpp b/unittests/AST/CommentParser.cpp index ed7681d5c5..47433aee2b 100644 --- a/unittests/AST/CommentParser.cpp +++ b/unittests/AST/CommentParser.cpp @@ -205,10 +205,15 @@ template << " direction, " "expected " << (IsDirectionExplicit ? "explicit" : "implicit"); + if (!PCC->hasParamName()) + return ::testing::AssertionFailure() + << "ParamCommandComment has no parameter name"; + StringRef ActualParamName = PCC->getParamName(); if (ActualParamName != ParamName) return ::testing::AssertionFailure() - << "ParamCommandComment has name \"" << ActualParamName.str() << "\", " + << "ParamCommandComment has parameter name \"" << ActualParamName.str() + << "\", " "expected \"" << ParamName.str() << "\""; Paragraph = PCC->getParagraph(); @@ -672,69 +677,102 @@ TEST_F(CommentParserTest, Paragraph4) { } TEST_F(CommentParserTest, ParamCommand1) { - const char *Source = - "// \\param aaa Bbb\n"; + const char *Sources[] = { + "// \\param aaa Bbb\n", + "// \\param\n" + "// aaa Bbb\n", + "// \\param \n" + "// aaa Bbb\n", + "// \\param aaa\n" + "// Bbb\n" + }; - FullComment *FC = parseString(Source); - ASSERT_TRUE(HasChildCount(FC, 2)); + for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) { + FullComment *FC = parseString(Sources[i]); + ASSERT_TRUE(HasChildCount(FC, 2)); - ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " ")); - { - ParamCommandComment *PCC; - ParagraphComment *PC; - ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param", - ParamCommandComment::In, - /* IsDirectionExplicit = */ false, - "aaa", PC)); - ASSERT_TRUE(HasChildCount(PCC, 1)); - ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb")); + ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " ")); + { + ParamCommandComment *PCC; + ParagraphComment *PC; + ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param", + ParamCommandComment::In, + /* IsDirectionExplicit = */ false, + "aaa", PC)); + ASSERT_TRUE(HasChildCount(PCC, 1)); + ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb")); + } } } TEST_F(CommentParserTest, ParamCommand2) { - const char *Source = - "// \\param [in] aaa Bbb\n"; + const char *Sources[] = { + "// \\param [in] aaa Bbb\n", + "// \\param\n" + "// [in] aaa Bbb\n", + "// \\param [in]\n" + "// aaa Bbb\n", + "// \\param [in] aaa\n" + "// Bbb\n", + }; - FullComment *FC = parseString(Source); - ASSERT_TRUE(HasChildCount(FC, 2)); + for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) { + FullComment *FC = parseString(Sources[i]); + ASSERT_TRUE(HasChildCount(FC, 2)); - ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " ")); - { - ParamCommandComment *PCC; - ParagraphComment *PC; - ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param", - ParamCommandComment::In, - /* IsDirectionExplicit = */ true, - "aaa", PC)); - ASSERT_TRUE(HasChildCount(PCC, 1)); - ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb")); + ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " ")); + { + ParamCommandComment *PCC; + ParagraphComment *PC; + ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param", + ParamCommandComment::In, + /* IsDirectionExplicit = */ true, + "aaa", PC)); + ASSERT_TRUE(HasChildCount(PCC, 1)); + ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb")); + } } } TEST_F(CommentParserTest, ParamCommand3) { - const char *Source = - "// \\param [out] aaa Bbb\n"; + const char *Sources[] = { + "// \\param [out] aaa Bbb\n", + "// \\param\n" + "// [out] aaa Bbb\n", + "// \\param [out]\n" + "// aaa Bbb\n", + "// \\param [out] aaa\n" + "// Bbb\n", + }; - FullComment *FC = parseString(Source); - ASSERT_TRUE(HasChildCount(FC, 2)); + for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) { + FullComment *FC = parseString(Sources[i]); + ASSERT_TRUE(HasChildCount(FC, 2)); - ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " ")); - { - ParamCommandComment *PCC; - ParagraphComment *PC; - ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param", - ParamCommandComment::Out, - /* IsDirectionExplicit = */ true, - "aaa", PC)); - ASSERT_TRUE(HasChildCount(PCC, 1)); - ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb")); + ASSERT_TRUE(HasParagraphCommentAt(FC, 0, " ")); + { + ParamCommandComment *PCC; + ParagraphComment *PC; + ASSERT_TRUE(HasParamCommandAt(FC, 1, PCC, "param", + ParamCommandComment::Out, + /* IsDirectionExplicit = */ true, + "aaa", PC)); + ASSERT_TRUE(HasChildCount(PCC, 1)); + ASSERT_TRUE(HasParagraphCommentAt(PCC, 0, " Bbb")); + } } } TEST_F(CommentParserTest, ParamCommand4) { const char *Sources[] = { "// \\param [in,out] aaa Bbb\n", - "// \\param [in, out] aaa Bbb\n" + "// \\param [in, out] aaa Bbb\n", + "// \\param [in,\n" + "// out] aaa Bbb\n", + "// \\param [in,out]\n" + "// aaa Bbb\n", + "// \\param [in,out] aaa\n" + "// Bbb\n" }; for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {